@adhdev/daemon-core 1.0.18-rc.20 → 1.0.18-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli-adapters/provider-cli-shared.d.ts +0 -33
- package/dist/commands/cli-manager.d.ts +0 -9
- package/dist/commands/upgrade-helper.d.ts +0 -1
- package/dist/commands/windows-atomic-upgrade.d.ts +0 -5
- package/dist/config/mesh-json-config.d.ts +0 -62
- package/dist/index.d.ts +2 -4
- package/dist/index.js +3998 -6112
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +3855 -5966
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/coordinator-prompt.d.ts +0 -76
- package/dist/mesh/mesh-active-work.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -28
- package/dist/mesh/mesh-node-identity.d.ts +0 -23
- package/dist/mesh/mesh-refine-gates.d.ts +0 -89
- package/dist/mesh/mesh-remote-event-pull.d.ts +0 -3
- package/dist/mesh/mesh-runtime-store.d.ts +0 -11
- package/dist/mesh/mesh-work-queue.d.ts +1 -24
- package/dist/providers/chat-message-normalization.d.ts +0 -22
- package/dist/providers/cli-provider-instance-types.d.ts +0 -4
- package/dist/providers/cli-provider-instance.d.ts +0 -78
- package/dist/providers/contracts.d.ts +0 -17
- package/dist/providers/provider-schema.d.ts +0 -1
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +0 -14
- package/dist/providers/types/interactive-prompt.d.ts +0 -18
- package/dist/repo-mesh-types.d.ts +6 -38
- package/dist/shared-types.d.ts +2 -4
- package/package.json +3 -3
- package/src/boot/daemon-lifecycle.ts +0 -10
- package/src/cli-adapters/cli-state-engine.ts +1 -17
- package/src/cli-adapters/provider-cli-adapter.ts +6 -2
- package/src/cli-adapters/provider-cli-shared.ts +0 -48
- package/src/commands/cli-manager.ts +8 -72
- package/src/commands/high-family/mesh-coordinator-launch.ts +2 -17
- package/src/commands/high-family/mesh-events.ts +2 -13
- package/src/commands/med-family/mesh-crud.ts +0 -155
- package/src/commands/med-family/mesh-queue.ts +1 -74
- package/src/commands/router-refine.ts +4 -71
- package/src/commands/router.ts +0 -8
- package/src/commands/upgrade-helper.ts +15 -78
- package/src/commands/windows-atomic-upgrade.ts +40 -194
- package/src/config/mesh-json-config.ts +0 -111
- package/src/index.ts +1 -21
- package/src/mesh/coordinator-prompt.ts +11 -258
- package/src/mesh/mesh-active-work.ts +1 -11
- package/src/mesh/mesh-completion-synthesis.ts +1 -12
- package/src/mesh/mesh-event-classify.ts +0 -19
- package/src/mesh/mesh-event-forwarding.ts +6 -64
- package/src/mesh/mesh-events-utils.ts +0 -42
- package/src/mesh/mesh-ledger.ts +0 -77
- package/src/mesh/mesh-node-identity.ts +0 -49
- package/src/mesh/mesh-queue-assignment.ts +11 -210
- package/src/mesh/mesh-reconcile-loop.ts +3 -306
- package/src/mesh/mesh-refine-gates.ts +0 -300
- package/src/mesh/mesh-remote-event-pull.ts +47 -68
- package/src/mesh/mesh-runtime-store.ts +0 -21
- package/src/mesh/mesh-work-queue.ts +2 -85
- package/src/providers/chat-message-normalization.ts +0 -53
- package/src/providers/cli-provider-instance-types.ts +0 -53
- package/src/providers/cli-provider-instance.ts +15 -497
- package/src/providers/contracts.ts +1 -25
- package/src/providers/provider-schema.ts +0 -83
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +0 -23
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +0 -20
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +0 -44
- package/src/providers/types/interactive-prompt.ts +0 -77
- package/src/repo-mesh-types.ts +12 -112
- package/src/shared-types.ts +1 -9
- package/src/status/reporter.ts +0 -1
- package/src/status/snapshot.ts +0 -2
- package/dist/mesh/mesh-disk-retention.d.ts +0 -105
- package/dist/providers/auto-approve-modes.d.ts +0 -14
- package/src/mesh/mesh-disk-retention.ts +0 -370
- package/src/providers/auto-approve-modes.ts +0 -103
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -33,11 +33,6 @@ export type MeshLedgerKind =
|
|
|
33
33
|
| 'task_failed'
|
|
34
34
|
| 'task_stalled'
|
|
35
35
|
| 'task_approval_needed'
|
|
36
|
-
// A worker is parked on an AskUserQuestion multi-choice prompt (waiting_choice) —
|
|
37
|
-
// distinct from task_approval_needed (a yes/no tool-consent modal). The coordinator
|
|
38
|
-
// answers a question with mesh_answer_question, never mesh_approve (mission f1d25e11).
|
|
39
|
-
// payload carries the full InteractivePrompt (promptId + questions + options).
|
|
40
|
-
| 'task_question_pending'
|
|
41
36
|
| 'p2p_dispatch_failed'
|
|
42
37
|
| 'session_launched'
|
|
43
38
|
| 'session_auto_launch'
|
|
@@ -93,14 +88,6 @@ export type MeshLedgerKind =
|
|
|
93
88
|
// payload: { keys: string[], hasDestructive: boolean, result: 'injected'|'refused'|'error',
|
|
94
89
|
// refused?: string, submits?: boolean, confirmDestructive?: boolean }
|
|
95
90
|
| 'key_injection'
|
|
96
|
-
// Disk/worktree retention (mission 86def38d): DETECTION-ONLY signal that a git
|
|
97
|
-
// worktree present on disk has no matching live mesh node — an orphan cleanup
|
|
98
|
-
// candidate. The reconcile loop emits this so the coordinator can decide whether
|
|
99
|
-
// to remove it; retention NEVER auto-deletes a worktree (manual/coordinator-driven).
|
|
100
|
-
// Keyed by worktreePath so a re-emit for the same orphan is idempotent (a prior
|
|
101
|
-
// unresolved entry within the dedupe window suppresses the repeat).
|
|
102
|
-
// payload: { worktreePath, branch?, head?, reason: 'no_matching_live_node', state: 'cleanup_candidate' }
|
|
103
|
-
| 'worktree_cleanup_candidate'
|
|
104
91
|
;
|
|
105
92
|
|
|
106
93
|
export interface MeshLedgerEntry {
|
|
@@ -330,70 +317,6 @@ export const OPERATING_NOTE_DEDUPE_WINDOW = 40;
|
|
|
330
317
|
// a coordinator actually sees while still capping unbounded store growth.
|
|
331
318
|
export const OPERATING_NOTE_KEEP_LATEST = 100;
|
|
332
319
|
|
|
333
|
-
// ─── Operating-note lifecycle: category TTL + expiry (read-side only) ──────────
|
|
334
|
-
// Minimal first cut of the operating-notes lifecycle. Expiry is READ/INJECTION
|
|
335
|
-
// side ONLY — the store prune (keep-latest-100 above) stays purely count-based
|
|
336
|
-
// and NEVER deletes by age, so audit history is preserved. isNoteExpired decides
|
|
337
|
-
// whether an UNPINNED note still rides into a coordinator prompt.
|
|
338
|
-
//
|
|
339
|
-
// Per-category retention (days). A category not listed here — including the
|
|
340
|
-
// uncategorized case — is durable (never expires). provider_quirk is durable
|
|
341
|
-
// because a runtime quirk stays true until the provider changes.
|
|
342
|
-
export const OPERATING_NOTE_CATEGORY_TTL_DAYS: Readonly<Record<string, number>> = {
|
|
343
|
-
recovery_lesson: 14,
|
|
344
|
-
pattern_to_avoid: 30,
|
|
345
|
-
// provider_quirk: durable (intentionally absent → never expires)
|
|
346
|
-
};
|
|
347
|
-
|
|
348
|
-
const MS_PER_DAY = 24 * 60 * 60 * 1000;
|
|
349
|
-
|
|
350
|
-
/**
|
|
351
|
-
* Shape isNoteExpired reads. Structural so mesh-ledger stays free of a
|
|
352
|
-
* coordinator-prompt import (CoordinatorOperatingNote satisfies this).
|
|
353
|
-
*/
|
|
354
|
-
export interface OperatingNoteExpiryInput {
|
|
355
|
-
category?: string;
|
|
356
|
-
pinned?: boolean;
|
|
357
|
-
createdAt?: string;
|
|
358
|
-
/** Explicit expiry override; wins over the category TTL when parseable. */
|
|
359
|
-
expiresAt?: string;
|
|
360
|
-
/** Fallback creation time (ledger entry timestamp) when createdAt absent. */
|
|
361
|
-
timestamp?: string;
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
/**
|
|
365
|
-
* Pure helper: is this UNPINNED operating note expired as of `now` (epoch ms)?
|
|
366
|
-
*
|
|
367
|
-
* Rules:
|
|
368
|
-
* - pinned notes NEVER expire (always false).
|
|
369
|
-
* - an explicit, parseable `expiresAt` in the past → expired.
|
|
370
|
-
* - otherwise the category TTL applies; a durable category (provider_quirk,
|
|
371
|
-
* uncategorized, or any category not in the TTL map) never expires.
|
|
372
|
-
* - age is measured from createdAt, falling back to `timestamp` (ledger entry
|
|
373
|
-
* time). If neither is a valid date, the note is treated as NOT expired
|
|
374
|
-
* (never silently drop a note we cannot age).
|
|
375
|
-
*/
|
|
376
|
-
export function isNoteExpired(note: OperatingNoteExpiryInput, now: number): boolean {
|
|
377
|
-
if (!note || note.pinned) return false;
|
|
378
|
-
|
|
379
|
-
// Explicit expiresAt wins when present and parseable.
|
|
380
|
-
if (typeof note.expiresAt === 'string') {
|
|
381
|
-
const exp = new Date(note.expiresAt).getTime();
|
|
382
|
-
if (!Number.isNaN(exp)) return exp <= now;
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
const ttlDays = note.category ? OPERATING_NOTE_CATEGORY_TTL_DAYS[note.category] : undefined;
|
|
386
|
-
if (typeof ttlDays !== 'number' || !Number.isFinite(ttlDays)) {
|
|
387
|
-
// Durable category (provider_quirk / uncategorized / unknown) → never expires.
|
|
388
|
-
return false;
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
const created = new Date(note.createdAt ?? note.timestamp ?? '').getTime();
|
|
392
|
-
if (Number.isNaN(created)) return false; // cannot age → keep
|
|
393
|
-
|
|
394
|
-
return now - created >= ttlDays * MS_PER_DAY;
|
|
395
|
-
}
|
|
396
|
-
|
|
397
320
|
// ─── Path Helpers ───────────────────────────────
|
|
398
321
|
|
|
399
322
|
export function getLedgerDir(): string {
|
|
@@ -907,55 +907,6 @@ export function isMeshNodeHealthLaunchable(node: any): boolean {
|
|
|
907
907
|
return health === 'online' || health === 'unknown';
|
|
908
908
|
}
|
|
909
909
|
|
|
910
|
-
/** Resolve the git telemetry object a node carries, in the same precedence
|
|
911
|
-
* resolveEffectiveMeshNodeHealth uses: a fresh `node.git`, else the cached inline
|
|
912
|
-
* status git (`node.cachedStatus.git`). Returns an empty record when neither is
|
|
913
|
-
* present (no telemetry). */
|
|
914
|
-
function resolveEffectiveNodeGit(node: any): Record<string, any> {
|
|
915
|
-
const git = readObjectRecord(node?.git);
|
|
916
|
-
if (Object.keys(git).length > 0) return git;
|
|
917
|
-
const cachedStatus = readObjectRecord(node?.cachedStatus);
|
|
918
|
-
return readObjectRecord(cachedStatus.git);
|
|
919
|
-
}
|
|
920
|
-
|
|
921
|
-
/**
|
|
922
|
-
* Launch FRESHNESS gate — distinct from the health gate above.
|
|
923
|
-
*
|
|
924
|
-
* `deriveMeshNodeHealthFromGit` reports a clean-tree node with `behind > 0` as
|
|
925
|
-
* 'online', so the health gate (isMeshNodeHealthLaunchable) happily lets a STALE
|
|
926
|
-
* node — one whose branch is N commits behind its upstream — win auto-launch fitness
|
|
927
|
-
* routing and run a fresh worker against out-of-date code. `behind` is deliberately
|
|
928
|
-
* NOT folded into deriveMeshNodeHealthFromGit because "behind" is not universally
|
|
929
|
-
* unhealthy (the MAGI planner and other callers share that resolver); it is only
|
|
930
|
-
* unhealthy for *spawning new work*. This gate encodes exactly that launch-time axis.
|
|
931
|
-
*
|
|
932
|
-
* Returns false (NOT fresh → skip / de-rank) only when git telemetry is PRESENT and
|
|
933
|
-
* proves staleness:
|
|
934
|
-
* - behind count exceeds `maxBehind` (default 0 — any behind blocks), OR
|
|
935
|
-
* - a submodule is out of sync (gitlink points off upstream — cannot be caught up
|
|
936
|
-
* by a simple worktree ff and would launch against a mismatched submodule).
|
|
937
|
-
*
|
|
938
|
-
* When telemetry is absent it returns true (fresh), preserving the online/unknown-pass
|
|
939
|
-
* philosophy: we never block on missing data, only on data that proves the node stale.
|
|
940
|
-
*/
|
|
941
|
-
export function isMeshNodeFreshEnoughToLaunch(node: any, opts?: { maxBehind?: number }): boolean {
|
|
942
|
-
const git = resolveEffectiveNodeGit(node);
|
|
943
|
-
// No git telemetry at all → cannot prove stale → do not block.
|
|
944
|
-
if (Object.keys(git).length === 0) return true;
|
|
945
|
-
// Not a git repo / no branch: leave this to the health gate (it returns 'degraded'
|
|
946
|
-
// and blocks there); freshness has nothing to add.
|
|
947
|
-
if (readBooleanValue(git.isGitRepo) === false) return true;
|
|
948
|
-
const submoduleDrift = getGitSubmoduleDriftState(git);
|
|
949
|
-
if (submoduleDrift.outOfSync) return false;
|
|
950
|
-
const behind = readNumberValue(git.behind);
|
|
951
|
-
// No behind datum reported → treat as fresh (don't infer staleness from absence).
|
|
952
|
-
if (behind === undefined) return true;
|
|
953
|
-
const maxBehind = Number.isFinite(opts?.maxBehind as number) && (opts?.maxBehind as number) >= 0
|
|
954
|
-
? Math.floor(opts!.maxBehind as number)
|
|
955
|
-
: 0;
|
|
956
|
-
return behind <= maxBehind;
|
|
957
|
-
}
|
|
958
|
-
|
|
959
910
|
function readMeshNodeLabel(status: Record<string, unknown>, node: any): string {
|
|
960
911
|
return readStringValue(status.nodeId, normalizeMeshNodeId(node)) ?? 'unknown';
|
|
961
912
|
}
|
|
@@ -6,22 +6,20 @@ import { getMesh } from '../config/mesh-config.js';
|
|
|
6
6
|
import { detectCLI } from '../detection/cli-detector.js';
|
|
7
7
|
import { LOG } from '../logging/logger.js';
|
|
8
8
|
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
9
|
-
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank
|
|
9
|
+
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
|
|
10
10
|
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
11
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
12
12
|
import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
|
|
13
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
14
14
|
import { traceMeshEventDrop } from './mesh-event-trace.js';
|
|
15
15
|
import { awaitWithWarmupDeadline, resolveWarmupDeadlineOpts } from './mesh-warmup-deadline.js';
|
|
16
|
-
import {
|
|
17
|
-
import { loadRepoMeshJsonConfig } from '../config/mesh-json-config.js';
|
|
18
|
-
import type { RepoMeshDeclarativeConfig } from '../config/mesh-json-config.js';
|
|
16
|
+
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy, resolveMaxParallelTasks, resolveMaxReadonlyParallelTasks, resolveCoordinatorIdlePushPolicy } from '../repo-mesh-types.js';
|
|
19
17
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
20
18
|
import { normalizeMeshNodeId, meshNodeIdMatches, daemonIdsEquivalent, canonicalDaemonId, expandDaemonIdForms, normalizeMeshWorkspaceForCompare, meshWorkspacesEquivalent, sessionIdsEquivalent, normalizeNodeCapabilitySlots, isMeshTaskDifficulty, withStatusProbeMarker, type MeshNodeIdentified, type NodeCapabilitySlot, type MeshTaskDifficulty } from '@adhdev/mesh-shared';
|
|
21
19
|
import { resolveNodeCapabilitySlots } from './mesh-node-slots.js';
|
|
22
20
|
import { findTerminalLedgerEvidenceForTask, hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
|
|
23
21
|
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
24
|
-
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable
|
|
22
|
+
import { readMeshNodeDaemonId, isMeshNodeHealthLaunchable } from './mesh-node-identity.js';
|
|
25
23
|
import { queuePendingMeshCoordinatorEvent, retractPendingDispatchBlockedEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
26
24
|
import { isWorktreeBootstrapStaleRunning, shouldDeferDispatchForBootstrap } from './worktree-bootstrap-config.js';
|
|
27
25
|
import { isWithinCloneBootstrapGrace } from './mesh-clone-grace.js';
|
|
@@ -92,25 +90,6 @@ export function __resetIdleAutoFastForwardForTests(): void {
|
|
|
92
90
|
autoFastForwardWorkspaceLease.clear();
|
|
93
91
|
}
|
|
94
92
|
|
|
95
|
-
/**
|
|
96
|
-
* Load the repo-shared `.adhdev/mesh.json` for a node's workspace, tolerating a
|
|
97
|
-
* missing/invalid file (returns null → resolver falls back to provider-spec
|
|
98
|
-
* defaults, i.e. exactly the pre-providerDefaults behavior). Only the
|
|
99
|
-
* `providerDefaults` zone influences the delegated-worker MODE selection; it never
|
|
100
|
-
* touches the ENABLE decision. When a node carries no workspace path (should not
|
|
101
|
-
* happen for a launchable node, but be defensive), we skip the read entirely.
|
|
102
|
-
*/
|
|
103
|
-
function loadRepoConfigForNode(node: any): RepoMeshDeclarativeConfig | null {
|
|
104
|
-
const workspace = typeof node?.workspace === 'string' && node.workspace.trim() ? node.workspace.trim() : '';
|
|
105
|
-
if (!workspace) return null;
|
|
106
|
-
try {
|
|
107
|
-
const result = loadRepoMeshJsonConfig(workspace);
|
|
108
|
-
return result.sourceType === 'repo_file' && result.config ? result.config : null;
|
|
109
|
-
} catch {
|
|
110
|
-
return null;
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
93
|
export function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
115
94
|
const localMesh = getMesh(meshId);
|
|
116
95
|
const cachedMesh = components.router?.getCachedInlineMesh(meshId);
|
|
@@ -711,7 +690,7 @@ export function tryAssignQueueTask(
|
|
|
711
690
|
if (inst && typeof inst.updateSettings === 'function') {
|
|
712
691
|
// Adopting a (possibly manually-opened) local session as a worker: apply the
|
|
713
692
|
// delegated-worker auto-approve policy here too, so a session that was launched
|
|
714
|
-
// without
|
|
693
|
+
// without autoApprove still auto-approves once the coordinator dispatches a task
|
|
715
694
|
// to it (the "approval notification fires only for certain delegated sessions"
|
|
716
695
|
// case). updateSettings preserves runtime mesh keys; passing autoApprove keeps it.
|
|
717
696
|
//
|
|
@@ -725,13 +704,7 @@ export function tryAssignQueueTask(
|
|
|
725
704
|
meshNodeFor: meshId,
|
|
726
705
|
meshNodeId: nodeId,
|
|
727
706
|
launchedByCoordinator: true,
|
|
728
|
-
|
|
729
|
-
mesh?.policy,
|
|
730
|
-
node?.policy,
|
|
731
|
-
components.providerLoader?.getMeta(providerType),
|
|
732
|
-
loadRepoConfigForNode(node),
|
|
733
|
-
providerType,
|
|
734
|
-
),
|
|
707
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
735
708
|
...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
|
|
736
709
|
// COMPLETION-PROPAGATION F5: (re)stamp the coordinator SESSION anchor from THIS
|
|
737
710
|
// task's sourceCoordinatorSessionId with PRIORITY — a manually-launched (or reused)
|
|
@@ -958,105 +931,6 @@ function isTargetNodeTransientlyUnresolved(mesh: any, task: MeshWorkQueueEntry):
|
|
|
958
931
|
return isWithinCloneBootstrapGrace(targetNodeId);
|
|
959
932
|
}
|
|
960
933
|
|
|
961
|
-
// ---------------------------------------------------------------------------
|
|
962
|
-
// DEAD-TARGET-SELFHEAL: unpin a queue task hard-pinned to a session/node that has
|
|
963
|
-
// died (absent from the live mesh) so a live idle session can claim it, instead of
|
|
964
|
-
// leaving it stranded 'pending' forever behind the target_session_constraint skip.
|
|
965
|
-
// ---------------------------------------------------------------------------
|
|
966
|
-
|
|
967
|
-
// Conservative age gate before a pinned-but-dead target is unpinned. A target that is
|
|
968
|
-
// merely briefly unassigned or momentarily reconnecting must not be reclaimed on the
|
|
969
|
-
// tick it drops out of view; we require the task to have been idle (no updatedAt bump)
|
|
970
|
-
// for at least this window first. Sized to comfortably outlast a transient P2P blip /
|
|
971
|
-
// reconnect while staying well inside the reclaim cadence of the rest of the file
|
|
972
|
-
// (AUTO_LAUNCH_AWAIT_CLAIM_MS is 90s; the stranded-reclaim watchdog fires on similar
|
|
973
|
-
// scales), so a real reconnect wins the race and the self-heal only fires on a target
|
|
974
|
-
// that is genuinely gone.
|
|
975
|
-
const DEAD_TARGET_GRACE_MS = 60_000;
|
|
976
|
-
|
|
977
|
-
interface DeadTargetVerdict {
|
|
978
|
-
/** The pinned target is confirmed dead and past the grace window → safe to unpin. */
|
|
979
|
-
dead: boolean;
|
|
980
|
-
/** True when the target NODE itself is absent from the live mesh (clear targetNodeId too). */
|
|
981
|
-
nodeDead: boolean;
|
|
982
|
-
/** Short reason string for the ledger/requeue. */
|
|
983
|
-
reason: string;
|
|
984
|
-
}
|
|
985
|
-
|
|
986
|
-
/**
|
|
987
|
-
* Decide whether a task's hard target pin (targetSessionId and/or targetNodeId) points at
|
|
988
|
-
* something that has DIED — i.e. is absent from the live mesh snapshot — and has been so
|
|
989
|
-
* long enough (DEAD_TARGET_GRACE_MS since the task's last update) that unpinning is safe.
|
|
990
|
-
*
|
|
991
|
-
* Two definitive death signals, deliberately conservative to never race a reconnecting node:
|
|
992
|
-
*
|
|
993
|
-
* (1) NODE dead — the task pins a targetNodeId that matches NO node in the live mesh
|
|
994
|
-
* (the same `meshNodeIdMatches`-over-mesh.nodes signal the targetPinUnmatched relabel
|
|
995
|
-
* uses at the empty-candidate site). A pinned session on an absent node is unreachable
|
|
996
|
-
* regardless, so the session pin is dead too. `nodeDead` ⇒ clear targetNodeId as well.
|
|
997
|
-
* Excluded: a target that is only TRANSIENTLY unresolved (a freshly-cloned worktree
|
|
998
|
-
* still propagating / bootstrapping) — isTargetNodeTransientlyUnresolved gates it out.
|
|
999
|
-
*
|
|
1000
|
-
* (2) SESSION dead on a LIVE LOCAL node — the target node IS present and is THIS daemon's
|
|
1001
|
-
* node, but the pinned session is absent from the local instance manager
|
|
1002
|
-
* (resolveSessionBusyVerdict === 'UNKNOWN'). Local session visibility is complete, so
|
|
1003
|
-
* absence here is genuine death, not a busy/generating flip. We KEEP targetNodeId (only
|
|
1004
|
-
* the session died; the node is healthy and can host a replacement claim).
|
|
1005
|
-
*
|
|
1006
|
-
* A live REMOTE node whose session is not in our idle view is NOT treated as dead: absence
|
|
1007
|
-
* from the remote-idle mirror is explicitly UNKNOWN liveness (the session may be busy or its
|
|
1008
|
-
* agent:ready pull merely lost), so unpinning it could race healthy in-flight work. Returns
|
|
1009
|
-
* dead=false in that case, leaving the existing skip in place.
|
|
1010
|
-
*/
|
|
1011
|
-
function resolveDeadTargetVerdict(components: DaemonComponents, meshId: string, mesh: any, task: MeshWorkQueueEntry): DeadTargetVerdict {
|
|
1012
|
-
const NOT_DEAD: DeadTargetVerdict = { dead: false, nodeDead: false, reason: '' };
|
|
1013
|
-
const targetSessionId = readNonEmptyString(task.targetSessionId);
|
|
1014
|
-
const targetNodeId = readNonEmptyString(task.targetNodeId);
|
|
1015
|
-
if (!targetSessionId && !targetNodeId) return NOT_DEAD;
|
|
1016
|
-
|
|
1017
|
-
// Age gate: never reclaim a pin younger than the grace window (guards against a target
|
|
1018
|
-
// that has only just dropped out of view for a momentary reconnect).
|
|
1019
|
-
const lastUpdateMs = Date.parse(task.updatedAt || task.createdAt || '');
|
|
1020
|
-
if (Number.isFinite(lastUpdateMs) && Date.now() - lastUpdateMs < DEAD_TARGET_GRACE_MS) return NOT_DEAD;
|
|
1021
|
-
|
|
1022
|
-
const nodes: any[] = Array.isArray(mesh?.nodes) ? mesh.nodes : [];
|
|
1023
|
-
|
|
1024
|
-
// (1) NODE-dead — a pinned node absent from the live mesh, and NOT merely transiently
|
|
1025
|
-
// unresolved (a propagating/bootstrapping clone). This is a permanent routing miss.
|
|
1026
|
-
if (targetNodeId) {
|
|
1027
|
-
const nodePresent = nodes.some(n => meshNodeIdMatches(n, targetNodeId));
|
|
1028
|
-
if (!nodePresent) {
|
|
1029
|
-
if (isTargetNodeTransientlyUnresolved(mesh, task)) return NOT_DEAD;
|
|
1030
|
-
return { dead: true, nodeDead: true, reason: 'dead_target_node_absent' };
|
|
1031
|
-
}
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
// (2) SESSION-dead on a LIVE LOCAL node. Only meaningful when a session is pinned.
|
|
1035
|
-
if (targetSessionId) {
|
|
1036
|
-
// Resolve the pinned target's node (if any) to decide whether we can trust local
|
|
1037
|
-
// absence. Without a targetNodeId, fall back to whichever live node hosts the session
|
|
1038
|
-
// is unknowable here; treat that as a LOCAL check only (a session id we cannot see
|
|
1039
|
-
// locally on a node we cannot resolve remotely stays UNKNOWN → not dead).
|
|
1040
|
-
const node = targetNodeId
|
|
1041
|
-
? nodes.find(n => meshNodeIdMatches(n, targetNodeId))
|
|
1042
|
-
: undefined;
|
|
1043
|
-
// A pinned session on a REMOTE live node: absence from our view is UNKNOWN, not death.
|
|
1044
|
-
// Only a LOCAL node (or no node pin at all — same-daemon assumption) lets us conclude
|
|
1045
|
-
// death from local instance-manager absence.
|
|
1046
|
-
const nodeIsLocal = node ? isLocalAutoLaunchNode(node) : true;
|
|
1047
|
-
if (nodeIsLocal) {
|
|
1048
|
-
const verdict = resolveSessionBusyVerdict(components, targetSessionId);
|
|
1049
|
-
if (verdict === 'UNKNOWN') {
|
|
1050
|
-
// Session absent from the complete local session view → genuinely gone.
|
|
1051
|
-
return { dead: true, nodeDead: false, reason: 'dead_target_session_absent' };
|
|
1052
|
-
}
|
|
1053
|
-
// GENERATING / IDLE_CONFIRMED → the session is alive (possibly busy). Never disturb.
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
|
|
1057
|
-
return NOT_DEAD;
|
|
1058
|
-
}
|
|
1059
|
-
|
|
1060
934
|
/**
|
|
1061
935
|
* FALSE-BLOCKER-CLONE-QUEUE (stale-event clear): once a task whose actionable blocker we
|
|
1062
936
|
* previously paged either gets claimed or transitions to a self-resolving state, re-arm the
|
|
@@ -1721,20 +1595,8 @@ function liveSessionCountForNode(components: DaemonComponents, meshId: string, n
|
|
|
1721
1595
|
* free claimer, and is excluded — so a read-only auto-launch onto a busy-but-no-idle node
|
|
1722
1596
|
* is still allowed. A node with NO live mesh session at all (dead, or never launched) does
|
|
1723
1597
|
* not match, preserving the legitimate first-session spawn.
|
|
1724
|
-
*
|
|
1725
|
-
* PROVIDER-MATCH gate (DISPATCH-DEADLOCK-PROVIDER-MISMATCH): a live session only suppresses
|
|
1726
|
-
* this launch if it could ACTUALLY claim THIS task. A session whose providerType does not
|
|
1727
|
-
* satisfy the task's requiredTags (e.g. a claude-cli coordinator/worker session on a node,
|
|
1728
|
-
* while the pending task is required_tags: [provider=codex-cli]) is NOT a pending claimer for
|
|
1729
|
-
* this task — claimNextQueueTask's nodeSatisfiesRequiredTags gate would reject its claim. Left
|
|
1730
|
-
* unchecked, such a mismatched session made this gate return true for a task it can never claim,
|
|
1731
|
-
* so the required-provider worker never auto-launched AND no session could claim → nobody made
|
|
1732
|
-
* progress → permanent silent deadlock. Mirror the claim path: build the session's own
|
|
1733
|
-
* capability tags (its providerType pinned onto the node) and only count it as a pending claimer
|
|
1734
|
-
* when those tags satisfy task.requiredTags. `node` is passed in so the tag set reflects this
|
|
1735
|
-
* node's os/arch/worktree/converge context, matching claimNextQueueTask exactly.
|
|
1736
1598
|
*/
|
|
1737
|
-
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string
|
|
1599
|
+
function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: string, nodeId: string): boolean {
|
|
1738
1600
|
// (A) AUTOLAUNCH-CLAIM-CHURN remote-awareness: a task whose auto-launch record targets this
|
|
1739
1601
|
// node and is still inside its await-claim window (base or backoff) already has a session on
|
|
1740
1602
|
// its way to claim — even when that session is REMOTE and thus invisible to the local
|
|
@@ -1773,20 +1635,7 @@ function nodeHasLiveSessionPendingClaim(components: DaemonComponents, meshId: st
|
|
|
1773
1635
|
if (isTerminalSessionStatus(status)) return false; // dead → no claimer here, allow launch
|
|
1774
1636
|
const sessionId = readNonEmptyString(state.instanceId);
|
|
1775
1637
|
if (sessionId && busySessionIds.has(sessionId)) return false; // busy with its own assigned task
|
|
1776
|
-
//
|
|
1777
|
-
// pending claimer if its own provider could satisfy THIS task's requiredTags. A session
|
|
1778
|
-
// whose providerType does not match (e.g. a claude-cli session while task requires
|
|
1779
|
-
// provider=codex-cli) can never claim this task via claimNextQueueTask's
|
|
1780
|
-
// nodeSatisfiesRequiredTags gate, so it must not suppress the required-provider launch —
|
|
1781
|
-
// otherwise the task deadlocks (mismatched session blocks launch, yet cannot claim). Mirror
|
|
1782
|
-
// the claim path: pin the session's providerType onto this node and check the tags.
|
|
1783
|
-
if (task.requiredTags?.length) {
|
|
1784
|
-
const sessionProviderType = state.type || readNonEmptyString(settings.providerType);
|
|
1785
|
-
if (sessionProviderType && !nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, sessionProviderType))) {
|
|
1786
|
-
return false; // provider mismatch → this session can't claim this task; not a pending claimer
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
return true; // live + unassigned + provider-capable → will claim the pending task itself
|
|
1638
|
+
return true; // live + unassigned → will claim the pending task itself
|
|
1790
1639
|
});
|
|
1791
1640
|
}
|
|
1792
1641
|
|
|
@@ -2039,11 +1888,6 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2039
1888
|
}
|
|
2040
1889
|
if (!pending.length) return false;
|
|
2041
1890
|
|
|
2042
|
-
// Launch-freshness threshold: reuse the auto-fast-forward policy's maxBehind (default
|
|
2043
|
-
// 0 → any behind blocks) so the launch gate and the repair path agree on how far
|
|
2044
|
-
// behind is tolerable. Resolved once per pass and shared across every candidate node.
|
|
2045
|
-
const freshnessGate = { maxBehind: resolveAutoFastForwardPolicy(mesh).maxBehind };
|
|
2046
|
-
|
|
2047
1891
|
// Write cap + read-only cap resolved through the shared helpers from the
|
|
2048
1892
|
// MACHINE-LOCAL stored mesh policy (no repo-file overlay). These are the same
|
|
2049
1893
|
// resolvers the observability projection uses, so the enforced and exposed
|
|
@@ -2085,32 +1929,6 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2085
1929
|
continue;
|
|
2086
1930
|
}
|
|
2087
1931
|
if (task.targetSessionId) {
|
|
2088
|
-
// DEAD-TARGET-SELFHEAL: before the unconditional target_session_constraint skip,
|
|
2089
|
-
// check whether the pinned session/node has DIED (absent from the live mesh). A
|
|
2090
|
-
// hard-pinned task whose target is gone can NEVER re-enter 'assigned' (the claim
|
|
2091
|
-
// gate refuses every non-matching session) and this skip fires forever with no
|
|
2092
|
-
// liveness check — the triple-walled stranded-pending defect. If the pin is
|
|
2093
|
-
// confirmed dead past the grace window, requeue it (clearing the dead session
|
|
2094
|
-
// pin, and the node pin too when the NODE itself is gone) so a live idle session
|
|
2095
|
-
// can claim it. requeueTask counts toward maxTaskRetries → bounded self-heal that
|
|
2096
|
-
// auto-fails past the cap (the desired terminal state, unblocking dependents).
|
|
2097
|
-
const deadTarget = resolveDeadTargetVerdict(components, meshId, mesh, task);
|
|
2098
|
-
if (deadTarget.dead) {
|
|
2099
|
-
const requeued = requeueTask(meshId, task.id, {
|
|
2100
|
-
reason: deadTarget.reason,
|
|
2101
|
-
clearTargetSession: true,
|
|
2102
|
-
// Keep the node pin if only the SESSION died on a still-live node; clear it
|
|
2103
|
-
// when the NODE itself is absent (nothing to pin to).
|
|
2104
|
-
clearTargetNode: deadTarget.nodeDead,
|
|
2105
|
-
});
|
|
2106
|
-
if (requeued) {
|
|
2107
|
-
LOG.warn('MeshQueue', `DEAD-TARGET-SELFHEAL: task ${task.id} (mesh ${meshId}) was pinned to a dead target (${deadTarget.reason}); requeued${deadTarget.nodeDead ? ' and unpinned node' : ''} (requeueCount=${requeued.requeueCount ?? '?'}, status=${requeued.status}).`);
|
|
2108
|
-
}
|
|
2109
|
-
// Keep the skip for THIS tick (the requeue already flipped the row to
|
|
2110
|
-
// pending/failed); a later tick assigns/launches the now-unpinned task.
|
|
2111
|
-
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_dead_requeued' });
|
|
2112
|
-
continue;
|
|
2113
|
-
}
|
|
2114
1932
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
2115
1933
|
continue;
|
|
2116
1934
|
}
|
|
@@ -2267,18 +2085,6 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2267
2085
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
2268
2086
|
continue;
|
|
2269
2087
|
}
|
|
2270
|
-
// FRESHNESS gate (distinct from the health gate above): a clean-tree node that
|
|
2271
|
-
// is `behind` its upstream reads as 'online' and passes isLaunchableNode, so
|
|
2272
|
-
// without this it could win fitness routing and run a fresh worker against
|
|
2273
|
-
// stale code. Skip a node whose git telemetry proves it stale (behind >
|
|
2274
|
-
// maxBehind, or a submodule out of sync). Reuse the auto-fast-forward policy's
|
|
2275
|
-
// maxBehind threshold so "how far behind is tolerable" is configured in ONE
|
|
2276
|
-
// place. Telemetry-absent nodes pass (never block on missing data). The 4s
|
|
2277
|
-
// reconcile retries once the node's auto-ff repair path catches it up.
|
|
2278
|
-
if (!isMeshNodeFreshEnoughToLaunch(node, freshnessGate)) {
|
|
2279
|
-
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_stale_behind_upstream', nodeId });
|
|
2280
|
-
continue;
|
|
2281
|
-
}
|
|
2282
2088
|
const launchTarget = resolveAutoLaunchTarget(components, node);
|
|
2283
2089
|
if (launchTarget.mode === 'skip') {
|
|
2284
2090
|
// Remote node we can't reach (no transport / no coordinator daemonId).
|
|
@@ -2298,7 +2104,7 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2298
2104
|
// busy-but-no-idle node is still allowed. Skip with a transient (non-actionable)
|
|
2299
2105
|
// reason so the coordinator is not paged; the 4s reconcile retries, and once the
|
|
2300
2106
|
// existing session goes terminal this gate clears and a legitimate launch proceeds.
|
|
2301
|
-
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId
|
|
2107
|
+
if (nodeHasLiveSessionPendingClaim(components, meshId, nodeId)) {
|
|
2302
2108
|
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_live_session_pending_claim', nodeId });
|
|
2303
2109
|
continue;
|
|
2304
2110
|
}
|
|
@@ -2370,14 +2176,8 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
2370
2176
|
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
2371
2177
|
// Coordinator-dispatched worker: auto-approve unless mesh/node policy
|
|
2372
2178
|
// opts out (default true). Lands in settingsOverride and beats the
|
|
2373
|
-
// global per-provider-type
|
|
2374
|
-
|
|
2375
|
-
mesh?.policy,
|
|
2376
|
-
node?.policy,
|
|
2377
|
-
components.providerLoader?.getMeta(resolved.providerType),
|
|
2378
|
-
loadRepoConfigForNode(node),
|
|
2379
|
-
resolved.providerType,
|
|
2380
|
-
),
|
|
2179
|
+
// global per-provider-type autoApprove config (see shouldAutoApprove).
|
|
2180
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
2381
2181
|
launchedByCoordinator: true,
|
|
2382
2182
|
autoLaunchedForQueueTaskId: task.id,
|
|
2383
2183
|
};
|
|
@@ -3042,3 +2842,4 @@ export function runIdleMaintenanceThenAssignQueue(components: DaemonComponents,
|
|
|
3042
2842
|
});
|
|
3043
2843
|
});
|
|
3044
2844
|
}
|
|
2845
|
+
|