@adhdev/daemon-core 0.9.82-rc.310 → 0.9.82-rc.311
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +4 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +525 -54
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +517 -54
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-coordinator.d.ts +31 -0
- package/dist/mesh/mesh-runtime-store.d.ts +18 -0
- package/dist/mesh/mesh-work-queue.d.ts +23 -0
- package/dist/providers/spec/cli-adapter.d.ts +34 -3
- package/dist/providers/spec/types.d.ts +36 -0
- package/dist/repo-mesh-types.d.ts +97 -9
- package/package.json +2 -2
- package/src/commands/chat-commands.ts +10 -2
- package/src/commands/router.ts +139 -3
- package/src/commands/stream-commands.ts +8 -0
- package/src/config/chat-history.ts +9 -0
- package/src/config/mesh-config.ts +17 -1
- package/src/index.ts +13 -2
- package/src/mesh/mesh-events-coordinator.ts +165 -9
- package/src/mesh/mesh-runtime-store.ts +52 -0
- package/src/mesh/mesh-work-queue.ts +105 -1
- package/src/providers/spec/cli-adapter.ts +155 -13
- package/src/providers/spec/fsm-driver.ts +14 -1
- package/src/providers/spec/native-history-executor.ts +114 -22
- package/src/providers/spec/types.ts +37 -0
- package/src/repo-mesh-types.ts +128 -9
|
@@ -21,7 +21,7 @@ import type {
|
|
|
21
21
|
RepoMeshHostMetadata,
|
|
22
22
|
RepoMeshDaemonRole,
|
|
23
23
|
} from '../repo-mesh-types.js';
|
|
24
|
-
import { DEFAULT_MESH_POLICY } from '../repo-mesh-types.js';
|
|
24
|
+
import { DEFAULT_MESH_POLICY, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
25
25
|
import { createDefaultMeshHostMetadata } from '../mesh/mesh-host-ownership.js';
|
|
26
26
|
|
|
27
27
|
// ─── Persistence ────────────────────────────────
|
|
@@ -118,6 +118,22 @@ function mergeMeshPolicy(base: RepoMeshPolicy | undefined, patch: Partial<RepoMe
|
|
|
118
118
|
if (!SPAWNED_SESSION_VISIBILITY_MODES.has(String(policy.spawnedSessionVisibility))) {
|
|
119
119
|
policy.spawnedSessionVisibility = 'visible';
|
|
120
120
|
}
|
|
121
|
+
// Load-balancing: normalize the scheduling strategy so an invalid/blank value
|
|
122
|
+
// falls back to 'first_eligible' (strict no-change). Only persist the field when
|
|
123
|
+
// it is explicitly a non-default value to keep existing meshes.json untouched.
|
|
124
|
+
const normalizedStrategy = normalizeMeshSchedulingStrategy(policy.schedulingStrategy);
|
|
125
|
+
if (normalizedStrategy === 'first_eligible') {
|
|
126
|
+
delete policy.schedulingStrategy;
|
|
127
|
+
} else {
|
|
128
|
+
policy.schedulingStrategy = normalizedStrategy;
|
|
129
|
+
}
|
|
130
|
+
// Convergence routing: strict opt-in (default false). Only persist when explicitly
|
|
131
|
+
// enabled so existing meshes.json stays byte-for-byte untouched.
|
|
132
|
+
if (policy.autoConvergeCodeChange === true) {
|
|
133
|
+
policy.autoConvergeCodeChange = true;
|
|
134
|
+
} else {
|
|
135
|
+
delete policy.autoConvergeCodeChange;
|
|
136
|
+
}
|
|
121
137
|
return policy;
|
|
122
138
|
}
|
|
123
139
|
|
package/src/index.ts
CHANGED
|
@@ -131,8 +131,19 @@ export type {
|
|
|
131
131
|
RepoMeshLedgerSummaryStatus,
|
|
132
132
|
RepoMeshLedgerStatus,
|
|
133
133
|
MeshAsyncJobLifecycle,
|
|
134
|
+
RepoMeshSchedulingStrategy,
|
|
135
|
+
} from './repo-mesh-types.js';
|
|
136
|
+
export {
|
|
137
|
+
DEFAULT_MESH_POLICY,
|
|
138
|
+
resolveDelegatedWorkerAutoApprove,
|
|
139
|
+
MESH_SCHEDULING_STRATEGIES,
|
|
140
|
+
DEFAULT_MESH_SCHEDULING_STRATEGY,
|
|
141
|
+
normalizeMeshSchedulingStrategy,
|
|
142
|
+
resolveNodeSchedulingPriority,
|
|
143
|
+
MESH_CONVERGE_REFINE_TAG,
|
|
144
|
+
MESH_CONVERGE_FAST_FORWARD_TAG,
|
|
145
|
+
resolveAutoConvergeCodeChange,
|
|
134
146
|
} from './repo-mesh-types.js';
|
|
135
|
-
export { DEFAULT_MESH_POLICY, resolveDelegatedWorkerAutoApprove } from './repo-mesh-types.js';
|
|
136
147
|
|
|
137
148
|
// ── Git Surface ──
|
|
138
149
|
export * from './git/index.js';
|
|
@@ -217,7 +228,7 @@ export { buildMeshLedgerReconciliationEvidence, buildMeshLedgerReplicaEvidence }
|
|
|
217
228
|
export type { AnyLedgerSlice, MeshLedgerReconciliationEvidence, MeshLedgerReplicaEvidence, MeshLedgerReplicaStatus } from './mesh/mesh-ledger-reconciliation.js';
|
|
218
229
|
|
|
219
230
|
// ── Mesh Work Queue (GUPP) ──
|
|
220
|
-
export { enqueueTask, recordDirectDispatchTask, getQueue, claimNextTask, updateTaskStatus, updateSessionTaskStatus, cancelTask, requeueTask, getMeshQueueStats, getMeshQueueRevision, normalizeMeshTaskMode, validateMeshTaskModeRequest, buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, normalizeMeshCapabilityTags, insertDirectDispatch, getActiveDirectDispatches, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, markStaleDirectDispatches, deleteDirectDispatchesByTaskId, recordMeshToolCall, assertNoDependencyCycle, hasPendingDependents, describeTaskDependencyState } from './mesh/mesh-work-queue.js';
|
|
231
|
+
export { enqueueTask, recordDirectDispatchTask, getQueue, claimNextTask, updateTaskStatus, updateSessionTaskStatus, cancelTask, requeueTask, getMeshQueueStats, getMeshQueueRevision, normalizeMeshTaskMode, validateMeshTaskModeRequest, buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, normalizeMeshCapabilityTags, resolveConvergeRequiredTags, insertDirectDispatch, getActiveDirectDispatches, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, markStaleDirectDispatches, deleteDirectDispatchesByTaskId, recordMeshToolCall, assertNoDependencyCycle, hasPendingDependents, describeTaskDependencyState } from './mesh/mesh-work-queue.js';
|
|
221
232
|
export type { MeshWorkQueueEntry, MeshTaskStatus, MeshTaskMode, MeshWorkQueueStats, MeshQueueMutationOptions, MeshTaskModeValidationResult, DirectDispatchRecord, MeshToolCallRateResult } from './mesh/mesh-work-queue.js';
|
|
222
233
|
export { buildCompactStaleDirectWorkSummary, buildMeshActiveWork, buildMeshActiveWorkSummary, classifyStaleDirectForPrune, PRUNABLE_ORPHAN_STALE_REASONS } from './mesh/mesh-active-work.js';
|
|
223
234
|
export type { StaleDirectPruneClassification } from './mesh/mesh-active-work.js';
|
|
@@ -14,7 +14,8 @@ import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } f
|
|
|
14
14
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
15
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
16
16
|
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
17
|
-
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel } from '../repo-mesh-types.js';
|
|
17
|
+
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
|
+
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
19
|
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
19
20
|
import {
|
|
20
21
|
findRecentTerminalLedgerEvidence,
|
|
@@ -499,6 +500,91 @@ function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
|
499
500
|
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
500
501
|
}
|
|
501
502
|
|
|
503
|
+
/** Active (status='assigned') task count for a node — the load metric for
|
|
504
|
+
* least-loaded / round-robin ranking. Lower = preferred. */
|
|
505
|
+
function nodeActiveLoad(meshId: string, nodeId: string): number {
|
|
506
|
+
return MeshRuntimeStore.getInstance().nodeActiveAssignmentCount(meshId, nodeId);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* The mesh-wide scheduling strategy. Defaults to 'first_eligible' (strict
|
|
511
|
+
* no-change) for any mesh that does not set it. Only governs the final tie-break;
|
|
512
|
+
* eligibility, capacity, and priority gates apply identically to every strategy.
|
|
513
|
+
*/
|
|
514
|
+
function resolveSchedulingStrategy(mesh: any): RepoMeshSchedulingStrategy {
|
|
515
|
+
return normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Order eligible nodes for assignment per the mesh scheduling pipeline:
|
|
520
|
+
* PRIORITY (schedulingPriority desc) → TIE-BREAK (strategy).
|
|
521
|
+
*
|
|
522
|
+
* The caller has already applied the TAG hard-filter and is responsible for the
|
|
523
|
+
* MAX-ALLOC capacity gate (the per-node launch/claim checks). This function only
|
|
524
|
+
* decides the *preference order* among nodes that are otherwise eligible.
|
|
525
|
+
*
|
|
526
|
+
* - 'first_eligible' (default): returns the input order verbatim and does NOT touch
|
|
527
|
+
* the round-robin cursor — byte-for-byte the pre-feature behavior.
|
|
528
|
+
* - 'priority_only': schedulingPriority desc, then input order (load ignored).
|
|
529
|
+
* - 'least_loaded': schedulingPriority desc, then active load asc, then input order.
|
|
530
|
+
* - 'round_robin': same as least_loaded, but among nodes tied at (priority, load)
|
|
531
|
+
* the input order is rotated by a per-mesh cursor that advances once per pass.
|
|
532
|
+
*
|
|
533
|
+
* `nodes` carries the original config/array index so the tie-break can fall back to
|
|
534
|
+
* deterministic input order. `bumpCursor` advances the round-robin cursor exactly
|
|
535
|
+
* once per scheduling pass (only consulted for 'round_robin').
|
|
536
|
+
*/
|
|
537
|
+
interface RankableNode { nodeId: string; node: any; index: number }
|
|
538
|
+
|
|
539
|
+
/** Test-only: the pure node-ordering stage (PRIORITY → TIE-BREAK). Exposed so the
|
|
540
|
+
* scheduling pipeline can be unit-tested without standing up live CLI sessions. */
|
|
541
|
+
export function __orderEligibleNodesForTests(
|
|
542
|
+
meshId: string,
|
|
543
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
544
|
+
nodes: RankableNode[],
|
|
545
|
+
opts?: { bumpCursor?: boolean },
|
|
546
|
+
): RankableNode[] {
|
|
547
|
+
return orderEligibleNodes(meshId, strategy, nodes, opts);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function orderEligibleNodes(
|
|
551
|
+
meshId: string,
|
|
552
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
553
|
+
nodes: RankableNode[],
|
|
554
|
+
opts?: { bumpCursor?: boolean },
|
|
555
|
+
): RankableNode[] {
|
|
556
|
+
if (strategy === 'first_eligible' || nodes.length <= 1) {
|
|
557
|
+
return nodes;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const priorityOf = (n: { node: any }) => resolveNodeSchedulingPriority(n.node?.policy);
|
|
561
|
+
|
|
562
|
+
// Round-robin rotation offset: rotate the deterministic input order by a
|
|
563
|
+
// per-mesh cursor so the tie-break winner among equal (priority, load) nodes
|
|
564
|
+
// cycles across passes. The cursor advances once per scheduling pass.
|
|
565
|
+
let rotation = 0;
|
|
566
|
+
if (strategy === 'round_robin') {
|
|
567
|
+
const cursor = opts?.bumpCursor
|
|
568
|
+
? MeshRuntimeStore.getInstance().bumpSchedulerCursor(meshId)
|
|
569
|
+
: MeshRuntimeStore.getInstance().getSchedulerCursor(meshId);
|
|
570
|
+
rotation = ((cursor % nodes.length) + nodes.length) % nodes.length;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// Rotation rank: position of each node after rotating input order by `rotation`.
|
|
574
|
+
// For non-round-robin strategies rotation is 0, so this is just the input index.
|
|
575
|
+
const rotationRank = (index: number) => (index - rotation + nodes.length) % nodes.length;
|
|
576
|
+
|
|
577
|
+
return [...nodes].sort((a, b) => {
|
|
578
|
+
const prioDelta = priorityOf(b) - priorityOf(a); // higher priority first
|
|
579
|
+
if (prioDelta !== 0) return prioDelta;
|
|
580
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
581
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
582
|
+
if (loadDelta !== 0) return loadDelta;
|
|
583
|
+
}
|
|
584
|
+
return rotationRank(a.index) - rotationRank(b.index);
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
|
|
502
588
|
/** Active assignments on a (node, provider) — pre-launch guard for the per-(node,
|
|
503
589
|
* provider) maxParallel cap. The authoritative enforcement is in the claim
|
|
504
590
|
* transaction; this only avoids spawning a session that would fail the claim. */
|
|
@@ -700,7 +786,25 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
700
786
|
continue;
|
|
701
787
|
}
|
|
702
788
|
|
|
703
|
-
|
|
789
|
+
// PRIORITY → TIE-BREAK: order the eligible (TAG-filtered) candidate nodes by
|
|
790
|
+
// the mesh scheduling strategy. 'first_eligible' (default) returns them in
|
|
791
|
+
// config/array order unchanged, so distribution is strictly opt-in. The
|
|
792
|
+
// per-node MAX-ALLOC capacity gate (nodeHasActiveAssignment, provider cap,
|
|
793
|
+
// maxConcurrentSessions) is still applied inside the loop below; this only
|
|
794
|
+
// chooses which eligible node is *tried first*.
|
|
795
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
796
|
+
const orderedCandidateNodes = strategy === 'first_eligible'
|
|
797
|
+
? candidateNodes
|
|
798
|
+
: orderEligibleNodes(
|
|
799
|
+
meshId,
|
|
800
|
+
strategy,
|
|
801
|
+
candidateNodes
|
|
802
|
+
.map((node: any, index: number) => ({ nodeId: readMeshNodeId(node), node, index }))
|
|
803
|
+
.filter((c: RankableNode) => c.nodeId),
|
|
804
|
+
{ bumpCursor: true },
|
|
805
|
+
).map((c: RankableNode) => c.node);
|
|
806
|
+
|
|
807
|
+
for (const node of orderedCandidateNodes) {
|
|
704
808
|
const nodeId = readMeshNodeId(node);
|
|
705
809
|
if (!nodeId) continue;
|
|
706
810
|
const launchKey = `${meshId}:${nodeId}`;
|
|
@@ -866,6 +970,19 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
866
970
|
};
|
|
867
971
|
}
|
|
868
972
|
|
|
973
|
+
// Collect every idle mesh session (local CLI instances + remote idle records)
|
|
974
|
+
// as drain candidates. The drain ORDER depends on the scheduling strategy:
|
|
975
|
+
// - 'first_eligible' (default): local-first, then remote, exactly as before.
|
|
976
|
+
// - otherwise: local + remote merged into one pool and drained in scheduling
|
|
977
|
+
// order (priority → load → tie-break). This local-first debias is required
|
|
978
|
+
// because without it the coordinator's own local node is always visited
|
|
979
|
+
// first and greedily absorbs all untargeted work before any remote idle
|
|
980
|
+
// session is even considered — the comparator alone can't spread work if
|
|
981
|
+
// local is always tried first.
|
|
982
|
+
type IdleCandidate = { nodeId: string; sessionId: string; providerType: string; origin: 'local' | 'remote'; node: any };
|
|
983
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
984
|
+
const localCandidates: IdleCandidate[] = [];
|
|
985
|
+
|
|
869
986
|
const cliInstances = components.instanceManager.getByCategory('cli');
|
|
870
987
|
for (const inst of cliInstances) {
|
|
871
988
|
const state = inst.getState();
|
|
@@ -893,7 +1010,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
893
1010
|
|
|
894
1011
|
if (providerType) {
|
|
895
1012
|
localIdleSessionsChecked += 1;
|
|
896
|
-
|
|
1013
|
+
localCandidates.push({ nodeId, sessionId, providerType, origin: 'local', node: mesh.nodes.find((n: any) => readMeshNodeId(n) === nodeId) });
|
|
897
1014
|
} else {
|
|
898
1015
|
skippedSessions.push({
|
|
899
1016
|
nodeId,
|
|
@@ -908,16 +1025,55 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
908
1025
|
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
909
1026
|
} catch { /* best-effort */ }
|
|
910
1027
|
|
|
1028
|
+
const remoteCandidates: IdleCandidate[] = [];
|
|
911
1029
|
for (const idle of remoteSessions) {
|
|
912
1030
|
const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
|
|
913
1031
|
if (node) {
|
|
914
1032
|
remoteIdleSessionsChecked += 1;
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1033
|
+
remoteCandidates.push({ nodeId: idle.nodeId, sessionId: idle.sessionId, providerType: idle.providerType, origin: 'remote', node });
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
const assignIdleCandidate = (candidate: IdleCandidate): void => {
|
|
1038
|
+
const assigned = tryAssignQueueTask(components, meshId, candidate.nodeId, candidate.sessionId, candidate.providerType);
|
|
1039
|
+
if (assigned && candidate.origin === 'remote') {
|
|
1040
|
+
try {
|
|
1041
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(candidate.nodeId, candidate.sessionId);
|
|
1042
|
+
} catch { /* best-effort */ }
|
|
1043
|
+
}
|
|
1044
|
+
};
|
|
1045
|
+
|
|
1046
|
+
if (strategy === 'first_eligible') {
|
|
1047
|
+
// Strict no-change: drain local idle sessions first (original order), then
|
|
1048
|
+
// remote idle sessions. tryAssignQueueTask is a no-op when nothing matches.
|
|
1049
|
+
for (const candidate of localCandidates) assignIdleCandidate(candidate);
|
|
1050
|
+
for (const candidate of remoteCandidates) assignIdleCandidate(candidate);
|
|
1051
|
+
} else {
|
|
1052
|
+
// Merge local + remote into one pool and drain in scheduling order. Each
|
|
1053
|
+
// assignment mutates a node's active load, and the next pick re-reads it,
|
|
1054
|
+
// so re-ranking after every assignment keeps the spread fair as load shifts.
|
|
1055
|
+
const pool = [...localCandidates, ...remoteCandidates];
|
|
1056
|
+
const baseIndex = new Map<string, number>();
|
|
1057
|
+
pool.forEach((c, i) => { if (!baseIndex.has(c.nodeId)) baseIndex.set(c.nodeId, i); });
|
|
1058
|
+
// Bump the round-robin cursor once for this whole drain pass.
|
|
1059
|
+
const uniqueNodes = [...new Set(pool.map(c => c.nodeId))]
|
|
1060
|
+
.map((nodeId, index) => ({ nodeId, node: pool.find(c => c.nodeId === nodeId)?.node, index }));
|
|
1061
|
+
const ranked = orderEligibleNodes(meshId, strategy, uniqueNodes, { bumpCursor: true });
|
|
1062
|
+
const rankIndex = new Map<string, number>(ranked.map((r, i) => [r.nodeId, i]));
|
|
1063
|
+
const remaining = [...pool];
|
|
1064
|
+
while (remaining.length > 0) {
|
|
1065
|
+
// Re-rank each pass so a node that just took work defers its next session.
|
|
1066
|
+
remaining.sort((a, b) => {
|
|
1067
|
+
const aPrio = resolveNodeSchedulingPriority(a.node?.policy);
|
|
1068
|
+
const bPrio = resolveNodeSchedulingPriority(b.node?.policy);
|
|
1069
|
+
if (aPrio !== bPrio) return bPrio - aPrio;
|
|
1070
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
1071
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
1072
|
+
if (loadDelta !== 0) return loadDelta;
|
|
1073
|
+
}
|
|
1074
|
+
return (rankIndex.get(a.nodeId) ?? 0) - (rankIndex.get(b.nodeId) ?? 0);
|
|
1075
|
+
});
|
|
1076
|
+
assignIdleCandidate(remaining.shift()!);
|
|
921
1077
|
}
|
|
922
1078
|
}
|
|
923
1079
|
|
|
@@ -268,6 +268,17 @@ export class MeshRuntimeStore {
|
|
|
268
268
|
|
|
269
269
|
CREATE INDEX IF NOT EXISTS idx_mesh_missions_mesh_status
|
|
270
270
|
ON mesh_missions(mesh_id, status, updated_at);
|
|
271
|
+
|
|
272
|
+
-- Load-balancing scheduler: per-mesh round-robin rotation cursor. When
|
|
273
|
+
-- the schedulingStrategy is 'round_robin', several eligible nodes tied at
|
|
274
|
+
-- the least load are rotated by this cursor so the tie-break winner cycles
|
|
275
|
+
-- across scheduling passes instead of always favouring the same array-order
|
|
276
|
+
-- node. Persisted (not a module Map) so rotation survives daemon restarts
|
|
277
|
+
-- and stays a single source of truth across scheduling entry points.
|
|
278
|
+
CREATE TABLE IF NOT EXISTS mesh_scheduler_cursor (
|
|
279
|
+
mesh_id TEXT PRIMARY KEY,
|
|
280
|
+
cursor INTEGER NOT NULL DEFAULT 0
|
|
281
|
+
);
|
|
271
282
|
`);
|
|
272
283
|
}
|
|
273
284
|
|
|
@@ -479,6 +490,47 @@ export class MeshRuntimeStore {
|
|
|
479
490
|
return row !== undefined;
|
|
480
491
|
}
|
|
481
492
|
|
|
493
|
+
/**
|
|
494
|
+
* Count active (status='assigned') tasks on a node, regardless of provider or
|
|
495
|
+
* task mode. This is the load metric for least-loaded / round-robin ranking:
|
|
496
|
+
* the scheduler prefers the node with the fewest active assignments so
|
|
497
|
+
* untargeted work spreads instead of piling onto whichever node asks first.
|
|
498
|
+
*/
|
|
499
|
+
nodeActiveAssignmentCount(meshId: string, nodeId: string): number {
|
|
500
|
+
const row = this.db.prepare(`
|
|
501
|
+
SELECT COUNT(*) as count FROM mesh_queue
|
|
502
|
+
WHERE mesh_id = ? AND status = 'assigned' AND assigned_node_id = ?
|
|
503
|
+
`).get(meshId, nodeId) as { count: number } | undefined;
|
|
504
|
+
return row?.count ?? 0;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Read the current per-mesh round-robin cursor (0 when unset). Used to rotate
|
|
509
|
+
* the tie-break winner among nodes tied at the least load.
|
|
510
|
+
*/
|
|
511
|
+
getSchedulerCursor(meshId: string): number {
|
|
512
|
+
const row = this.db.prepare(
|
|
513
|
+
'SELECT cursor FROM mesh_scheduler_cursor WHERE mesh_id = ?'
|
|
514
|
+
).get(meshId) as { cursor: number } | undefined;
|
|
515
|
+
return row?.cursor ?? 0;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Atomically advance the per-mesh round-robin cursor by one and return the
|
|
520
|
+
* value that was current BEFORE the bump (the value the caller should rotate
|
|
521
|
+
* by for this pass). UPSERT keeps it lock-free across concurrent passes.
|
|
522
|
+
*/
|
|
523
|
+
bumpSchedulerCursor(meshId: string): number {
|
|
524
|
+
return this.transaction(() => {
|
|
525
|
+
const current = this.getSchedulerCursor(meshId);
|
|
526
|
+
this.db.prepare(`
|
|
527
|
+
INSERT INTO mesh_scheduler_cursor (mesh_id, cursor) VALUES (?, ?)
|
|
528
|
+
ON CONFLICT(mesh_id) DO UPDATE SET cursor = excluded.cursor
|
|
529
|
+
`).run(meshId, current + 1);
|
|
530
|
+
return current;
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
|
|
482
534
|
/**
|
|
483
535
|
* Count active (status='assigned') tasks on a (node, provider) combination,
|
|
484
536
|
* matched by the assignedProviderType stamped on the payload at claim time.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from 'crypto';
|
|
2
2
|
import { requireMeshHostQueueOwner } from './mesh-host-ownership.js';
|
|
3
3
|
import type { RepoMeshDaemonRole } from '../repo-mesh-types.js';
|
|
4
|
+
import { MESH_CONVERGE_REFINE_TAG, resolveAutoConvergeCodeChange } from '../repo-mesh-types.js';
|
|
4
5
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
5
6
|
import { getMesh } from '../config/mesh-config.js';
|
|
6
7
|
|
|
@@ -203,6 +204,45 @@ function firstProviderPriority(policy: unknown): string | undefined {
|
|
|
203
204
|
return raw.find(type => typeof type === 'string' && type.trim())?.trim();
|
|
204
205
|
}
|
|
205
206
|
|
|
207
|
+
/**
|
|
208
|
+
* Synthetic `role=<x>` capability tags advertised by a node's policy.providerRoles.
|
|
209
|
+
*
|
|
210
|
+
* When a specific `providerType` is being evaluated, only that provider's declared
|
|
211
|
+
* role is emitted — so role-based routing (requiredTags: ["role=validation"]) gates
|
|
212
|
+
* the *selected* provider through the ordinary capability-tag filter. When no
|
|
213
|
+
* provider is selected (node-level eligibility scan), every declared role is emitted
|
|
214
|
+
* so the node passes the filter if ANY of its providers could satisfy the role; the
|
|
215
|
+
* per-provider tag set then narrows it during provider selection.
|
|
216
|
+
*
|
|
217
|
+
* Roles are lowercased and deduped. Missing/empty providerRoles emits nothing, so a
|
|
218
|
+
* node that never declares roles advertises no `role=` tags and is therefore only
|
|
219
|
+
* matched by role-unconstrained tasks (full backward compatibility).
|
|
220
|
+
*/
|
|
221
|
+
function roleCapabilityTags(policy: unknown, providerType: string | undefined): string[] {
|
|
222
|
+
const roles = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
223
|
+
? (policy as Record<string, unknown>).providerRoles
|
|
224
|
+
: undefined;
|
|
225
|
+
if (!Array.isArray(roles)) return [];
|
|
226
|
+
const wantedProvider = typeof providerType === 'string' && providerType.trim()
|
|
227
|
+
? providerType.trim().toLowerCase()
|
|
228
|
+
: '';
|
|
229
|
+
const out: string[] = [];
|
|
230
|
+
for (const entry of roles) {
|
|
231
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
232
|
+
const type = typeof (entry as any).providerType === 'string'
|
|
233
|
+
? (entry as any).providerType.trim().toLowerCase()
|
|
234
|
+
: '';
|
|
235
|
+
const role = typeof (entry as any).role === 'string'
|
|
236
|
+
? (entry as any).role.trim().toLowerCase()
|
|
237
|
+
: '';
|
|
238
|
+
if (!role) continue;
|
|
239
|
+
// When narrowing to a selected provider, only emit that provider's role.
|
|
240
|
+
if (wantedProvider && type && type !== wantedProvider) continue;
|
|
241
|
+
out.push(`role=${role}`);
|
|
242
|
+
}
|
|
243
|
+
return out;
|
|
244
|
+
}
|
|
245
|
+
|
|
206
246
|
export function buildMeshNodeCapabilityTags(
|
|
207
247
|
node: { capabilities?: unknown; policy?: unknown; isLocalWorktree?: unknown; worktreeBranch?: unknown } | undefined,
|
|
208
248
|
providerType?: string,
|
|
@@ -222,6 +262,24 @@ export function buildMeshNodeCapabilityTags(
|
|
|
222
262
|
// mesh_enqueue_task with required_tags: ["worktree=<branch>"] routes
|
|
223
263
|
// only to the matching worktree node.
|
|
224
264
|
...(node?.isLocalWorktree === true && worktreeBranch ? [`worktree=${worktreeBranch}`] : []),
|
|
265
|
+
// Convergence routing: advertise how this node can land its work onto base.
|
|
266
|
+
// - converge=refine: local worktree nodes (on ANY machine — refine_mesh_node
|
|
267
|
+
// now forwards to the owning daemon) can run the Refinery merge → push →
|
|
268
|
+
// cleanup against their own checkout, so they accept code_change tasks.
|
|
269
|
+
// - converge=fast_forward: non-worktree nodes (the machine itself) can only
|
|
270
|
+
// ff/push an already-converged branch; they are NOT a destination for
|
|
271
|
+
// code_change work (a worktree is created first, and that worktree node
|
|
272
|
+
// receives the task instead). Reuses the ordinary required-tags filter —
|
|
273
|
+
// the load-balancing scheduler auto-injects converge=refine for code_change
|
|
274
|
+
// so such work is hard-filtered onto refine-capable nodes.
|
|
275
|
+
...(node?.isLocalWorktree === true ? ['converge=refine'] : ['converge=fast_forward']),
|
|
276
|
+
// Role-based routing: advertise role=<x> for each (node, provider) role
|
|
277
|
+
// declared in policy.providerRoles. Narrowed to the selected provider when
|
|
278
|
+
// one is given so the chosen provider must match a task's required role;
|
|
279
|
+
// when no provider is selected, all declared roles are advertised for the
|
|
280
|
+
// node-level eligibility scan. Reuses the ordinary required-tags filter —
|
|
281
|
+
// no separate role field/gate.
|
|
282
|
+
...roleCapabilityTags(node?.policy, providerType),
|
|
225
283
|
]);
|
|
226
284
|
}
|
|
227
285
|
|
|
@@ -232,6 +290,44 @@ export function nodeSatisfiesRequiredTags(requiredTags: unknown, capabilityTags:
|
|
|
232
290
|
return required.every(tag => available.has(tag));
|
|
233
291
|
}
|
|
234
292
|
|
|
293
|
+
/**
|
|
294
|
+
* Convergence-aware required-tags resolution (load-balancing scheduler, opt-in).
|
|
295
|
+
*
|
|
296
|
+
* When the mesh enables policy.autoConvergeCodeChange, a `converge=refine` required
|
|
297
|
+
* tag is merged into a code_change task's required tags at enqueue time, so the
|
|
298
|
+
* scheduler hard-filters the task onto refine-capable worktree nodes only (on any
|
|
299
|
+
* machine — refine_mesh_node forwards to the owning daemon). Because the tag is
|
|
300
|
+
* persisted on the queue entry, BOTH the eligibility scan (maybeAutoLaunchOneQueueSession)
|
|
301
|
+
* and the claim transaction (claimNextQueueTask → nodeSatisfiesRequiredTags) enforce
|
|
302
|
+
* it consistently.
|
|
303
|
+
*
|
|
304
|
+
* Strict backward compatibility — the injection is skipped (returns the explicit tags
|
|
305
|
+
* unchanged) when ANY of:
|
|
306
|
+
* - the mesh does not opt in (autoConvergeCodeChange !== true), or
|
|
307
|
+
* - the task is not code_change (validation / live_debug_readonly / launch_app /
|
|
308
|
+
* convergence carry no merge cost and may run anywhere), or
|
|
309
|
+
* - the task is explicitly targeted (targetNodeId): the operator chose the node, so
|
|
310
|
+
* we do not second-guess it by filtering on convergence capability.
|
|
311
|
+
* Idempotent: normalizeMeshCapabilityTags dedupes, so re-injection is a no-op.
|
|
312
|
+
*/
|
|
313
|
+
export function resolveConvergeRequiredTags(
|
|
314
|
+
meshId: string,
|
|
315
|
+
taskMode: MeshTaskMode | undefined,
|
|
316
|
+
explicitRequiredTags: string[],
|
|
317
|
+
opts?: { targetNodeId?: string },
|
|
318
|
+
): string[] {
|
|
319
|
+
if (taskMode !== 'code_change') return explicitRequiredTags;
|
|
320
|
+
if (typeof opts?.targetNodeId === 'string' && opts.targetNodeId.trim()) return explicitRequiredTags;
|
|
321
|
+
let optedIn = false;
|
|
322
|
+
try {
|
|
323
|
+
optedIn = resolveAutoConvergeCodeChange(getMesh(meshId)?.policy as any);
|
|
324
|
+
} catch {
|
|
325
|
+
optedIn = false;
|
|
326
|
+
}
|
|
327
|
+
if (!optedIn) return explicitRequiredTags;
|
|
328
|
+
return normalizeMeshCapabilityTags([...explicitRequiredTags, MESH_CONVERGE_REFINE_TAG]);
|
|
329
|
+
}
|
|
330
|
+
|
|
235
331
|
function withQueueLock<T>(_meshId: string, fn: () => T): T {
|
|
236
332
|
return MeshRuntimeStore.getInstance().transaction(fn);
|
|
237
333
|
}
|
|
@@ -325,7 +421,15 @@ export function enqueueTask(
|
|
|
325
421
|
taskMode: modeValidation.taskMode,
|
|
326
422
|
targetNodeId: opts?.targetNodeId,
|
|
327
423
|
targetSessionId: opts?.targetSessionId,
|
|
328
|
-
|
|
424
|
+
// Convergence routing (opt-in): auto-inject converge=refine for code_change
|
|
425
|
+
// tasks so they hard-filter onto refine-capable worktree nodes. No-op unless
|
|
426
|
+
// the mesh opts in; explicit target_node_id / required_tags are preserved.
|
|
427
|
+
requiredTags: resolveConvergeRequiredTags(
|
|
428
|
+
meshId,
|
|
429
|
+
modeValidation.taskMode,
|
|
430
|
+
normalizeMeshCapabilityTags(opts?.requiredTags),
|
|
431
|
+
{ targetNodeId: opts?.targetNodeId },
|
|
432
|
+
),
|
|
329
433
|
...(dependsOn.length > 0 ? { dependsOn } : {}),
|
|
330
434
|
...(typeof opts?.missionId === 'string' && opts.missionId.trim() ? { missionId: opts.missionId.trim() } : {}),
|
|
331
435
|
createdAt: new Date().toISOString(),
|