@adhdev/daemon-core 0.9.82-rc.310 → 0.9.82-rc.312

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,100 @@
1
+ /**
2
+ * Log redactor — mask secrets before a raw daemon log line leaves the machine.
3
+ *
4
+ * Daemon logs can incidentally contain credentials: ADHDev API keys (adk_*),
5
+ * machine secrets (adm_*), provider keys (adp_*), bearer tokens, JWTs, TURN
6
+ * `username:credential` pairs, and `SECRET=...` style env dumps. The mesh
7
+ * `get_mesh_node_logs` command ships a log tail over P2P to the coordinator, so
8
+ * every line MUST pass through redactLogLine() first — otherwise a secret in a
9
+ * remote daemon's log is exfiltrated to whoever is driving the coordinator.
10
+ *
11
+ * Patterns are intentionally conservative: each masks the secret material while
12
+ * preserving enough surrounding shape that the line stays useful for debugging
13
+ * (e.g. `adk_••••1234`, `Bearer ••••redacted`). When in doubt, mask.
14
+ */
15
+
16
+ const MASK = '••••redacted';
17
+
18
+ /** Keep the last 4 chars of a token so logs stay correlatable without leaking it. */
19
+ function maskKeepTail(token: string): string {
20
+ if (token.length <= 8) return MASK;
21
+ return `${MASK}${token.slice(-4)}`;
22
+ }
23
+
24
+ interface RedactionRule {
25
+ readonly name: string;
26
+ readonly pattern: RegExp;
27
+ readonly replace: (match: string, ...groups: string[]) => string;
28
+ }
29
+
30
+ // NOTE: order matters — more specific rules (key=value, Bearer, TURN) run before
31
+ // the bare-token rules so the structured forms aren't half-masked by a greedy
32
+ // generic rule.
33
+ const RULES: RedactionRule[] = [
34
+ // `JWT_SECRET=...`, `TOKEN=...`, `API_KEY=...`, `password: ...` env/config dumps.
35
+ // Captures the key + delimiter and masks only the value.
36
+ {
37
+ name: 'key_value_secret',
38
+ pattern: /\b([A-Z0-9_]*(?:SECRET|TOKEN|API[_-]?KEY|PASSWORD|PASSWD|PRIVATE[_-]?KEY|CREDENTIAL|CLIENT[_-]?SECRET)[A-Z0-9_]*)(\s*[:=]\s*)(["']?)([^\s"',;]+)\3/gi,
39
+ replace: (_m, key: string, delim: string, quote: string) => `${key}${delim}${quote}${MASK}${quote}`,
40
+ },
41
+ // Authorization: Bearer <token>
42
+ {
43
+ name: 'bearer_token',
44
+ pattern: /\b(Bearer\s+)([A-Za-z0-9._\-+/=]{8,})/g,
45
+ replace: (_m, prefix: string, token: string) => `${prefix}${maskKeepTail(token)}`,
46
+ },
47
+ // ADHDev credential prefixes: API key (adk_), machine secret (adm_), provider key (adp_).
48
+ {
49
+ name: 'adhdev_prefixed_secret',
50
+ pattern: /\b(ad[kmp]_)([A-Za-z0-9]{6,})/g,
51
+ replace: (_m, prefix: string, token: string) => `${prefix}${maskKeepTail(prefix + token)}`,
52
+ },
53
+ // JWT: three base64url segments separated by dots, header starts with eyJ.
54
+ {
55
+ name: 'jwt',
56
+ pattern: /\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g,
57
+ replace: () => MASK,
58
+ },
59
+ // TURN credential: a long credential value following a `credential` key in
60
+ // any common shape — `credential: x`, `credential=x`, or `credential "x"`.
61
+ // Mask the credential value only, preserving the key + delimiter/quote.
62
+ {
63
+ name: 'turn_credential',
64
+ pattern: /\b(credential["']?\s*(?:[:=]\s*)?["']?)([^\s"',;]{6,})/gi,
65
+ replace: (_m, prefix: string) => `${prefix}${MASK}`,
66
+ },
67
+ // TURN REST username:credential of the form `<expiry-ts>:<base64hmac>`,
68
+ // where the hmac part is long base64. Mask the hmac.
69
+ {
70
+ name: 'turn_rest_pair',
71
+ pattern: /\b(\d{10,}:)([A-Za-z0-9+/]{20,}={0,2})\b/g,
72
+ replace: (_m, prefix: string) => `${prefix}${MASK}`,
73
+ },
74
+ ];
75
+
76
+ /**
77
+ * Mask secrets in a single log line. Idempotent-ish: re-running over an
78
+ * already-masked line leaves the MASK token in place (it contains no secret
79
+ * shape). Never throws — a redaction failure must not crash the log path.
80
+ */
81
+ export function redactLogLine(line: string): string {
82
+ if (!line) return line;
83
+ let out = line;
84
+ for (const rule of RULES) {
85
+ try {
86
+ out = out.replace(rule.pattern, rule.replace as (substring: string, ...args: any[]) => string);
87
+ } catch {
88
+ // A pathological line must never break log shipping — skip this rule.
89
+ }
90
+ }
91
+ return out;
92
+ }
93
+
94
+ /** Redact an array of log lines in place-safe fashion (returns a new array). */
95
+ export function redactLogLines(lines: string[]): string[] {
96
+ return lines.map((line) => redactLogLine(line));
97
+ }
98
+
99
+ /** Exposed for tests/introspection: the rule names applied, in order. */
100
+ export const LOG_REDACTION_RULE_NAMES: readonly string[] = RULES.map((r) => r.name);
@@ -0,0 +1,220 @@
1
+ /**
2
+ * Daemon log tail reader — read the last N bytes of a daemon log file, newest
3
+ * bytes first, bounded so the result is safe to ship over a mesh P2P channel.
4
+ *
5
+ * Used by the mesh `get_mesh_node_logs` command: the coordinator asks a (possibly
6
+ * remote) daemon for its recent log tail instead of having to open a session and
7
+ * grep the file by hand. Because the mesh RPC envelope is sent as a single
8
+ * datachannel message (~256KB SCTP ceiling, no chunking), the returned tail is
9
+ * HARD-bounded by `tailBytes` (default 64KB, capped at MAX_TAIL_BYTES=128KB) and
10
+ * flags `truncated:true` when the file was larger.
11
+ *
12
+ * Boundary-safe: lines are cut on the newline byte (0x0A) only, which never
13
+ * appears inside a multibyte UTF-8 sequence, so decoding each complete byte
14
+ * segment never splits a multibyte char.
15
+ */
16
+
17
+ import * as fs from 'fs';
18
+ import { getCurrentDaemonLogPath, getDaemonLogDir } from './logger.js';
19
+
20
+ export const DEFAULT_TAIL_BYTES = 64 * 1024;
21
+ export const MAX_TAIL_BYTES = 128 * 1024;
22
+ const READ_CHUNK_BYTES = 64 * 1024;
23
+
24
+ export interface ReadDaemonLogTailArgs {
25
+ /** Date of the log file to read (defaults to today). YYYY-MM-DD string or Date. */
26
+ date?: string | Date;
27
+ /** Max bytes of tail to return. Clamped to (0, MAX_TAIL_BYTES]. Default 64KB. */
28
+ tailBytes?: number;
29
+ /** Optional regex source string; only lines matching (case-insensitive) are kept. */
30
+ grep?: string;
31
+ /** Optional epoch-ms floor; only lines whose leading [HH:MM:SS...] / ISO ts >= this are kept. */
32
+ sinceMs?: number;
33
+ }
34
+
35
+ export interface DaemonLogTailResult {
36
+ success: boolean;
37
+ error?: string;
38
+ lines: string[];
39
+ truncated: boolean;
40
+ logPath: string;
41
+ platform: NodeJS.Platform;
42
+ bytesReturned: number;
43
+ /** True when a grep/since filter dropped lines from the raw tail window. */
44
+ filtered: boolean;
45
+ /** The grep source actually applied (echoed back for clarity). */
46
+ grep?: string;
47
+ }
48
+
49
+ function resolveLogPath(date?: string | Date): string {
50
+ if (date instanceof Date) return getCurrentDaemonLogPath(date);
51
+ if (typeof date === 'string' && date.trim()) {
52
+ const parsed = new Date(`${date.trim()}T00:00:00.000Z`);
53
+ if (!Number.isNaN(parsed.getTime())) return getCurrentDaemonLogPath(parsed);
54
+ }
55
+ return getCurrentDaemonLogPath();
56
+ }
57
+
58
+ function clampTailBytes(tailBytes?: number): number {
59
+ if (!Number.isFinite(tailBytes) || (tailBytes as number) <= 0) return DEFAULT_TAIL_BYTES;
60
+ return Math.min(Math.floor(tailBytes as number), MAX_TAIL_BYTES);
61
+ }
62
+
63
+ /**
64
+ * Read up to `limitBytes` from the end of `filePath`, on a UTF-8 line boundary.
65
+ * Returns the decoded text, whether the read was truncated (file bigger than the
66
+ * window), and the number of bytes actually decoded.
67
+ */
68
+ function readByteBoundedTail(filePath: string, limitBytes: number): { text: string; truncated: boolean; bytesReturned: number } {
69
+ const fd = fs.openSync(filePath, 'r');
70
+ try {
71
+ const stat = fs.fstatSync(fd);
72
+ const size = stat.size;
73
+ if (size === 0) return { text: '', truncated: false, bytesReturned: 0 };
74
+
75
+ const want = Math.min(limitBytes, size);
76
+ let start = size - want;
77
+ const truncated = start > 0;
78
+
79
+ // Collect chunks newest-last into a buffer covering [start, size).
80
+ const buffers: Buffer[] = [];
81
+ let position = start;
82
+ while (position < size) {
83
+ const chunkSize = Math.min(READ_CHUNK_BYTES, size - position);
84
+ const chunk = Buffer.alloc(chunkSize);
85
+ fs.readSync(fd, chunk, 0, chunkSize, position);
86
+ buffers.push(chunk);
87
+ position += chunkSize;
88
+ }
89
+ let buf = Buffer.concat(buffers);
90
+
91
+ // If we truncated mid-line, drop the leading partial line so we never emit
92
+ // a half-decoded line (and never split a multibyte char at the window edge).
93
+ if (truncated) {
94
+ const firstNewline = buf.indexOf(0x0a);
95
+ if (firstNewline >= 0) {
96
+ buf = buf.subarray(firstNewline + 1);
97
+ }
98
+ }
99
+ return { text: buf.toString('utf-8'), truncated, bytesReturned: buf.length };
100
+ } finally {
101
+ fs.closeSync(fd);
102
+ }
103
+ }
104
+
105
+ // Parse a leading timestamp from a log line into epoch ms. The unified logger
106
+ // writes `[HH:MM:SS.mmm]` (local time, today's date) and the startup banner uses
107
+ // a full timestamp; we best-effort parse `[HH:MM:SS...]` against the file's date.
108
+ // Returns null when no timestamp can be extracted (line is then kept by sinceMs).
109
+ function parseLineEpochMs(line: string, fileDate: Date): number | null {
110
+ const m = line.match(/^\[(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?\]/);
111
+ if (!m) {
112
+ // Try an embedded ISO timestamp as a fallback.
113
+ const iso = line.match(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?/);
114
+ if (iso) {
115
+ const t = Date.parse(iso[0].replace(' ', 'T'));
116
+ return Number.isNaN(t) ? null : t;
117
+ }
118
+ return null;
119
+ }
120
+ const d = new Date(fileDate);
121
+ d.setHours(Number(m[1]), Number(m[2]), Number(m[3]), m[4] ? Number(m[4].padEnd(3, '0')) : 0);
122
+ return d.getTime();
123
+ }
124
+
125
+ /**
126
+ * Read the daemon log tail for `date` (default today), bounded to `tailBytes`,
127
+ * with optional grep (regex source) and sinceMs filters. Falls back to the
128
+ * size-rotation backup (`*.1.log`) when the primary file does not exist.
129
+ */
130
+ export function readDaemonLogTail(args: ReadDaemonLogTailArgs = {}): DaemonLogTailResult {
131
+ const platform = process.platform;
132
+ const limitBytes = clampTailBytes(args.tailBytes);
133
+ let logPath = resolveLogPath(args.date);
134
+
135
+ // Fall back to the size-rotation backup if the active file is absent.
136
+ if (!fs.existsSync(logPath)) {
137
+ const backup = logPath.replace(/\.log$/, '.1.log');
138
+ if (fs.existsSync(backup)) {
139
+ logPath = backup;
140
+ } else {
141
+ return {
142
+ success: false,
143
+ error: `No daemon log file at ${logPath} (dir: ${getDaemonLogDir()})`,
144
+ lines: [],
145
+ truncated: false,
146
+ logPath,
147
+ platform,
148
+ bytesReturned: 0,
149
+ filtered: false,
150
+ };
151
+ }
152
+ }
153
+
154
+ let raw: { text: string; truncated: boolean; bytesReturned: number };
155
+ try {
156
+ raw = readByteBoundedTail(logPath, limitBytes);
157
+ } catch (e: any) {
158
+ return {
159
+ success: false,
160
+ error: `Failed to read ${logPath}: ${e?.message ?? String(e)}`,
161
+ lines: [],
162
+ truncated: false,
163
+ logPath,
164
+ platform,
165
+ bytesReturned: 0,
166
+ filtered: false,
167
+ };
168
+ }
169
+
170
+ let lines = raw.text.split('\n');
171
+ // A trailing newline yields a final empty element — drop it.
172
+ if (lines.length && lines[lines.length - 1] === '') lines.pop();
173
+ const rawCount = lines.length;
174
+
175
+ // since filter
176
+ if (Number.isFinite(args.sinceMs)) {
177
+ const fileDate = args.date instanceof Date
178
+ ? args.date
179
+ : typeof args.date === 'string' && args.date.trim()
180
+ ? new Date(`${args.date.trim()}T00:00:00.000Z`)
181
+ : new Date();
182
+ const floor = args.sinceMs as number;
183
+ lines = lines.filter((line) => {
184
+ const ts = parseLineEpochMs(line, fileDate);
185
+ // Keep lines with no parseable timestamp (continuation/stack lines).
186
+ return ts === null || ts >= floor;
187
+ });
188
+ }
189
+
190
+ // grep filter
191
+ let appliedGrep: string | undefined;
192
+ if (typeof args.grep === 'string' && args.grep.trim()) {
193
+ appliedGrep = args.grep.trim();
194
+ let re: RegExp | null = null;
195
+ try {
196
+ re = new RegExp(appliedGrep, 'i');
197
+ } catch {
198
+ re = null;
199
+ }
200
+ if (re) {
201
+ const compiled = re;
202
+ lines = lines.filter((line) => compiled.test(line));
203
+ } else {
204
+ // Invalid regex → fall back to a literal substring match.
205
+ const needle = appliedGrep.toLowerCase();
206
+ lines = lines.filter((line) => line.toLowerCase().includes(needle));
207
+ }
208
+ }
209
+
210
+ return {
211
+ success: true,
212
+ lines,
213
+ truncated: raw.truncated,
214
+ logPath,
215
+ platform,
216
+ bytesReturned: raw.bytesReturned,
217
+ filtered: lines.length !== rawCount,
218
+ ...(appliedGrep ? { grep: appliedGrep } : {}),
219
+ };
220
+ }
@@ -342,6 +342,7 @@ const TOOLS_SECTION = `## Available Tools
342
342
  | \`mesh_read_debug\` | Collect a daemon-side chat/parser debug bundle for a session |
343
343
  | \`mesh_task_history\` | Read the task ledger — dispatches, completions, failures. Use to understand what has been done before deciding next steps |
344
344
  | \`mesh_git_status\` | Check git status on a specific node |
345
+ | \`mesh_read_node_logs\` | Fetch a remote node's daemon log tail directly over P2P (grep/since/byte-bounded, secrets redacted) — no session/PowerShell needed to debug a node's daemon |
345
346
  | \`mesh_fast_forward_node\` | Safely dry-run or explicitly execute an obvious clean fast-forward without launching an agent session |
346
347
  | \`mesh_checkpoint\` | Create a git checkpoint on a node |
347
348
  | \`mesh_approve\` | Approve/reject a pending agent action |
@@ -14,7 +14,8 @@ import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } f
14
14
  import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
15
15
  import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
16
16
  import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
17
- import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel } from '../repo-mesh-types.js';
17
+ import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
18
+ import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
18
19
  import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
19
20
  import {
20
21
  findRecentTerminalLedgerEvidence,
@@ -499,6 +500,91 @@ function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
499
500
  return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
500
501
  }
501
502
 
503
+ /** Active (status='assigned') task count for a node — the load metric for
504
+ * least-loaded / round-robin ranking. Lower = preferred. */
505
+ function nodeActiveLoad(meshId: string, nodeId: string): number {
506
+ return MeshRuntimeStore.getInstance().nodeActiveAssignmentCount(meshId, nodeId);
507
+ }
508
+
509
+ /**
510
+ * The mesh-wide scheduling strategy. Defaults to 'first_eligible' (strict
511
+ * no-change) for any mesh that does not set it. Only governs the final tie-break;
512
+ * eligibility, capacity, and priority gates apply identically to every strategy.
513
+ */
514
+ function resolveSchedulingStrategy(mesh: any): RepoMeshSchedulingStrategy {
515
+ return normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
516
+ }
517
+
518
+ /**
519
+ * Order eligible nodes for assignment per the mesh scheduling pipeline:
520
+ * PRIORITY (schedulingPriority desc) → TIE-BREAK (strategy).
521
+ *
522
+ * The caller has already applied the TAG hard-filter and is responsible for the
523
+ * MAX-ALLOC capacity gate (the per-node launch/claim checks). This function only
524
+ * decides the *preference order* among nodes that are otherwise eligible.
525
+ *
526
+ * - 'first_eligible' (default): returns the input order verbatim and does NOT touch
527
+ * the round-robin cursor — byte-for-byte the pre-feature behavior.
528
+ * - 'priority_only': schedulingPriority desc, then input order (load ignored).
529
+ * - 'least_loaded': schedulingPriority desc, then active load asc, then input order.
530
+ * - 'round_robin': same as least_loaded, but among nodes tied at (priority, load)
531
+ * the input order is rotated by a per-mesh cursor that advances once per pass.
532
+ *
533
+ * `nodes` carries the original config/array index so the tie-break can fall back to
534
+ * deterministic input order. `bumpCursor` advances the round-robin cursor exactly
535
+ * once per scheduling pass (only consulted for 'round_robin').
536
+ */
537
+ interface RankableNode { nodeId: string; node: any; index: number }
538
+
539
+ /** Test-only: the pure node-ordering stage (PRIORITY → TIE-BREAK). Exposed so the
540
+ * scheduling pipeline can be unit-tested without standing up live CLI sessions. */
541
+ export function __orderEligibleNodesForTests(
542
+ meshId: string,
543
+ strategy: RepoMeshSchedulingStrategy,
544
+ nodes: RankableNode[],
545
+ opts?: { bumpCursor?: boolean },
546
+ ): RankableNode[] {
547
+ return orderEligibleNodes(meshId, strategy, nodes, opts);
548
+ }
549
+
550
+ function orderEligibleNodes(
551
+ meshId: string,
552
+ strategy: RepoMeshSchedulingStrategy,
553
+ nodes: RankableNode[],
554
+ opts?: { bumpCursor?: boolean },
555
+ ): RankableNode[] {
556
+ if (strategy === 'first_eligible' || nodes.length <= 1) {
557
+ return nodes;
558
+ }
559
+
560
+ const priorityOf = (n: { node: any }) => resolveNodeSchedulingPriority(n.node?.policy);
561
+
562
+ // Round-robin rotation offset: rotate the deterministic input order by a
563
+ // per-mesh cursor so the tie-break winner among equal (priority, load) nodes
564
+ // cycles across passes. The cursor advances once per scheduling pass.
565
+ let rotation = 0;
566
+ if (strategy === 'round_robin') {
567
+ const cursor = opts?.bumpCursor
568
+ ? MeshRuntimeStore.getInstance().bumpSchedulerCursor(meshId)
569
+ : MeshRuntimeStore.getInstance().getSchedulerCursor(meshId);
570
+ rotation = ((cursor % nodes.length) + nodes.length) % nodes.length;
571
+ }
572
+
573
+ // Rotation rank: position of each node after rotating input order by `rotation`.
574
+ // For non-round-robin strategies rotation is 0, so this is just the input index.
575
+ const rotationRank = (index: number) => (index - rotation + nodes.length) % nodes.length;
576
+
577
+ return [...nodes].sort((a, b) => {
578
+ const prioDelta = priorityOf(b) - priorityOf(a); // higher priority first
579
+ if (prioDelta !== 0) return prioDelta;
580
+ if (strategy === 'least_loaded' || strategy === 'round_robin') {
581
+ const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
582
+ if (loadDelta !== 0) return loadDelta;
583
+ }
584
+ return rotationRank(a.index) - rotationRank(b.index);
585
+ });
586
+ }
587
+
502
588
  /** Active assignments on a (node, provider) — pre-launch guard for the per-(node,
503
589
  * provider) maxParallel cap. The authoritative enforcement is in the claim
504
590
  * transaction; this only avoids spawning a session that would fail the claim. */
@@ -700,7 +786,25 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
700
786
  continue;
701
787
  }
702
788
 
703
- for (const node of candidateNodes) {
789
+ // PRIORITY TIE-BREAK: order the eligible (TAG-filtered) candidate nodes by
790
+ // the mesh scheduling strategy. 'first_eligible' (default) returns them in
791
+ // config/array order unchanged, so distribution is strictly opt-in. The
792
+ // per-node MAX-ALLOC capacity gate (nodeHasActiveAssignment, provider cap,
793
+ // maxConcurrentSessions) is still applied inside the loop below; this only
794
+ // chooses which eligible node is *tried first*.
795
+ const strategy = resolveSchedulingStrategy(mesh);
796
+ const orderedCandidateNodes = strategy === 'first_eligible'
797
+ ? candidateNodes
798
+ : orderEligibleNodes(
799
+ meshId,
800
+ strategy,
801
+ candidateNodes
802
+ .map((node: any, index: number) => ({ nodeId: readMeshNodeId(node), node, index }))
803
+ .filter((c: RankableNode) => c.nodeId),
804
+ { bumpCursor: true },
805
+ ).map((c: RankableNode) => c.node);
806
+
807
+ for (const node of orderedCandidateNodes) {
704
808
  const nodeId = readMeshNodeId(node);
705
809
  if (!nodeId) continue;
706
810
  const launchKey = `${meshId}:${nodeId}`;
@@ -866,6 +970,19 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
866
970
  };
867
971
  }
868
972
 
973
+ // Collect every idle mesh session (local CLI instances + remote idle records)
974
+ // as drain candidates. The drain ORDER depends on the scheduling strategy:
975
+ // - 'first_eligible' (default): local-first, then remote, exactly as before.
976
+ // - otherwise: local + remote merged into one pool and drained in scheduling
977
+ // order (priority → load → tie-break). This local-first debias is required
978
+ // because without it the coordinator's own local node is always visited
979
+ // first and greedily absorbs all untargeted work before any remote idle
980
+ // session is even considered — the comparator alone can't spread work if
981
+ // local is always tried first.
982
+ type IdleCandidate = { nodeId: string; sessionId: string; providerType: string; origin: 'local' | 'remote'; node: any };
983
+ const strategy = resolveSchedulingStrategy(mesh);
984
+ const localCandidates: IdleCandidate[] = [];
985
+
869
986
  const cliInstances = components.instanceManager.getByCategory('cli');
870
987
  for (const inst of cliInstances) {
871
988
  const state = inst.getState();
@@ -893,7 +1010,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
893
1010
 
894
1011
  if (providerType) {
895
1012
  localIdleSessionsChecked += 1;
896
- tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
1013
+ localCandidates.push({ nodeId, sessionId, providerType, origin: 'local', node: mesh.nodes.find((n: any) => readMeshNodeId(n) === nodeId) });
897
1014
  } else {
898
1015
  skippedSessions.push({
899
1016
  nodeId,
@@ -908,16 +1025,55 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
908
1025
  remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
909
1026
  } catch { /* best-effort */ }
910
1027
 
1028
+ const remoteCandidates: IdleCandidate[] = [];
911
1029
  for (const idle of remoteSessions) {
912
1030
  const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
913
1031
  if (node) {
914
1032
  remoteIdleSessionsChecked += 1;
915
- const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
916
- if (assigned) {
917
- try {
918
- MeshRuntimeStore.getInstance().deleteRemoteIdleSession(idle.nodeId, idle.sessionId);
919
- } catch { /* best-effort */ }
920
- }
1033
+ remoteCandidates.push({ nodeId: idle.nodeId, sessionId: idle.sessionId, providerType: idle.providerType, origin: 'remote', node });
1034
+ }
1035
+ }
1036
+
1037
+ const assignIdleCandidate = (candidate: IdleCandidate): void => {
1038
+ const assigned = tryAssignQueueTask(components, meshId, candidate.nodeId, candidate.sessionId, candidate.providerType);
1039
+ if (assigned && candidate.origin === 'remote') {
1040
+ try {
1041
+ MeshRuntimeStore.getInstance().deleteRemoteIdleSession(candidate.nodeId, candidate.sessionId);
1042
+ } catch { /* best-effort */ }
1043
+ }
1044
+ };
1045
+
1046
+ if (strategy === 'first_eligible') {
1047
+ // Strict no-change: drain local idle sessions first (original order), then
1048
+ // remote idle sessions. tryAssignQueueTask is a no-op when nothing matches.
1049
+ for (const candidate of localCandidates) assignIdleCandidate(candidate);
1050
+ for (const candidate of remoteCandidates) assignIdleCandidate(candidate);
1051
+ } else {
1052
+ // Merge local + remote into one pool and drain in scheduling order. Each
1053
+ // assignment mutates a node's active load, and the next pick re-reads it,
1054
+ // so re-ranking after every assignment keeps the spread fair as load shifts.
1055
+ const pool = [...localCandidates, ...remoteCandidates];
1056
+ const baseIndex = new Map<string, number>();
1057
+ pool.forEach((c, i) => { if (!baseIndex.has(c.nodeId)) baseIndex.set(c.nodeId, i); });
1058
+ // Bump the round-robin cursor once for this whole drain pass.
1059
+ const uniqueNodes = [...new Set(pool.map(c => c.nodeId))]
1060
+ .map((nodeId, index) => ({ nodeId, node: pool.find(c => c.nodeId === nodeId)?.node, index }));
1061
+ const ranked = orderEligibleNodes(meshId, strategy, uniqueNodes, { bumpCursor: true });
1062
+ const rankIndex = new Map<string, number>(ranked.map((r, i) => [r.nodeId, i]));
1063
+ const remaining = [...pool];
1064
+ while (remaining.length > 0) {
1065
+ // Re-rank each pass so a node that just took work defers its next session.
1066
+ remaining.sort((a, b) => {
1067
+ const aPrio = resolveNodeSchedulingPriority(a.node?.policy);
1068
+ const bPrio = resolveNodeSchedulingPriority(b.node?.policy);
1069
+ if (aPrio !== bPrio) return bPrio - aPrio;
1070
+ if (strategy === 'least_loaded' || strategy === 'round_robin') {
1071
+ const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
1072
+ if (loadDelta !== 0) return loadDelta;
1073
+ }
1074
+ return (rankIndex.get(a.nodeId) ?? 0) - (rankIndex.get(b.nodeId) ?? 0);
1075
+ });
1076
+ assignIdleCandidate(remaining.shift()!);
921
1077
  }
922
1078
  }
923
1079
 
@@ -268,6 +268,17 @@ export class MeshRuntimeStore {
268
268
 
269
269
  CREATE INDEX IF NOT EXISTS idx_mesh_missions_mesh_status
270
270
  ON mesh_missions(mesh_id, status, updated_at);
271
+
272
+ -- Load-balancing scheduler: per-mesh round-robin rotation cursor. When
273
+ -- the schedulingStrategy is 'round_robin', several eligible nodes tied at
274
+ -- the least load are rotated by this cursor so the tie-break winner cycles
275
+ -- across scheduling passes instead of always favouring the same array-order
276
+ -- node. Persisted (not a module Map) so rotation survives daemon restarts
277
+ -- and stays a single source of truth across scheduling entry points.
278
+ CREATE TABLE IF NOT EXISTS mesh_scheduler_cursor (
279
+ mesh_id TEXT PRIMARY KEY,
280
+ cursor INTEGER NOT NULL DEFAULT 0
281
+ );
271
282
  `);
272
283
  }
273
284
 
@@ -479,6 +490,47 @@ export class MeshRuntimeStore {
479
490
  return row !== undefined;
480
491
  }
481
492
 
493
+ /**
494
+ * Count active (status='assigned') tasks on a node, regardless of provider or
495
+ * task mode. This is the load metric for least-loaded / round-robin ranking:
496
+ * the scheduler prefers the node with the fewest active assignments so
497
+ * untargeted work spreads instead of piling onto whichever node asks first.
498
+ */
499
+ nodeActiveAssignmentCount(meshId: string, nodeId: string): number {
500
+ const row = this.db.prepare(`
501
+ SELECT COUNT(*) as count FROM mesh_queue
502
+ WHERE mesh_id = ? AND status = 'assigned' AND assigned_node_id = ?
503
+ `).get(meshId, nodeId) as { count: number } | undefined;
504
+ return row?.count ?? 0;
505
+ }
506
+
507
+ /**
508
+ * Read the current per-mesh round-robin cursor (0 when unset). Used to rotate
509
+ * the tie-break winner among nodes tied at the least load.
510
+ */
511
+ getSchedulerCursor(meshId: string): number {
512
+ const row = this.db.prepare(
513
+ 'SELECT cursor FROM mesh_scheduler_cursor WHERE mesh_id = ?'
514
+ ).get(meshId) as { cursor: number } | undefined;
515
+ return row?.cursor ?? 0;
516
+ }
517
+
518
+ /**
519
+ * Atomically advance the per-mesh round-robin cursor by one and return the
520
+ * value that was current BEFORE the bump (the value the caller should rotate
521
+ * by for this pass). UPSERT keeps it lock-free across concurrent passes.
522
+ */
523
+ bumpSchedulerCursor(meshId: string): number {
524
+ return this.transaction(() => {
525
+ const current = this.getSchedulerCursor(meshId);
526
+ this.db.prepare(`
527
+ INSERT INTO mesh_scheduler_cursor (mesh_id, cursor) VALUES (?, ?)
528
+ ON CONFLICT(mesh_id) DO UPDATE SET cursor = excluded.cursor
529
+ `).run(meshId, current + 1);
530
+ return current;
531
+ });
532
+ }
533
+
482
534
  /**
483
535
  * Count active (status='assigned') tasks on a (node, provider) combination,
484
536
  * matched by the assignedProviderType stamped on the payload at claim time.