@adhdev/daemon-core 0.9.82-rc.310 → 0.9.82-rc.312
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +19 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.js +1070 -263
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1069 -270
- package/dist/index.mjs.map +1 -1
- package/dist/logging/log-redactor.d.ts +24 -0
- package/dist/logging/log-tail-reader.d.ts +46 -0
- package/dist/mesh/mesh-events-coordinator.d.ts +31 -0
- package/dist/mesh/mesh-runtime-store.d.ts +18 -0
- package/dist/mesh/mesh-work-queue.d.ts +23 -0
- package/dist/providers/spec/cli-adapter.d.ts +34 -3
- package/dist/providers/spec/types.d.ts +36 -0
- package/dist/repo-mesh-types.d.ts +103 -9
- package/package.json +2 -2
- package/src/commands/chat-commands.ts +10 -2
- package/src/commands/router.ts +323 -6
- package/src/commands/stream-commands.ts +8 -0
- package/src/config/chat-history.ts +9 -0
- package/src/config/mesh-config.ts +17 -1
- package/src/index.ts +16 -2
- package/src/logging/log-redactor.ts +100 -0
- package/src/logging/log-tail-reader.ts +220 -0
- package/src/mesh/coordinator-prompt.ts +1 -0
- package/src/mesh/mesh-events-coordinator.ts +165 -9
- package/src/mesh/mesh-runtime-store.ts +52 -0
- package/src/mesh/mesh-work-queue.ts +105 -1
- package/src/providers/spec/cli-adapter.ts +155 -13
- package/src/providers/spec/fsm-driver.ts +14 -1
- package/src/providers/spec/native-history-executor.ts +114 -22
- package/src/providers/spec/types.ts +37 -0
- package/src/repo-mesh-types.ts +134 -9
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Log redactor — mask secrets before a raw daemon log line leaves the machine.
|
|
3
|
+
*
|
|
4
|
+
* Daemon logs can incidentally contain credentials: ADHDev API keys (adk_*),
|
|
5
|
+
* machine secrets (adm_*), provider keys (adp_*), bearer tokens, JWTs, TURN
|
|
6
|
+
* `username:credential` pairs, and `SECRET=...` style env dumps. The mesh
|
|
7
|
+
* `get_mesh_node_logs` command ships a log tail over P2P to the coordinator, so
|
|
8
|
+
* every line MUST pass through redactLogLine() first — otherwise a secret in a
|
|
9
|
+
* remote daemon's log is exfiltrated to whoever is driving the coordinator.
|
|
10
|
+
*
|
|
11
|
+
* Patterns are intentionally conservative: each masks the secret material while
|
|
12
|
+
* preserving enough surrounding shape that the line stays useful for debugging
|
|
13
|
+
* (e.g. `adk_••••1234`, `Bearer ••••redacted`). When in doubt, mask.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const MASK = '••••redacted';
|
|
17
|
+
|
|
18
|
+
/** Keep the last 4 chars of a token so logs stay correlatable without leaking it. */
|
|
19
|
+
function maskKeepTail(token: string): string {
|
|
20
|
+
if (token.length <= 8) return MASK;
|
|
21
|
+
return `${MASK}${token.slice(-4)}`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface RedactionRule {
|
|
25
|
+
readonly name: string;
|
|
26
|
+
readonly pattern: RegExp;
|
|
27
|
+
readonly replace: (match: string, ...groups: string[]) => string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// NOTE: order matters — more specific rules (key=value, Bearer, TURN) run before
|
|
31
|
+
// the bare-token rules so the structured forms aren't half-masked by a greedy
|
|
32
|
+
// generic rule.
|
|
33
|
+
const RULES: RedactionRule[] = [
|
|
34
|
+
// `JWT_SECRET=...`, `TOKEN=...`, `API_KEY=...`, `password: ...` env/config dumps.
|
|
35
|
+
// Captures the key + delimiter and masks only the value.
|
|
36
|
+
{
|
|
37
|
+
name: 'key_value_secret',
|
|
38
|
+
pattern: /\b([A-Z0-9_]*(?:SECRET|TOKEN|API[_-]?KEY|PASSWORD|PASSWD|PRIVATE[_-]?KEY|CREDENTIAL|CLIENT[_-]?SECRET)[A-Z0-9_]*)(\s*[:=]\s*)(["']?)([^\s"',;]+)\3/gi,
|
|
39
|
+
replace: (_m, key: string, delim: string, quote: string) => `${key}${delim}${quote}${MASK}${quote}`,
|
|
40
|
+
},
|
|
41
|
+
// Authorization: Bearer <token>
|
|
42
|
+
{
|
|
43
|
+
name: 'bearer_token',
|
|
44
|
+
pattern: /\b(Bearer\s+)([A-Za-z0-9._\-+/=]{8,})/g,
|
|
45
|
+
replace: (_m, prefix: string, token: string) => `${prefix}${maskKeepTail(token)}`,
|
|
46
|
+
},
|
|
47
|
+
// ADHDev credential prefixes: API key (adk_), machine secret (adm_), provider key (adp_).
|
|
48
|
+
{
|
|
49
|
+
name: 'adhdev_prefixed_secret',
|
|
50
|
+
pattern: /\b(ad[kmp]_)([A-Za-z0-9]{6,})/g,
|
|
51
|
+
replace: (_m, prefix: string, token: string) => `${prefix}${maskKeepTail(prefix + token)}`,
|
|
52
|
+
},
|
|
53
|
+
// JWT: three base64url segments separated by dots, header starts with eyJ.
|
|
54
|
+
{
|
|
55
|
+
name: 'jwt',
|
|
56
|
+
pattern: /\beyJ[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}\.[A-Za-z0-9_-]{4,}/g,
|
|
57
|
+
replace: () => MASK,
|
|
58
|
+
},
|
|
59
|
+
// TURN credential: a long credential value following a `credential` key in
|
|
60
|
+
// any common shape — `credential: x`, `credential=x`, or `credential "x"`.
|
|
61
|
+
// Mask the credential value only, preserving the key + delimiter/quote.
|
|
62
|
+
{
|
|
63
|
+
name: 'turn_credential',
|
|
64
|
+
pattern: /\b(credential["']?\s*(?:[:=]\s*)?["']?)([^\s"',;]{6,})/gi,
|
|
65
|
+
replace: (_m, prefix: string) => `${prefix}${MASK}`,
|
|
66
|
+
},
|
|
67
|
+
// TURN REST username:credential of the form `<expiry-ts>:<base64hmac>`,
|
|
68
|
+
// where the hmac part is long base64. Mask the hmac.
|
|
69
|
+
{
|
|
70
|
+
name: 'turn_rest_pair',
|
|
71
|
+
pattern: /\b(\d{10,}:)([A-Za-z0-9+/]{20,}={0,2})\b/g,
|
|
72
|
+
replace: (_m, prefix: string) => `${prefix}${MASK}`,
|
|
73
|
+
},
|
|
74
|
+
];
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Mask secrets in a single log line. Idempotent-ish: re-running over an
|
|
78
|
+
* already-masked line leaves the MASK token in place (it contains no secret
|
|
79
|
+
* shape). Never throws — a redaction failure must not crash the log path.
|
|
80
|
+
*/
|
|
81
|
+
export function redactLogLine(line: string): string {
|
|
82
|
+
if (!line) return line;
|
|
83
|
+
let out = line;
|
|
84
|
+
for (const rule of RULES) {
|
|
85
|
+
try {
|
|
86
|
+
out = out.replace(rule.pattern, rule.replace as (substring: string, ...args: any[]) => string);
|
|
87
|
+
} catch {
|
|
88
|
+
// A pathological line must never break log shipping — skip this rule.
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** Redact an array of log lines in place-safe fashion (returns a new array). */
|
|
95
|
+
export function redactLogLines(lines: string[]): string[] {
|
|
96
|
+
return lines.map((line) => redactLogLine(line));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Exposed for tests/introspection: the rule names applied, in order. */
|
|
100
|
+
export const LOG_REDACTION_RULE_NAMES: readonly string[] = RULES.map((r) => r.name);
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Daemon log tail reader — read the last N bytes of a daemon log file, newest
|
|
3
|
+
* bytes first, bounded so the result is safe to ship over a mesh P2P channel.
|
|
4
|
+
*
|
|
5
|
+
* Used by the mesh `get_mesh_node_logs` command: the coordinator asks a (possibly
|
|
6
|
+
* remote) daemon for its recent log tail instead of having to open a session and
|
|
7
|
+
* grep the file by hand. Because the mesh RPC envelope is sent as a single
|
|
8
|
+
* datachannel message (~256KB SCTP ceiling, no chunking), the returned tail is
|
|
9
|
+
* HARD-bounded by `tailBytes` (default 64KB, capped at MAX_TAIL_BYTES=128KB) and
|
|
10
|
+
* flags `truncated:true` when the file was larger.
|
|
11
|
+
*
|
|
12
|
+
* Boundary-safe: lines are cut on the newline byte (0x0A) only, which never
|
|
13
|
+
* appears inside a multibyte UTF-8 sequence, so decoding each complete byte
|
|
14
|
+
* segment never splits a multibyte char.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import * as fs from 'fs';
|
|
18
|
+
import { getCurrentDaemonLogPath, getDaemonLogDir } from './logger.js';
|
|
19
|
+
|
|
20
|
+
export const DEFAULT_TAIL_BYTES = 64 * 1024;
|
|
21
|
+
export const MAX_TAIL_BYTES = 128 * 1024;
|
|
22
|
+
const READ_CHUNK_BYTES = 64 * 1024;
|
|
23
|
+
|
|
24
|
+
export interface ReadDaemonLogTailArgs {
|
|
25
|
+
/** Date of the log file to read (defaults to today). YYYY-MM-DD string or Date. */
|
|
26
|
+
date?: string | Date;
|
|
27
|
+
/** Max bytes of tail to return. Clamped to (0, MAX_TAIL_BYTES]. Default 64KB. */
|
|
28
|
+
tailBytes?: number;
|
|
29
|
+
/** Optional regex source string; only lines matching (case-insensitive) are kept. */
|
|
30
|
+
grep?: string;
|
|
31
|
+
/** Optional epoch-ms floor; only lines whose leading [HH:MM:SS...] / ISO ts >= this are kept. */
|
|
32
|
+
sinceMs?: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface DaemonLogTailResult {
|
|
36
|
+
success: boolean;
|
|
37
|
+
error?: string;
|
|
38
|
+
lines: string[];
|
|
39
|
+
truncated: boolean;
|
|
40
|
+
logPath: string;
|
|
41
|
+
platform: NodeJS.Platform;
|
|
42
|
+
bytesReturned: number;
|
|
43
|
+
/** True when a grep/since filter dropped lines from the raw tail window. */
|
|
44
|
+
filtered: boolean;
|
|
45
|
+
/** The grep source actually applied (echoed back for clarity). */
|
|
46
|
+
grep?: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function resolveLogPath(date?: string | Date): string {
|
|
50
|
+
if (date instanceof Date) return getCurrentDaemonLogPath(date);
|
|
51
|
+
if (typeof date === 'string' && date.trim()) {
|
|
52
|
+
const parsed = new Date(`${date.trim()}T00:00:00.000Z`);
|
|
53
|
+
if (!Number.isNaN(parsed.getTime())) return getCurrentDaemonLogPath(parsed);
|
|
54
|
+
}
|
|
55
|
+
return getCurrentDaemonLogPath();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function clampTailBytes(tailBytes?: number): number {
|
|
59
|
+
if (!Number.isFinite(tailBytes) || (tailBytes as number) <= 0) return DEFAULT_TAIL_BYTES;
|
|
60
|
+
return Math.min(Math.floor(tailBytes as number), MAX_TAIL_BYTES);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Read up to `limitBytes` from the end of `filePath`, on a UTF-8 line boundary.
|
|
65
|
+
* Returns the decoded text, whether the read was truncated (file bigger than the
|
|
66
|
+
* window), and the number of bytes actually decoded.
|
|
67
|
+
*/
|
|
68
|
+
function readByteBoundedTail(filePath: string, limitBytes: number): { text: string; truncated: boolean; bytesReturned: number } {
|
|
69
|
+
const fd = fs.openSync(filePath, 'r');
|
|
70
|
+
try {
|
|
71
|
+
const stat = fs.fstatSync(fd);
|
|
72
|
+
const size = stat.size;
|
|
73
|
+
if (size === 0) return { text: '', truncated: false, bytesReturned: 0 };
|
|
74
|
+
|
|
75
|
+
const want = Math.min(limitBytes, size);
|
|
76
|
+
let start = size - want;
|
|
77
|
+
const truncated = start > 0;
|
|
78
|
+
|
|
79
|
+
// Collect chunks newest-last into a buffer covering [start, size).
|
|
80
|
+
const buffers: Buffer[] = [];
|
|
81
|
+
let position = start;
|
|
82
|
+
while (position < size) {
|
|
83
|
+
const chunkSize = Math.min(READ_CHUNK_BYTES, size - position);
|
|
84
|
+
const chunk = Buffer.alloc(chunkSize);
|
|
85
|
+
fs.readSync(fd, chunk, 0, chunkSize, position);
|
|
86
|
+
buffers.push(chunk);
|
|
87
|
+
position += chunkSize;
|
|
88
|
+
}
|
|
89
|
+
let buf = Buffer.concat(buffers);
|
|
90
|
+
|
|
91
|
+
// If we truncated mid-line, drop the leading partial line so we never emit
|
|
92
|
+
// a half-decoded line (and never split a multibyte char at the window edge).
|
|
93
|
+
if (truncated) {
|
|
94
|
+
const firstNewline = buf.indexOf(0x0a);
|
|
95
|
+
if (firstNewline >= 0) {
|
|
96
|
+
buf = buf.subarray(firstNewline + 1);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return { text: buf.toString('utf-8'), truncated, bytesReturned: buf.length };
|
|
100
|
+
} finally {
|
|
101
|
+
fs.closeSync(fd);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Parse a leading timestamp from a log line into epoch ms. The unified logger
|
|
106
|
+
// writes `[HH:MM:SS.mmm]` (local time, today's date) and the startup banner uses
|
|
107
|
+
// a full timestamp; we best-effort parse `[HH:MM:SS...]` against the file's date.
|
|
108
|
+
// Returns null when no timestamp can be extracted (line is then kept by sinceMs).
|
|
109
|
+
function parseLineEpochMs(line: string, fileDate: Date): number | null {
|
|
110
|
+
const m = line.match(/^\[(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,3}))?\]/);
|
|
111
|
+
if (!m) {
|
|
112
|
+
// Try an embedded ISO timestamp as a fallback.
|
|
113
|
+
const iso = line.match(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?/);
|
|
114
|
+
if (iso) {
|
|
115
|
+
const t = Date.parse(iso[0].replace(' ', 'T'));
|
|
116
|
+
return Number.isNaN(t) ? null : t;
|
|
117
|
+
}
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
const d = new Date(fileDate);
|
|
121
|
+
d.setHours(Number(m[1]), Number(m[2]), Number(m[3]), m[4] ? Number(m[4].padEnd(3, '0')) : 0);
|
|
122
|
+
return d.getTime();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Read the daemon log tail for `date` (default today), bounded to `tailBytes`,
|
|
127
|
+
* with optional grep (regex source) and sinceMs filters. Falls back to the
|
|
128
|
+
* size-rotation backup (`*.1.log`) when the primary file does not exist.
|
|
129
|
+
*/
|
|
130
|
+
export function readDaemonLogTail(args: ReadDaemonLogTailArgs = {}): DaemonLogTailResult {
|
|
131
|
+
const platform = process.platform;
|
|
132
|
+
const limitBytes = clampTailBytes(args.tailBytes);
|
|
133
|
+
let logPath = resolveLogPath(args.date);
|
|
134
|
+
|
|
135
|
+
// Fall back to the size-rotation backup if the active file is absent.
|
|
136
|
+
if (!fs.existsSync(logPath)) {
|
|
137
|
+
const backup = logPath.replace(/\.log$/, '.1.log');
|
|
138
|
+
if (fs.existsSync(backup)) {
|
|
139
|
+
logPath = backup;
|
|
140
|
+
} else {
|
|
141
|
+
return {
|
|
142
|
+
success: false,
|
|
143
|
+
error: `No daemon log file at ${logPath} (dir: ${getDaemonLogDir()})`,
|
|
144
|
+
lines: [],
|
|
145
|
+
truncated: false,
|
|
146
|
+
logPath,
|
|
147
|
+
platform,
|
|
148
|
+
bytesReturned: 0,
|
|
149
|
+
filtered: false,
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
let raw: { text: string; truncated: boolean; bytesReturned: number };
|
|
155
|
+
try {
|
|
156
|
+
raw = readByteBoundedTail(logPath, limitBytes);
|
|
157
|
+
} catch (e: any) {
|
|
158
|
+
return {
|
|
159
|
+
success: false,
|
|
160
|
+
error: `Failed to read ${logPath}: ${e?.message ?? String(e)}`,
|
|
161
|
+
lines: [],
|
|
162
|
+
truncated: false,
|
|
163
|
+
logPath,
|
|
164
|
+
platform,
|
|
165
|
+
bytesReturned: 0,
|
|
166
|
+
filtered: false,
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let lines = raw.text.split('\n');
|
|
171
|
+
// A trailing newline yields a final empty element — drop it.
|
|
172
|
+
if (lines.length && lines[lines.length - 1] === '') lines.pop();
|
|
173
|
+
const rawCount = lines.length;
|
|
174
|
+
|
|
175
|
+
// since filter
|
|
176
|
+
if (Number.isFinite(args.sinceMs)) {
|
|
177
|
+
const fileDate = args.date instanceof Date
|
|
178
|
+
? args.date
|
|
179
|
+
: typeof args.date === 'string' && args.date.trim()
|
|
180
|
+
? new Date(`${args.date.trim()}T00:00:00.000Z`)
|
|
181
|
+
: new Date();
|
|
182
|
+
const floor = args.sinceMs as number;
|
|
183
|
+
lines = lines.filter((line) => {
|
|
184
|
+
const ts = parseLineEpochMs(line, fileDate);
|
|
185
|
+
// Keep lines with no parseable timestamp (continuation/stack lines).
|
|
186
|
+
return ts === null || ts >= floor;
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// grep filter
|
|
191
|
+
let appliedGrep: string | undefined;
|
|
192
|
+
if (typeof args.grep === 'string' && args.grep.trim()) {
|
|
193
|
+
appliedGrep = args.grep.trim();
|
|
194
|
+
let re: RegExp | null = null;
|
|
195
|
+
try {
|
|
196
|
+
re = new RegExp(appliedGrep, 'i');
|
|
197
|
+
} catch {
|
|
198
|
+
re = null;
|
|
199
|
+
}
|
|
200
|
+
if (re) {
|
|
201
|
+
const compiled = re;
|
|
202
|
+
lines = lines.filter((line) => compiled.test(line));
|
|
203
|
+
} else {
|
|
204
|
+
// Invalid regex → fall back to a literal substring match.
|
|
205
|
+
const needle = appliedGrep.toLowerCase();
|
|
206
|
+
lines = lines.filter((line) => line.toLowerCase().includes(needle));
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
success: true,
|
|
212
|
+
lines,
|
|
213
|
+
truncated: raw.truncated,
|
|
214
|
+
logPath,
|
|
215
|
+
platform,
|
|
216
|
+
bytesReturned: raw.bytesReturned,
|
|
217
|
+
filtered: lines.length !== rawCount,
|
|
218
|
+
...(appliedGrep ? { grep: appliedGrep } : {}),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
@@ -342,6 +342,7 @@ const TOOLS_SECTION = `## Available Tools
|
|
|
342
342
|
| \`mesh_read_debug\` | Collect a daemon-side chat/parser debug bundle for a session |
|
|
343
343
|
| \`mesh_task_history\` | Read the task ledger — dispatches, completions, failures. Use to understand what has been done before deciding next steps |
|
|
344
344
|
| \`mesh_git_status\` | Check git status on a specific node |
|
|
345
|
+
| \`mesh_read_node_logs\` | Fetch a remote node's daemon log tail directly over P2P (grep/since/byte-bounded, secrets redacted) — no session/PowerShell needed to debug a node's daemon |
|
|
345
346
|
| \`mesh_fast_forward_node\` | Safely dry-run or explicitly execute an obvious clean fast-forward without launching an agent session |
|
|
346
347
|
| \`mesh_checkpoint\` | Create a git checkpoint on a node |
|
|
347
348
|
| \`mesh_approve\` | Approve/reject a pending agent action |
|
|
@@ -14,7 +14,8 @@ import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } f
|
|
|
14
14
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
15
|
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
16
16
|
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
17
|
-
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel } from '../repo-mesh-types.js';
|
|
17
|
+
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
|
+
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
19
|
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
19
20
|
import {
|
|
20
21
|
findRecentTerminalLedgerEvidence,
|
|
@@ -499,6 +500,91 @@ function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
|
499
500
|
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
500
501
|
}
|
|
501
502
|
|
|
503
|
+
/** Active (status='assigned') task count for a node — the load metric for
|
|
504
|
+
* least-loaded / round-robin ranking. Lower = preferred. */
|
|
505
|
+
function nodeActiveLoad(meshId: string, nodeId: string): number {
|
|
506
|
+
return MeshRuntimeStore.getInstance().nodeActiveAssignmentCount(meshId, nodeId);
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* The mesh-wide scheduling strategy. Defaults to 'first_eligible' (strict
|
|
511
|
+
* no-change) for any mesh that does not set it. Only governs the final tie-break;
|
|
512
|
+
* eligibility, capacity, and priority gates apply identically to every strategy.
|
|
513
|
+
*/
|
|
514
|
+
function resolveSchedulingStrategy(mesh: any): RepoMeshSchedulingStrategy {
|
|
515
|
+
return normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Order eligible nodes for assignment per the mesh scheduling pipeline:
|
|
520
|
+
* PRIORITY (schedulingPriority desc) → TIE-BREAK (strategy).
|
|
521
|
+
*
|
|
522
|
+
* The caller has already applied the TAG hard-filter and is responsible for the
|
|
523
|
+
* MAX-ALLOC capacity gate (the per-node launch/claim checks). This function only
|
|
524
|
+
* decides the *preference order* among nodes that are otherwise eligible.
|
|
525
|
+
*
|
|
526
|
+
* - 'first_eligible' (default): returns the input order verbatim and does NOT touch
|
|
527
|
+
* the round-robin cursor — byte-for-byte the pre-feature behavior.
|
|
528
|
+
* - 'priority_only': schedulingPriority desc, then input order (load ignored).
|
|
529
|
+
* - 'least_loaded': schedulingPriority desc, then active load asc, then input order.
|
|
530
|
+
* - 'round_robin': same as least_loaded, but among nodes tied at (priority, load)
|
|
531
|
+
* the input order is rotated by a per-mesh cursor that advances once per pass.
|
|
532
|
+
*
|
|
533
|
+
* `nodes` carries the original config/array index so the tie-break can fall back to
|
|
534
|
+
* deterministic input order. `bumpCursor` advances the round-robin cursor exactly
|
|
535
|
+
* once per scheduling pass (only consulted for 'round_robin').
|
|
536
|
+
*/
|
|
537
|
+
interface RankableNode { nodeId: string; node: any; index: number }
|
|
538
|
+
|
|
539
|
+
/** Test-only: the pure node-ordering stage (PRIORITY → TIE-BREAK). Exposed so the
|
|
540
|
+
* scheduling pipeline can be unit-tested without standing up live CLI sessions. */
|
|
541
|
+
export function __orderEligibleNodesForTests(
|
|
542
|
+
meshId: string,
|
|
543
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
544
|
+
nodes: RankableNode[],
|
|
545
|
+
opts?: { bumpCursor?: boolean },
|
|
546
|
+
): RankableNode[] {
|
|
547
|
+
return orderEligibleNodes(meshId, strategy, nodes, opts);
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
function orderEligibleNodes(
|
|
551
|
+
meshId: string,
|
|
552
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
553
|
+
nodes: RankableNode[],
|
|
554
|
+
opts?: { bumpCursor?: boolean },
|
|
555
|
+
): RankableNode[] {
|
|
556
|
+
if (strategy === 'first_eligible' || nodes.length <= 1) {
|
|
557
|
+
return nodes;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
const priorityOf = (n: { node: any }) => resolveNodeSchedulingPriority(n.node?.policy);
|
|
561
|
+
|
|
562
|
+
// Round-robin rotation offset: rotate the deterministic input order by a
|
|
563
|
+
// per-mesh cursor so the tie-break winner among equal (priority, load) nodes
|
|
564
|
+
// cycles across passes. The cursor advances once per scheduling pass.
|
|
565
|
+
let rotation = 0;
|
|
566
|
+
if (strategy === 'round_robin') {
|
|
567
|
+
const cursor = opts?.bumpCursor
|
|
568
|
+
? MeshRuntimeStore.getInstance().bumpSchedulerCursor(meshId)
|
|
569
|
+
: MeshRuntimeStore.getInstance().getSchedulerCursor(meshId);
|
|
570
|
+
rotation = ((cursor % nodes.length) + nodes.length) % nodes.length;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
// Rotation rank: position of each node after rotating input order by `rotation`.
|
|
574
|
+
// For non-round-robin strategies rotation is 0, so this is just the input index.
|
|
575
|
+
const rotationRank = (index: number) => (index - rotation + nodes.length) % nodes.length;
|
|
576
|
+
|
|
577
|
+
return [...nodes].sort((a, b) => {
|
|
578
|
+
const prioDelta = priorityOf(b) - priorityOf(a); // higher priority first
|
|
579
|
+
if (prioDelta !== 0) return prioDelta;
|
|
580
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
581
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
582
|
+
if (loadDelta !== 0) return loadDelta;
|
|
583
|
+
}
|
|
584
|
+
return rotationRank(a.index) - rotationRank(b.index);
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
|
|
502
588
|
/** Active assignments on a (node, provider) — pre-launch guard for the per-(node,
|
|
503
589
|
* provider) maxParallel cap. The authoritative enforcement is in the claim
|
|
504
590
|
* transaction; this only avoids spawning a session that would fail the claim. */
|
|
@@ -700,7 +786,25 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
700
786
|
continue;
|
|
701
787
|
}
|
|
702
788
|
|
|
703
|
-
|
|
789
|
+
// PRIORITY → TIE-BREAK: order the eligible (TAG-filtered) candidate nodes by
|
|
790
|
+
// the mesh scheduling strategy. 'first_eligible' (default) returns them in
|
|
791
|
+
// config/array order unchanged, so distribution is strictly opt-in. The
|
|
792
|
+
// per-node MAX-ALLOC capacity gate (nodeHasActiveAssignment, provider cap,
|
|
793
|
+
// maxConcurrentSessions) is still applied inside the loop below; this only
|
|
794
|
+
// chooses which eligible node is *tried first*.
|
|
795
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
796
|
+
const orderedCandidateNodes = strategy === 'first_eligible'
|
|
797
|
+
? candidateNodes
|
|
798
|
+
: orderEligibleNodes(
|
|
799
|
+
meshId,
|
|
800
|
+
strategy,
|
|
801
|
+
candidateNodes
|
|
802
|
+
.map((node: any, index: number) => ({ nodeId: readMeshNodeId(node), node, index }))
|
|
803
|
+
.filter((c: RankableNode) => c.nodeId),
|
|
804
|
+
{ bumpCursor: true },
|
|
805
|
+
).map((c: RankableNode) => c.node);
|
|
806
|
+
|
|
807
|
+
for (const node of orderedCandidateNodes) {
|
|
704
808
|
const nodeId = readMeshNodeId(node);
|
|
705
809
|
if (!nodeId) continue;
|
|
706
810
|
const launchKey = `${meshId}:${nodeId}`;
|
|
@@ -866,6 +970,19 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
866
970
|
};
|
|
867
971
|
}
|
|
868
972
|
|
|
973
|
+
// Collect every idle mesh session (local CLI instances + remote idle records)
|
|
974
|
+
// as drain candidates. The drain ORDER depends on the scheduling strategy:
|
|
975
|
+
// - 'first_eligible' (default): local-first, then remote, exactly as before.
|
|
976
|
+
// - otherwise: local + remote merged into one pool and drained in scheduling
|
|
977
|
+
// order (priority → load → tie-break). This local-first debias is required
|
|
978
|
+
// because without it the coordinator's own local node is always visited
|
|
979
|
+
// first and greedily absorbs all untargeted work before any remote idle
|
|
980
|
+
// session is even considered — the comparator alone can't spread work if
|
|
981
|
+
// local is always tried first.
|
|
982
|
+
type IdleCandidate = { nodeId: string; sessionId: string; providerType: string; origin: 'local' | 'remote'; node: any };
|
|
983
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
984
|
+
const localCandidates: IdleCandidate[] = [];
|
|
985
|
+
|
|
869
986
|
const cliInstances = components.instanceManager.getByCategory('cli');
|
|
870
987
|
for (const inst of cliInstances) {
|
|
871
988
|
const state = inst.getState();
|
|
@@ -893,7 +1010,7 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
893
1010
|
|
|
894
1011
|
if (providerType) {
|
|
895
1012
|
localIdleSessionsChecked += 1;
|
|
896
|
-
|
|
1013
|
+
localCandidates.push({ nodeId, sessionId, providerType, origin: 'local', node: mesh.nodes.find((n: any) => readMeshNodeId(n) === nodeId) });
|
|
897
1014
|
} else {
|
|
898
1015
|
skippedSessions.push({
|
|
899
1016
|
nodeId,
|
|
@@ -908,16 +1025,55 @@ export async function triggerMeshQueue(components: DaemonComponents, meshId: str
|
|
|
908
1025
|
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
909
1026
|
} catch { /* best-effort */ }
|
|
910
1027
|
|
|
1028
|
+
const remoteCandidates: IdleCandidate[] = [];
|
|
911
1029
|
for (const idle of remoteSessions) {
|
|
912
1030
|
const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
|
|
913
1031
|
if (node) {
|
|
914
1032
|
remoteIdleSessionsChecked += 1;
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
1033
|
+
remoteCandidates.push({ nodeId: idle.nodeId, sessionId: idle.sessionId, providerType: idle.providerType, origin: 'remote', node });
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
const assignIdleCandidate = (candidate: IdleCandidate): void => {
|
|
1038
|
+
const assigned = tryAssignQueueTask(components, meshId, candidate.nodeId, candidate.sessionId, candidate.providerType);
|
|
1039
|
+
if (assigned && candidate.origin === 'remote') {
|
|
1040
|
+
try {
|
|
1041
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(candidate.nodeId, candidate.sessionId);
|
|
1042
|
+
} catch { /* best-effort */ }
|
|
1043
|
+
}
|
|
1044
|
+
};
|
|
1045
|
+
|
|
1046
|
+
if (strategy === 'first_eligible') {
|
|
1047
|
+
// Strict no-change: drain local idle sessions first (original order), then
|
|
1048
|
+
// remote idle sessions. tryAssignQueueTask is a no-op when nothing matches.
|
|
1049
|
+
for (const candidate of localCandidates) assignIdleCandidate(candidate);
|
|
1050
|
+
for (const candidate of remoteCandidates) assignIdleCandidate(candidate);
|
|
1051
|
+
} else {
|
|
1052
|
+
// Merge local + remote into one pool and drain in scheduling order. Each
|
|
1053
|
+
// assignment mutates a node's active load, and the next pick re-reads it,
|
|
1054
|
+
// so re-ranking after every assignment keeps the spread fair as load shifts.
|
|
1055
|
+
const pool = [...localCandidates, ...remoteCandidates];
|
|
1056
|
+
const baseIndex = new Map<string, number>();
|
|
1057
|
+
pool.forEach((c, i) => { if (!baseIndex.has(c.nodeId)) baseIndex.set(c.nodeId, i); });
|
|
1058
|
+
// Bump the round-robin cursor once for this whole drain pass.
|
|
1059
|
+
const uniqueNodes = [...new Set(pool.map(c => c.nodeId))]
|
|
1060
|
+
.map((nodeId, index) => ({ nodeId, node: pool.find(c => c.nodeId === nodeId)?.node, index }));
|
|
1061
|
+
const ranked = orderEligibleNodes(meshId, strategy, uniqueNodes, { bumpCursor: true });
|
|
1062
|
+
const rankIndex = new Map<string, number>(ranked.map((r, i) => [r.nodeId, i]));
|
|
1063
|
+
const remaining = [...pool];
|
|
1064
|
+
while (remaining.length > 0) {
|
|
1065
|
+
// Re-rank each pass so a node that just took work defers its next session.
|
|
1066
|
+
remaining.sort((a, b) => {
|
|
1067
|
+
const aPrio = resolveNodeSchedulingPriority(a.node?.policy);
|
|
1068
|
+
const bPrio = resolveNodeSchedulingPriority(b.node?.policy);
|
|
1069
|
+
if (aPrio !== bPrio) return bPrio - aPrio;
|
|
1070
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
1071
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
1072
|
+
if (loadDelta !== 0) return loadDelta;
|
|
1073
|
+
}
|
|
1074
|
+
return (rankIndex.get(a.nodeId) ?? 0) - (rankIndex.get(b.nodeId) ?? 0);
|
|
1075
|
+
});
|
|
1076
|
+
assignIdleCandidate(remaining.shift()!);
|
|
921
1077
|
}
|
|
922
1078
|
}
|
|
923
1079
|
|
|
@@ -268,6 +268,17 @@ export class MeshRuntimeStore {
|
|
|
268
268
|
|
|
269
269
|
CREATE INDEX IF NOT EXISTS idx_mesh_missions_mesh_status
|
|
270
270
|
ON mesh_missions(mesh_id, status, updated_at);
|
|
271
|
+
|
|
272
|
+
-- Load-balancing scheduler: per-mesh round-robin rotation cursor. When
|
|
273
|
+
-- the schedulingStrategy is 'round_robin', several eligible nodes tied at
|
|
274
|
+
-- the least load are rotated by this cursor so the tie-break winner cycles
|
|
275
|
+
-- across scheduling passes instead of always favouring the same array-order
|
|
276
|
+
-- node. Persisted (not a module Map) so rotation survives daemon restarts
|
|
277
|
+
-- and stays a single source of truth across scheduling entry points.
|
|
278
|
+
CREATE TABLE IF NOT EXISTS mesh_scheduler_cursor (
|
|
279
|
+
mesh_id TEXT PRIMARY KEY,
|
|
280
|
+
cursor INTEGER NOT NULL DEFAULT 0
|
|
281
|
+
);
|
|
271
282
|
`);
|
|
272
283
|
}
|
|
273
284
|
|
|
@@ -479,6 +490,47 @@ export class MeshRuntimeStore {
|
|
|
479
490
|
return row !== undefined;
|
|
480
491
|
}
|
|
481
492
|
|
|
493
|
+
/**
|
|
494
|
+
* Count active (status='assigned') tasks on a node, regardless of provider or
|
|
495
|
+
* task mode. This is the load metric for least-loaded / round-robin ranking:
|
|
496
|
+
* the scheduler prefers the node with the fewest active assignments so
|
|
497
|
+
* untargeted work spreads instead of piling onto whichever node asks first.
|
|
498
|
+
*/
|
|
499
|
+
nodeActiveAssignmentCount(meshId: string, nodeId: string): number {
|
|
500
|
+
const row = this.db.prepare(`
|
|
501
|
+
SELECT COUNT(*) as count FROM mesh_queue
|
|
502
|
+
WHERE mesh_id = ? AND status = 'assigned' AND assigned_node_id = ?
|
|
503
|
+
`).get(meshId, nodeId) as { count: number } | undefined;
|
|
504
|
+
return row?.count ?? 0;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Read the current per-mesh round-robin cursor (0 when unset). Used to rotate
|
|
509
|
+
* the tie-break winner among nodes tied at the least load.
|
|
510
|
+
*/
|
|
511
|
+
getSchedulerCursor(meshId: string): number {
|
|
512
|
+
const row = this.db.prepare(
|
|
513
|
+
'SELECT cursor FROM mesh_scheduler_cursor WHERE mesh_id = ?'
|
|
514
|
+
).get(meshId) as { cursor: number } | undefined;
|
|
515
|
+
return row?.cursor ?? 0;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* Atomically advance the per-mesh round-robin cursor by one and return the
|
|
520
|
+
* value that was current BEFORE the bump (the value the caller should rotate
|
|
521
|
+
* by for this pass). UPSERT keeps it lock-free across concurrent passes.
|
|
522
|
+
*/
|
|
523
|
+
bumpSchedulerCursor(meshId: string): number {
|
|
524
|
+
return this.transaction(() => {
|
|
525
|
+
const current = this.getSchedulerCursor(meshId);
|
|
526
|
+
this.db.prepare(`
|
|
527
|
+
INSERT INTO mesh_scheduler_cursor (mesh_id, cursor) VALUES (?, ?)
|
|
528
|
+
ON CONFLICT(mesh_id) DO UPDATE SET cursor = excluded.cursor
|
|
529
|
+
`).run(meshId, current + 1);
|
|
530
|
+
return current;
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
|
|
482
534
|
/**
|
|
483
535
|
* Count active (status='assigned') tasks on a (node, provider) combination,
|
|
484
536
|
* matched by the assignedProviderType stamped on the payload at claim time.
|