c8ctl-plugin-nano 1.44.3 → 1.44.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -4
- package/c8ctl-plugin.js +827 -91
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
unwatchFile,
|
|
52
52
|
} from 'node:fs';
|
|
53
53
|
import { createConnection, createServer } from 'node:net';
|
|
54
|
+
import { lookup as dnsLookup } from 'node:dns/promises';
|
|
54
55
|
import { randomUUID, createHash, randomBytes } from 'node:crypto';
|
|
55
56
|
import { homedir, platform as osPlatform, devNull, tmpdir, hostname } from 'node:os';
|
|
56
57
|
import { join, isAbsolute, resolve as resolvePath, dirname, basename, sep } from 'node:path';
|
|
@@ -2208,6 +2209,254 @@ function killTree(child) {
|
|
|
2208
2209
|
try { child.kill('SIGKILL'); } catch { /* already gone */ }
|
|
2209
2210
|
}
|
|
2210
2211
|
|
|
2212
|
+
// ---- Progress-aware idle liveness (#130) ---------------------------------
|
|
2213
|
+
// The idle-liveness kill used to treat terminal silence as "wedged": a harness
|
|
2214
|
+
// that emitted no stdout/stderr for `idleTimeoutMs` was killed, which stops the
|
|
2215
|
+
// broker lock extension and lets the job be reclaimed. But an agent running a
|
|
2216
|
+
// long, QUIET subprocess (e.g. a cold `./mvnw -q … | tail` reactor build) can
|
|
2217
|
+
// produce zero output for 10+ minutes while actively progressing — a silence
|
|
2218
|
+
// that is NOT a hang. The fix: before the idle-kill fires, probe the spawned
|
|
2219
|
+
// process subtree; if a descendant is doing work (aggregate CPU time across the
|
|
2220
|
+
// tree is advancing) treat the agent as live and defer the kill instead of
|
|
2221
|
+
// reclaiming a job that is genuinely making forward progress. Only a quiescent
|
|
2222
|
+
// tree — no live descendant, or descendants burning no CPU across the window —
|
|
2223
|
+
// ages out and is killed, so a truly wedged agent is still reclaimed.
|
|
2224
|
+
|
|
2225
|
+
/**
|
|
2226
|
+
* Parse a `ps`-style CPU time field (`[[dd-]hh:]mm:ss[.frac]`) into whole
|
|
2227
|
+
* seconds. Best-effort: an unrecognised token yields 0 so a single odd line
|
|
2228
|
+
* never poisons the aggregate.
|
|
2229
|
+
*/
|
|
2230
|
+
function parsePsTime(token) {
|
|
2231
|
+
if (typeof token !== 'string') return 0;
|
|
2232
|
+
let s = token.trim();
|
|
2233
|
+
let days = 0;
|
|
2234
|
+
const dash = s.indexOf('-');
|
|
2235
|
+
if (dash !== -1) { days = Number(s.slice(0, dash)) || 0; s = s.slice(dash + 1); }
|
|
2236
|
+
const parts = s.split(':').map((p) => Number.parseFloat(p));
|
|
2237
|
+
if (parts.length === 0 || parts.some((n) => !Number.isFinite(n))) return 0;
|
|
2238
|
+
let secs = 0;
|
|
2239
|
+
for (const p of parts) secs = secs * 60 + p;
|
|
2240
|
+
return Math.floor(days * 86_400 + secs);
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
/**
|
|
2244
|
+
* Sample the process subtree rooted at `rootPid` from Linux `/proc`.
|
|
2245
|
+
* Returns `{ descendants, cpu }` — the count of live descendant processes
|
|
2246
|
+
* (excluding the root) and the aggregate CPU time (utime+stime, in clock ticks)
|
|
2247
|
+
* summed across the WHOLE subtree including the root — or `null` when the tree
|
|
2248
|
+
* cannot be read (e.g. `/proc` unavailable). The descendant count is the
|
|
2249
|
+
* presence gate (a childless silent agent still ages out on silence); the CPU
|
|
2250
|
+
* aggregate is the progress signal (a present-but-hung `java` burning no CPU
|
|
2251
|
+
* still ages out).
|
|
2252
|
+
*/
|
|
2253
|
+
function sampleSubtreeCpuProc(rootPid) {
|
|
2254
|
+
let entries;
|
|
2255
|
+
try { entries = readdirSync('/proc'); } catch { return null; }
|
|
2256
|
+
const children = new Map(); // ppid -> [pid]
|
|
2257
|
+
const cpu = new Map(); // pid -> aggregate ticks (utime+stime)
|
|
2258
|
+
for (const name of entries) {
|
|
2259
|
+
if (name.charCodeAt(0) < 48 || name.charCodeAt(0) > 57) continue; // fast /^\d/ gate
|
|
2260
|
+
if (!/^\d+$/.test(name)) continue;
|
|
2261
|
+
const pid = Number(name);
|
|
2262
|
+
let stat;
|
|
2263
|
+
try { stat = readFileSync(`/proc/${name}/stat`, 'utf8'); } catch { continue; }
|
|
2264
|
+
// `comm` (field #2) is parenthesised and may itself contain spaces or
|
|
2265
|
+
// parentheses, so split on the LAST ')' — everything after it is the
|
|
2266
|
+
// space-separated tail starting at field #3 (state).
|
|
2267
|
+
const rparen = stat.lastIndexOf(')');
|
|
2268
|
+
if (rparen < 0) continue;
|
|
2269
|
+
// Split the post-`comm` tail on RUNS of whitespace (not a single space): a
|
|
2270
|
+
// stray double space would otherwise yield empty tokens and shift the
|
|
2271
|
+
// ppid/utime/stime field indexes, mis-sampling the subtree.
|
|
2272
|
+
const tail = stat.slice(rparen + 1).trim().split(/\s+/);
|
|
2273
|
+
// tail[0]=state(#3), tail[1]=ppid(#4) … utime(#14)=tail[11], stime(#15)=tail[12].
|
|
2274
|
+
const ppid = Number(tail[1]);
|
|
2275
|
+
if (!Number.isFinite(ppid)) continue;
|
|
2276
|
+
const utime = Number(tail[11]);
|
|
2277
|
+
const stime = Number(tail[12]);
|
|
2278
|
+
if (!children.has(ppid)) children.set(ppid, []);
|
|
2279
|
+
children.get(ppid).push(pid);
|
|
2280
|
+
cpu.set(pid, (Number.isFinite(utime) ? utime : 0) + (Number.isFinite(stime) ? stime : 0));
|
|
2281
|
+
}
|
|
2282
|
+
if (!cpu.has(rootPid) && !children.has(rootPid)) return null; // root already gone
|
|
2283
|
+
let descendants = 0;
|
|
2284
|
+
let totalCpu = cpu.get(rootPid) || 0;
|
|
2285
|
+
const seen = new Set([rootPid]);
|
|
2286
|
+
const stack = [...(children.get(rootPid) || [])];
|
|
2287
|
+
while (stack.length) {
|
|
2288
|
+
const pid = stack.pop();
|
|
2289
|
+
if (seen.has(pid)) continue;
|
|
2290
|
+
seen.add(pid);
|
|
2291
|
+
descendants += 1;
|
|
2292
|
+
totalCpu += cpu.get(pid) || 0;
|
|
2293
|
+
const kids = children.get(pid);
|
|
2294
|
+
if (kids) for (const k of kids) if (!seen.has(k)) stack.push(k);
|
|
2295
|
+
}
|
|
2296
|
+
return { descendants, cpu: totalCpu };
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
/**
|
|
2300
|
+
* Non-Linux fallback: sample the subtree via a one-shot `ps`. Heavier than the
|
|
2301
|
+
* `/proc` read (it spawns a process) but only runs during an idle window, which
|
|
2302
|
+
* is rare by construction. Returns the same `{ descendants, cpu }` shape (cpu in
|
|
2303
|
+
* whole seconds here) plus `resolutionMs: 1000` to flag the coarse whole-second
|
|
2304
|
+
* CPU granularity — the monitor uses that so a probe window shorter than a
|
|
2305
|
+
* whole-second tick can't read flat CPU and false-kill a still-progressing but
|
|
2306
|
+
* low-churn subtree. Returns `null` when `ps` is unavailable/unparsable.
|
|
2307
|
+
*/
|
|
2308
|
+
function sampleSubtreeCpuPs(rootPid) {
|
|
2309
|
+
let out;
|
|
2310
|
+
try {
|
|
2311
|
+
const r = spawnSync('ps', ['-A', '-o', 'pid=,ppid=,time='], { encoding: 'utf8', timeout: 5_000 });
|
|
2312
|
+
if (r.status !== 0 || !r.stdout) return null;
|
|
2313
|
+
out = r.stdout;
|
|
2314
|
+
} catch { return null; }
|
|
2315
|
+
const children = new Map();
|
|
2316
|
+
const cpu = new Map();
|
|
2317
|
+
for (const line of out.split('\n')) {
|
|
2318
|
+
const m = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)$/);
|
|
2319
|
+
if (!m) continue;
|
|
2320
|
+
const pid = Number(m[1]);
|
|
2321
|
+
const ppid = Number(m[2]);
|
|
2322
|
+
if (!children.has(ppid)) children.set(ppid, []);
|
|
2323
|
+
children.get(ppid).push(pid);
|
|
2324
|
+
cpu.set(pid, parsePsTime(m[3]));
|
|
2325
|
+
}
|
|
2326
|
+
if (!cpu.has(rootPid) && !children.has(rootPid)) return null;
|
|
2327
|
+
let descendants = 0;
|
|
2328
|
+
let totalCpu = cpu.get(rootPid) || 0;
|
|
2329
|
+
const seen = new Set([rootPid]);
|
|
2330
|
+
const stack = [...(children.get(rootPid) || [])];
|
|
2331
|
+
while (stack.length) {
|
|
2332
|
+
const pid = stack.pop();
|
|
2333
|
+
if (seen.has(pid)) continue;
|
|
2334
|
+
seen.add(pid);
|
|
2335
|
+
descendants += 1;
|
|
2336
|
+
totalCpu += cpu.get(pid) || 0;
|
|
2337
|
+
const kids = children.get(pid);
|
|
2338
|
+
if (kids) for (const k of kids) if (!seen.has(k)) stack.push(k);
|
|
2339
|
+
}
|
|
2340
|
+
return { descendants, cpu: totalCpu, resolutionMs: 1000 };
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/**
|
|
2344
|
+
* Sample the aggregate CPU + descendant count of the process subtree rooted at
|
|
2345
|
+
* `rootPid`. Linux reads `/proc`; other POSIX platforms fall back to `ps`.
|
|
2346
|
+
* Returns `null` when sampling is unsupported/failed, so callers degrade to the
|
|
2347
|
+
* legacy silence-only liveness rather than mis-killing.
|
|
2348
|
+
*/
|
|
2349
|
+
function sampleSubtreeCpu(rootPid) {
|
|
2350
|
+
if (typeof rootPid !== 'number' || !(rootPid > 0)) return null;
|
|
2351
|
+
if (process.platform === 'win32') return null;
|
|
2352
|
+
if (process.platform === 'linux') return sampleSubtreeCpuProc(rootPid);
|
|
2353
|
+
return sampleSubtreeCpuPs(rootPid);
|
|
2354
|
+
}
|
|
2355
|
+
|
|
2356
|
+
/**
|
|
2357
|
+
* Build a progress-aware idle-liveness monitor shared by the pipe/PTY/ACP
|
|
2358
|
+
* capture paths. `arm()` is called on every output chunk (and once at start) to
|
|
2359
|
+
* (re)start the silence window; `stop()` is called when the run settles.
|
|
2360
|
+
*
|
|
2361
|
+
* When the silence window elapses the monitor probes the subtree via `sampleCpu`
|
|
2362
|
+
* instead of killing outright:
|
|
2363
|
+
* - no usable sample OR no live descendant → the tree is childless/quiescent →
|
|
2364
|
+
* `onIdleKill()` (preserves the legacy "silence == wedged" behaviour, so a
|
|
2365
|
+
* genuinely wedged agent with no children still ages out).
|
|
2366
|
+
* - a live descendant whose aggregate CPU advanced since the last probe → the
|
|
2367
|
+
* tree is doing work → defer another probe window (`recoveryWindowMs`, or the
|
|
2368
|
+
* idle window when unset) instead of killing.
|
|
2369
|
+
* - a live descendant whose aggregate CPU regressed → a busy descendant exited
|
|
2370
|
+
* (its ticks no longer count) → that is progress, not a stall → re-baseline
|
|
2371
|
+
* and defer instead of killing.
|
|
2372
|
+
* - a live descendant whose CPU was exactly unchanged across a probe window →
|
|
2373
|
+
* a candidate stall, but only a *confirmed* stall once the cumulative flat
|
|
2374
|
+
* time reaches the sampler's CPU-time resolution (`sample.resolutionMs`, e.g.
|
|
2375
|
+
* the whole-second `ps` fallback reports 1000ms). A probe window shorter than
|
|
2376
|
+
* that resolution can read flat CPU while a low-churn subtree is still
|
|
2377
|
+
* progressing but hasn't crossed the next whole-second tick, so a single flat
|
|
2378
|
+
* read under a coarse sampler must NOT kill; the fine-grained `/proc` path
|
|
2379
|
+
* carries no resolution and so ages out on the first flat window as before →
|
|
2380
|
+
* `onIdleKill()`.
|
|
2381
|
+
* The first elapse with live descendants takes a baseline (a single sample can't
|
|
2382
|
+
* show movement) and re-probes after the recovery window. The absolute
|
|
2383
|
+
* `--job-timeout` hard cap remains the ultimate backstop, untouched by this.
|
|
2384
|
+
*/
|
|
2385
|
+
function createIdleLivenessMonitor({ getPid, idleTimeoutMs, recoveryWindowMs, onIdleKill, isSettled, sampleCpu = sampleSubtreeCpu }) {
|
|
2386
|
+
const enabled = idleTimeoutMs && idleTimeoutMs > 0;
|
|
2387
|
+
const probeWindow = recoveryWindowMs && recoveryWindowMs > 0 ? recoveryWindowMs : idleTimeoutMs;
|
|
2388
|
+
let timer = null;
|
|
2389
|
+
let prev = null; // last subtree CPU probe taken during the current silent stretch
|
|
2390
|
+
let flatMs = 0; // cumulative time observed with flat CPU since the last movement/baseline
|
|
2391
|
+
let lastWindow = idleTimeoutMs; // duration of the currently pending probe window
|
|
2392
|
+
const settled = () => (typeof isSettled === 'function' ? isSettled() : false);
|
|
2393
|
+
const clear = () => { if (timer) { clearTimeout(timer); timer = null; } };
|
|
2394
|
+
const schedule = (ms) => { lastWindow = ms; timer = setTimeout(onElapse, ms); };
|
|
2395
|
+
function onElapse() {
|
|
2396
|
+
timer = null;
|
|
2397
|
+
if (settled()) return;
|
|
2398
|
+
const elapsed = lastWindow;
|
|
2399
|
+
const pid = typeof getPid === 'function' ? getPid() : undefined;
|
|
2400
|
+
const sample = pid != null ? sampleCpu(pid) : null;
|
|
2401
|
+
// Childless / unsampleable / dead tree → silence really is a hang: kill.
|
|
2402
|
+
if (!sample || sample.descendants <= 0) { onIdleKill(); return; }
|
|
2403
|
+
if (prev != null) {
|
|
2404
|
+
// Aggregate CPU advanced across the window → the tree is doing work; defer.
|
|
2405
|
+
if (sample.cpu > prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
|
|
2406
|
+
// Aggregate CPU REGRESSED → `sampleSubtreeCpu*` only sums CPU for live
|
|
2407
|
+
// PIDs, so the total can legitimately drop when a busy descendant exits
|
|
2408
|
+
// (its ticks stop counting) even though the subtree just made progress.
|
|
2409
|
+
// A regression is activity, not a stall: re-baseline and defer rather
|
|
2410
|
+
// than mis-killing a silent-but-busy job.
|
|
2411
|
+
if (sample.cpu < prev.cpu) { prev = sample; flatMs = 0; schedule(probeWindow); return; }
|
|
2412
|
+
// Live descendants but CPU exactly unchanged across a full window → a
|
|
2413
|
+
// candidate stall. Only confirm it once we have observed flat CPU for at
|
|
2414
|
+
// least the sampler's CPU-time resolution: a probe window shorter than the
|
|
2415
|
+
// granularity (the whole-second `ps` fallback) can read flat while a
|
|
2416
|
+
// low-churn subtree is still progressing but hasn't crossed the next tick.
|
|
2417
|
+
flatMs += elapsed;
|
|
2418
|
+
const resolutionMs = Number.isFinite(sample.resolutionMs) ? sample.resolutionMs : 0;
|
|
2419
|
+
if (flatMs >= resolutionMs) { onIdleKill(); return; }
|
|
2420
|
+
schedule(probeWindow);
|
|
2421
|
+
return;
|
|
2422
|
+
}
|
|
2423
|
+
// First elapse with live descendants: baseline, then re-probe (can't judge
|
|
2424
|
+
// progress from one sample).
|
|
2425
|
+
prev = sample;
|
|
2426
|
+
flatMs = 0;
|
|
2427
|
+
schedule(probeWindow);
|
|
2428
|
+
}
|
|
2429
|
+
return {
|
|
2430
|
+
enabled: !!enabled,
|
|
2431
|
+
arm() {
|
|
2432
|
+
if (!enabled || settled()) return;
|
|
2433
|
+
prev = null; // output resumed → a fresh silent stretch starts
|
|
2434
|
+
flatMs = 0;
|
|
2435
|
+
clear();
|
|
2436
|
+
schedule(idleTimeoutMs);
|
|
2437
|
+
},
|
|
2438
|
+
stop() { clear(); },
|
|
2439
|
+
};
|
|
2440
|
+
}
|
|
2441
|
+
|
|
2442
|
+
/**
|
|
2443
|
+
* Resolve the effective per-job liveness values (#130) from the task envelope
|
|
2444
|
+
* with precedence: envelope override → worker-flag default → built-in default.
|
|
2445
|
+
* Each override is clamped to a sane max so a task envelope can't request an
|
|
2446
|
+
* unbounded window. Absent/invalid envelope fields fall through to the
|
|
2447
|
+
* worker-flag value unchanged. `hardCapMs` (0 = no absolute cap) is preserved as
|
|
2448
|
+
* the ultimate backstop; an envelope `timeoutMs` can raise it (clamped).
|
|
2449
|
+
*/
|
|
2450
|
+
function resolveLivenessOverrides(envelopeTask, { idleTimeoutMs, recoveryWindowMs, hardCapMs } = {}) {
|
|
2451
|
+
const task = isPlainObject(envelopeTask) ? envelopeTask : {};
|
|
2452
|
+
const clampPos = (v, max) => (Number.isFinite(v) && v > 0 ? Math.min(v, max) : undefined);
|
|
2453
|
+
return {
|
|
2454
|
+
idleTimeoutMs: clampPos(task.idleTimeoutMs, MAX_TASK_IDLE_TIMEOUT_MS) ?? idleTimeoutMs,
|
|
2455
|
+
recoveryWindowMs: clampPos(task.recoveryWindowMs, MAX_TASK_RECOVERY_WINDOW_MS) ?? recoveryWindowMs,
|
|
2456
|
+
hardCapMs: clampPos(task.timeoutMs, MAX_TASK_HARD_CAP_MS) ?? hardCapMs,
|
|
2457
|
+
};
|
|
2458
|
+
}
|
|
2459
|
+
|
|
2211
2460
|
// ===========================================================================
|
|
2212
2461
|
// Agent task envelope + sandboxed execution (issue #8, increment 1)
|
|
2213
2462
|
// ===========================================================================
|
|
@@ -2228,6 +2477,14 @@ const AGENT_RESULT_KEY = 'io.nanobpm.agentResult';
|
|
|
2228
2477
|
const LINKED_RESOURCES_HEADER = 'linkedResources';
|
|
2229
2478
|
const DEFAULT_PROMPT_LINK_NAME = 'prompt';
|
|
2230
2479
|
const TASK_ENVELOPE_SCHEMA_VERSION = 1;
|
|
2480
|
+
// Upper bounds for per-task liveness overrides (#130). A task envelope may widen
|
|
2481
|
+
// the idle/recovery window (and the absolute hard cap) for a long-idle task
|
|
2482
|
+
// class, but never unboundedly — a runaway or malicious envelope can't pin a
|
|
2483
|
+
// worker forever. Generous but finite: 1h of silence-with-progress, 1h recovery
|
|
2484
|
+
// cadence, 24h absolute runtime cap.
|
|
2485
|
+
const MAX_TASK_IDLE_TIMEOUT_MS = 60 * 60_000;
|
|
2486
|
+
const MAX_TASK_RECOVERY_WINDOW_MS = 60 * 60_000;
|
|
2487
|
+
const MAX_TASK_HARD_CAP_MS = 24 * 60 * 60_000;
|
|
2231
2488
|
// The result-envelope version is intentionally independent of the task-envelope
|
|
2232
2489
|
// version so the two contracts can evolve separately without silently coupling.
|
|
2233
2490
|
const RESULT_ENVELOPE_SCHEMA_VERSION = 1;
|
|
@@ -2472,6 +2729,16 @@ function normalizeTaskEnvelope(customHeaders, variables, opts = {}) {
|
|
|
2472
2729
|
promptFile: str(task.promptFile),
|
|
2473
2730
|
maxIterations: coerceInt(task.maxIterations, undefined),
|
|
2474
2731
|
timeoutMs: coerceInt(task.timeoutMs, undefined),
|
|
2732
|
+
// #130: per-task liveness overrides. `idleTimeoutMs` is the max silence
|
|
2733
|
+
// (with no subtree CPU progress) before the harness is treated as wedged;
|
|
2734
|
+
// `recoveryWindowMs` is the cadence at which a silent-but-working tree is
|
|
2735
|
+
// re-probed and also the broker lock recovery window. Both let the
|
|
2736
|
+
// orchestrator widen the window for a specific long-idle task class (e.g. a
|
|
2737
|
+
// JVM-heavy monorepo build) without a global `--idle-timeout` widen. Absent
|
|
2738
|
+
// → the worker-flag defaults apply, unchanged. Clamped to a sane max at the
|
|
2739
|
+
// point of use so a task can't request an unbounded window.
|
|
2740
|
+
idleTimeoutMs: coerceInt(task.idleTimeoutMs, undefined),
|
|
2741
|
+
recoveryWindowMs: coerceInt(task.recoveryWindowMs, undefined),
|
|
2475
2742
|
allowPr: coerceBool(task.allowPr, false),
|
|
2476
2743
|
prBase: str(task.prBase),
|
|
2477
2744
|
};
|
|
@@ -2479,6 +2746,71 @@ function normalizeTaskEnvelope(customHeaders, variables, opts = {}) {
|
|
|
2479
2746
|
return env;
|
|
2480
2747
|
}
|
|
2481
2748
|
|
|
2749
|
+
// Does a string look like a usable git clone URL? Used to fail-closed on a
|
|
2750
|
+
// half-specified repository envelope (issue #129) rather than silently degrading
|
|
2751
|
+
// to a launch/temp cwd. Deliberately permissive — it only rejects values git
|
|
2752
|
+
// clone could never resolve to a remote:
|
|
2753
|
+
// - `scheme://host/…` (https/http/ssh/git), host required;
|
|
2754
|
+
// - `file://…` with a path;
|
|
2755
|
+
// - scp-like `user@host:path` (e.g. `git@github.com:o/r.git`);
|
|
2756
|
+
// - a POSIX local filesystem path (`/…`, `./…`, `../…`);
|
|
2757
|
+
// - a Windows local path — drive-absolute (`C:\repo`, `C:/repo`), UNC
|
|
2758
|
+
// (`\\server\share\repo`), or backslash-relative (`.\repo`, `..\repo`) —
|
|
2759
|
+
// since the plugin also runs on win32 hosts.
|
|
2760
|
+
// Anything else (a bare word, `github.com/o/r` with no scheme) is treated as
|
|
2761
|
+
// malformed.
|
|
2762
|
+
function isPlausibleRepoUrl(url) {
|
|
2763
|
+
const s = String(url == null ? '' : url).trim();
|
|
2764
|
+
if (!s) return false;
|
|
2765
|
+
// scp-like syntax has no scheme: user@host:path (the colon is not a port).
|
|
2766
|
+
if (/^[^\s/@]+@[^\s/:]+:.+/.test(s)) return true;
|
|
2767
|
+
// Windows local paths — checked BEFORE new URL() because a drive letter like
|
|
2768
|
+
// `C:` parses as a (hostname-less) URL scheme and would otherwise be rejected.
|
|
2769
|
+
if (/^[A-Za-z]:[\\/]/.test(s) || /^\\\\[^\\]/.test(s) || s.startsWith('.\\') || s.startsWith('..\\')) return true;
|
|
2770
|
+
try {
|
|
2771
|
+
const u = new URL(s);
|
|
2772
|
+
if (u.protocol === 'file:') return !!u.pathname && u.pathname !== '/';
|
|
2773
|
+
return !!u.hostname;
|
|
2774
|
+
} catch { /* not a URL — fall through to the local-path check */ }
|
|
2775
|
+
return s.startsWith('/') || s.startsWith('./') || s.startsWith('../');
|
|
2776
|
+
}
|
|
2777
|
+
|
|
2778
|
+
// Classify the RAW `repository` block of a task envelope BEFORE normalization
|
|
2779
|
+
// drops it (issue #129). `normalizeTaskEnvelope` only emits `env.repository`
|
|
2780
|
+
// when a truthy `url` is present, so a half-specified repo block (other
|
|
2781
|
+
// `repository.*` fields, but `url` absent — or a `url` that isn't a usable clone
|
|
2782
|
+
// target) is invisible downstream and silently degrades to a repo-less run in
|
|
2783
|
+
// the launch/temp cwd. This surfaces that intent so the job handler can
|
|
2784
|
+
// fail-closed on it.
|
|
2785
|
+
//
|
|
2786
|
+
// Returns { present, url, malformed }:
|
|
2787
|
+
// - present — repository intent was supplied: a `repository` block that
|
|
2788
|
+
// either carries at least one non-empty field OR explicitly
|
|
2789
|
+
// includes a `url` key at all (even blank/whitespace — spelling
|
|
2790
|
+
// out `repository.url` IS intent to clone, so it must fail-closed
|
|
2791
|
+
// rather than silently degrade). A task with NO repository block,
|
|
2792
|
+
// or one with only blank non-url fields, is legitimately
|
|
2793
|
+
// repo-less and is `present: false`.
|
|
2794
|
+
// - url — the trimmed url string when present & non-empty, else null.
|
|
2795
|
+
// - malformed — repository intent is expected but unusable: the block is
|
|
2796
|
+
// present with fields but the `url` is absent/blank, or the `url`
|
|
2797
|
+
// is a non-empty string that isn't a plausible clone target.
|
|
2798
|
+
function classifyRepoEnvelope(customHeaders, variables) {
|
|
2799
|
+
const raw = deepMerge(collectEnvelopeFrom(customHeaders), collectEnvelopeFrom(variables));
|
|
2800
|
+
const repo = raw.repository;
|
|
2801
|
+
if (!isPlainObject(repo)) return { present: false, url: null, malformed: false };
|
|
2802
|
+
// A `url` key present at all — even blank/whitespace/null — is repository
|
|
2803
|
+
// intent, so an envelope carrying only a blank `repository.url` must
|
|
2804
|
+
// fail-closed instead of silently running repo-less in the temp cwd.
|
|
2805
|
+
const hasUrlKey = Object.prototype.hasOwnProperty.call(repo, 'url');
|
|
2806
|
+
const hasAnyField = Object.keys(repo).some((k) => repo[k] != null && String(repo[k]).trim() !== '');
|
|
2807
|
+
if (!hasUrlKey && !hasAnyField) return { present: false, url: null, malformed: false };
|
|
2808
|
+
const url = repo.url == null ? '' : String(repo.url).trim();
|
|
2809
|
+
if (!url) return { present: true, url: null, malformed: true };
|
|
2810
|
+
if (!isPlausibleRepoUrl(url)) return { present: true, url, malformed: true };
|
|
2811
|
+
return { present: true, url, malformed: false };
|
|
2812
|
+
}
|
|
2813
|
+
|
|
2482
2814
|
// ---- Linked resources → live agent prompt (issue #63) ----------------------
|
|
2483
2815
|
// A job's `linkedResources` activation header carries KEYS (not content); the
|
|
2484
2816
|
// worker resolves each to the LATEST deployed bytes over the broker REST API, so
|
|
@@ -3789,7 +4121,7 @@ const MAX_CAPTURE_BYTES = 1_048_576; // 1 MiB per stream
|
|
|
3789
4121
|
// Spawn a child, pipe `stdinData`, capture byte-capped stdout/stderr, enforce a
|
|
3790
4122
|
// timeout (invoking `onTimeout(child)` to tear the child down), and resolve to a
|
|
3791
4123
|
// uniform result. Used by both the host and container executors.
|
|
3792
|
-
function spawnCaptureOneShot({ command, args = [], shell = false, detached = false, cwd, env, stdinData, timeoutMs, idleTimeoutMs, onTimeout, stream = false, streamPrefix = '', onStreamOut, onStreamErr, relayTap = null }) {
|
|
4124
|
+
function spawnCaptureOneShot({ command, args = [], shell = false, detached = false, cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, onTimeout, stream = false, streamPrefix = '', onStreamOut, onStreamErr, relayTap = null }) {
|
|
3793
4125
|
return new Promise((resolve) => {
|
|
3794
4126
|
let child;
|
|
3795
4127
|
const stdoutChunks = [];
|
|
@@ -3800,7 +4132,7 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
3800
4132
|
let stderrTruncated = false;
|
|
3801
4133
|
let settled = false;
|
|
3802
4134
|
let timer = null;
|
|
3803
|
-
let
|
|
4135
|
+
let idleMon = null;
|
|
3804
4136
|
|
|
3805
4137
|
// Live "spy" tee (--stream): mirror the child's output line-by-line to a
|
|
3806
4138
|
// caller-supplied emitter (the worker routes these through c8ctl's
|
|
@@ -3837,7 +4169,7 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
3837
4169
|
if (settled) return;
|
|
3838
4170
|
settled = true;
|
|
3839
4171
|
if (timer) clearTimeout(timer);
|
|
3840
|
-
if (
|
|
4172
|
+
if (idleMon) idleMon.stop();
|
|
3841
4173
|
if (teeOut) teeOut('', true);
|
|
3842
4174
|
if (teeErr) teeErr('', true);
|
|
3843
4175
|
resolve(result);
|
|
@@ -3857,21 +4189,27 @@ function spawnCaptureOneShot({ command, args = [], shell = false, detached = fal
|
|
|
3857
4189
|
}, timeoutMs)
|
|
3858
4190
|
: null;
|
|
3859
4191
|
|
|
3860
|
-
// Idle-liveness kill: if the child emits no stdout/stderr for `idleTimeoutMs
|
|
3861
|
-
//
|
|
3862
|
-
//
|
|
3863
|
-
//
|
|
3864
|
-
//
|
|
3865
|
-
//
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
4192
|
+
// Idle-liveness kill: if the child emits no stdout/stderr for `idleTimeoutMs`
|
|
4193
|
+
// AND its process subtree is quiescent (no live descendant, or descendants
|
|
4194
|
+
// burning no CPU across the window), treat it as wedged and kill the tree.
|
|
4195
|
+
// This is the liveness signal the worker's lock-extender relies on — a silent
|
|
4196
|
+
// hang stops producing output, we kill it here, `runAgentJob` resolves, and
|
|
4197
|
+
// the worker fails the job (retryable) so the broker reclaims it. Distinct
|
|
4198
|
+
// from the absolute `timeoutMs` hard cap: this fires on *silence without
|
|
4199
|
+
// progress*, not total runtime. A long, QUIET subprocess that keeps a
|
|
4200
|
+
// descendant doing work (advancing subtree CPU) is deferred, not killed
|
|
4201
|
+
// (#130). Re-armed on every chunk.
|
|
4202
|
+
idleMon = createIdleLivenessMonitor({
|
|
4203
|
+
getPid: () => child?.pid,
|
|
4204
|
+
idleTimeoutMs,
|
|
4205
|
+
recoveryWindowMs,
|
|
4206
|
+
isSettled: () => settled,
|
|
4207
|
+
onIdleKill: () => {
|
|
3871
4208
|
try { if (onTimeout) onTimeout(child); } catch { /* best effort */ }
|
|
3872
|
-
finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output for ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
|
|
3873
|
-
},
|
|
3874
|
-
};
|
|
4209
|
+
finish({ ok: false, exitCode: null, stdout: joinCapped(stdoutChunks), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: stdoutTruncated, stderrTruncated });
|
|
4210
|
+
},
|
|
4211
|
+
});
|
|
4212
|
+
const armIdle = () => idleMon.arm();
|
|
3875
4213
|
armIdle();
|
|
3876
4214
|
|
|
3877
4215
|
child.stdout.on('data', (d) => {
|
|
@@ -3953,7 +4291,7 @@ function ptyAvailable(ptyFactory) {
|
|
|
3953
4291
|
// spawnCaptureOneShot. A PTY merges stdout+stderr into one stream, so stderr is
|
|
3954
4292
|
// always '' here; that is expected for a live terminal. `ptyFactory` is
|
|
3955
4293
|
// injectable for tests (defaults to node-pty).
|
|
3956
|
-
function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, cols = 120, rows = 30, ptyFactory, relayTap = null, stream = false, streamPrefix = '', onStreamOut }) {
|
|
4294
|
+
function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, cols = 120, rows = 30, ptyFactory, relayTap = null, stream = false, streamPrefix = '', onStreamOut }) {
|
|
3957
4295
|
return new Promise((resolve) => {
|
|
3958
4296
|
const factory = ptyFactory || loadPtyModule();
|
|
3959
4297
|
if (!factory || typeof factory.spawn !== 'function') {
|
|
@@ -3966,7 +4304,7 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
3966
4304
|
let truncated = false;
|
|
3967
4305
|
let settled = false;
|
|
3968
4306
|
let timer = null;
|
|
3969
|
-
let
|
|
4307
|
+
let idleMon = null;
|
|
3970
4308
|
let detachSteer = null;
|
|
3971
4309
|
let term;
|
|
3972
4310
|
|
|
@@ -3997,7 +4335,7 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
3997
4335
|
if (settled) return;
|
|
3998
4336
|
settled = true;
|
|
3999
4337
|
if (timer) clearTimeout(timer);
|
|
4000
|
-
if (
|
|
4338
|
+
if (idleMon) idleMon.stop();
|
|
4001
4339
|
if (detachSteer) { try { detachSteer(); } catch { /* best effort */ } detachSteer = null; }
|
|
4002
4340
|
if (teeSink) tee('', true);
|
|
4003
4341
|
resolve(result);
|
|
@@ -4017,15 +4355,17 @@ function spawnCapturePty({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
4017
4355
|
}, timeoutMs)
|
|
4018
4356
|
: null;
|
|
4019
4357
|
|
|
4020
|
-
const armIdle = () =>
|
|
4021
|
-
|
|
4022
|
-
|
|
4023
|
-
|
|
4024
|
-
|
|
4358
|
+
const armIdle = () => idleMon.arm();
|
|
4359
|
+
idleMon = createIdleLivenessMonitor({
|
|
4360
|
+
getPid: () => term?.pid,
|
|
4361
|
+
idleTimeoutMs,
|
|
4362
|
+
recoveryWindowMs,
|
|
4363
|
+
isSettled: () => settled,
|
|
4364
|
+
onIdleKill: () => {
|
|
4025
4365
|
killTerm();
|
|
4026
|
-
finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output for ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
|
|
4027
|
-
},
|
|
4028
|
-
};
|
|
4366
|
+
finish({ ok: false, exitCode: null, stdout: joinCapped(chunks), stderr: '', error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated, stderrTruncated: false });
|
|
4367
|
+
},
|
|
4368
|
+
});
|
|
4029
4369
|
armIdle();
|
|
4030
4370
|
|
|
4031
4371
|
term.onData((d) => {
|
|
@@ -4204,7 +4544,7 @@ const ACP_MAX_LINE_BYTES = 8 * 1024 * 1024; // 8 MiB
|
|
|
4204
4544
|
// and every caller work unchanged. Because the raw stream is JSON-RPC (not human
|
|
4205
4545
|
// output), `stdout` here is the accumulated human-readable transcript text (what
|
|
4206
4546
|
// we relay), and `stderr` is the child's real stderr (agent diagnostics).
|
|
4207
|
-
function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, relayTap = null, stream = false, streamPrefix = '', onStreamOut, onStreamErr, permission = 'yolo', shell = false }) {
|
|
4547
|
+
function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, relayTap = null, stream = false, streamPrefix = '', onStreamOut, onStreamErr, permission = 'yolo', shell = false }) {
|
|
4208
4548
|
return new Promise((resolve) => {
|
|
4209
4549
|
const logger = getLogger();
|
|
4210
4550
|
const humanChunks = [];
|
|
@@ -4215,7 +4555,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
4215
4555
|
let stderrTruncated = false;
|
|
4216
4556
|
let settled = false;
|
|
4217
4557
|
let timer = null;
|
|
4218
|
-
let
|
|
4558
|
+
let idleMon = null;
|
|
4219
4559
|
let detachSteer = null;
|
|
4220
4560
|
let child;
|
|
4221
4561
|
let sessionId = null;
|
|
@@ -4294,7 +4634,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
4294
4634
|
if (settled) return;
|
|
4295
4635
|
settled = true;
|
|
4296
4636
|
if (timer) clearTimeout(timer);
|
|
4297
|
-
if (
|
|
4637
|
+
if (idleMon) idleMon.stop();
|
|
4298
4638
|
if (settleTimer) { clearTimeout(settleTimer); settleTimer = null; }
|
|
4299
4639
|
if (detachSteer) { try { detachSteer(); } catch { /* best effort */ } detachSteer = null; }
|
|
4300
4640
|
if (teeSink) { tee('', true); teeErr('', true); }
|
|
@@ -4485,15 +4825,17 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
4485
4825
|
}, timeoutMs)
|
|
4486
4826
|
: null;
|
|
4487
4827
|
|
|
4488
|
-
const armIdle = () =>
|
|
4489
|
-
|
|
4490
|
-
|
|
4491
|
-
|
|
4492
|
-
|
|
4828
|
+
const armIdle = () => idleMon.arm();
|
|
4829
|
+
idleMon = createIdleLivenessMonitor({
|
|
4830
|
+
getPid: () => child?.pid,
|
|
4831
|
+
idleTimeoutMs,
|
|
4832
|
+
recoveryWindowMs,
|
|
4833
|
+
isSettled: () => settled,
|
|
4834
|
+
onIdleKill: () => {
|
|
4493
4835
|
try { killTree(child); } catch { /* best effort */ }
|
|
4494
|
-
finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output for ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
|
|
4495
|
-
},
|
|
4496
|
-
};
|
|
4836
|
+
finish({ ok: false, exitCode: null, stdout: humanStdout(), stderr: joinCapped(stderrChunks), error: `no output and no subtree progress for >= ${idleTimeoutMs}ms (idle)`, timedOut: true, idle: true, truncated: humanTruncated, stderrTruncated });
|
|
4837
|
+
},
|
|
4838
|
+
});
|
|
4497
4839
|
armIdle();
|
|
4498
4840
|
|
|
4499
4841
|
// Newline-delimited JSON-RPC parser over stdout. Progress on stdout re-arms
|
|
@@ -4781,7 +5123,7 @@ function startLockExtender(job, windowMs, intervalMs, tag, logger) {
|
|
|
4781
5123
|
* Both paths resolve to the same result contract.
|
|
4782
5124
|
*/
|
|
4783
5125
|
function runAgentJob(profile, job, opts = {}) {
|
|
4784
|
-
const { timeoutMs, idleTimeoutMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory } = opts;
|
|
5126
|
+
const { timeoutMs, idleTimeoutMs, recoveryWindowMs, envelope, sandbox = 'none', image, runId, secretEnv = {}, passThroughSecretNames = [], cwd, extraEnv = {}, profileEnv = {}, resultFile, stream = false, streamPrefix = '', onStreamOut, onStreamErr, args: commandArgs, terminal = 'pipe', protocol = 'pipe', permission = 'yolo', relaySession = null, ptyFactory } = opts;
|
|
4785
5127
|
// #110: `protocol`/`permission` drive the ACP executor branch below. The
|
|
4786
5128
|
// pipe/PTY paths are unchanged, so `protocol === 'pipe'` behaviour is identical.
|
|
4787
5129
|
const payload = JSON.stringify(buildAgentPayload(profile, job, envelope));
|
|
@@ -4851,6 +5193,7 @@ function runAgentJob(profile, job, opts = {}) {
|
|
|
4851
5193
|
stdinData: payload,
|
|
4852
5194
|
timeoutMs,
|
|
4853
5195
|
idleTimeoutMs,
|
|
5196
|
+
recoveryWindowMs,
|
|
4854
5197
|
relayTap,
|
|
4855
5198
|
stream,
|
|
4856
5199
|
streamPrefix,
|
|
@@ -4873,6 +5216,7 @@ function runAgentJob(profile, job, opts = {}) {
|
|
|
4873
5216
|
stdinData: payload,
|
|
4874
5217
|
timeoutMs,
|
|
4875
5218
|
idleTimeoutMs,
|
|
5219
|
+
recoveryWindowMs,
|
|
4876
5220
|
ptyFactory,
|
|
4877
5221
|
relayTap,
|
|
4878
5222
|
stream,
|
|
@@ -4894,6 +5238,7 @@ function runAgentJob(profile, job, opts = {}) {
|
|
|
4894
5238
|
stdinData: payload,
|
|
4895
5239
|
timeoutMs,
|
|
4896
5240
|
idleTimeoutMs,
|
|
5241
|
+
recoveryWindowMs,
|
|
4897
5242
|
onTimeout: (child) => killTree(child),
|
|
4898
5243
|
stream,
|
|
4899
5244
|
streamPrefix,
|
|
@@ -4955,6 +5300,7 @@ function runAgentJob(profile, job, opts = {}) {
|
|
|
4955
5300
|
stdinData: payload,
|
|
4956
5301
|
timeoutMs,
|
|
4957
5302
|
idleTimeoutMs,
|
|
5303
|
+
recoveryWindowMs,
|
|
4958
5304
|
stream,
|
|
4959
5305
|
streamPrefix,
|
|
4960
5306
|
onStreamOut,
|
|
@@ -5128,6 +5474,23 @@ function wsHostPart(host) {
|
|
|
5128
5474
|
return h.includes(':') && !h.startsWith('[') ? `[${h}]` : h;
|
|
5129
5475
|
}
|
|
5130
5476
|
|
|
5477
|
+
/**
|
|
5478
|
+
* URL-encode the zone id of an IPv6 scoped address so it is safe to embed in a
|
|
5479
|
+
* URL host (#133). A resolver may return a link-local address carrying an
|
|
5480
|
+
* interface zone id (e.g. `fe80::1%en0`); the raw `%` is not URL-safe and yields
|
|
5481
|
+
* an invalid `ws://[fe80::1%en0]:…` that fails to parse even when the address is
|
|
5482
|
+
* reachable. Percent-encode the delimiter (`%` → `%25`) so the bracketed host is
|
|
5483
|
+
* valid. Idempotent — an already-encoded zone id (`%25`) is left untouched — and
|
|
5484
|
+
* a no-op for any host without a zone id.
|
|
5485
|
+
*
|
|
5486
|
+
* @param {string} host a raw address, possibly `<ipv6>%<zone>`
|
|
5487
|
+
* @returns {string}
|
|
5488
|
+
*/
|
|
5489
|
+
function encodeIpZoneId(host) {
|
|
5490
|
+
const h = String(host || '');
|
|
5491
|
+
return h.replace(/%(25)?/g, '%25');
|
|
5492
|
+
}
|
|
5493
|
+
|
|
5131
5494
|
/**
|
|
5132
5495
|
* Normalise the engine's `GET /console/api/projects` payload into the running
|
|
5133
5496
|
* embedded apps that advertise an agentic UI port. Accepts the shapes the
|
|
@@ -5207,6 +5570,136 @@ function probeAgenticChannel(port, {
|
|
|
5207
5570
|
});
|
|
5208
5571
|
}
|
|
5209
5572
|
|
|
5573
|
+
/**
|
|
5574
|
+
* Is `address` a link-local address? Node's default `dns.lookup` can return an
|
|
5575
|
+
* IPv6 link-local `fe80::…` (interface-scoped) address FIRST for a `.local`/mDNS
|
|
5576
|
+
* host, and on some hosts (notably macOS) opening a TCP/WS connection to such a
|
|
5577
|
+
* peer stalls for hundreds of ms — long enough to blow the discovery budget and
|
|
5578
|
+
* strand the worker in `advisory` (#133). Covers IPv6 `fe80::/10` and IPv4
|
|
5579
|
+
* link-local `169.254.0.0/16`; tolerates URL brackets and a zone-id suffix.
|
|
5580
|
+
* @param {string} address
|
|
5581
|
+
* @returns {boolean}
|
|
5582
|
+
*/
|
|
5583
|
+
function isLinkLocalAddress(address) {
|
|
5584
|
+
const a = String(address || '')
|
|
5585
|
+
.trim().toLowerCase()
|
|
5586
|
+
.replace(/^\[|\]$/g, '')
|
|
5587
|
+
.replace(/%.*$/, '');
|
|
5588
|
+
// IPv6 link-local is the full fe80::/10 block, whose first hextet spans
|
|
5589
|
+
// fe80–febf (the low byte's top two bits are `10`), not just the fe80:
|
|
5590
|
+
// prefix — match the whole range so e.g. fe90::/fea0::/febf:: are covered too.
|
|
5591
|
+
return /^fe[89ab][0-9a-f]:/.test(a) || /^169\.254\./.test(a);
|
|
5592
|
+
}
|
|
5593
|
+
|
|
5594
|
+
/**
|
|
5595
|
+
* Is `host` an IP literal (v4 or bracketed/bare v6) that needs no DNS
|
|
5596
|
+
* resolution? Used to skip the resolver for an address that is already concrete.
|
|
5597
|
+
* @param {string} host
|
|
5598
|
+
* @returns {boolean}
|
|
5599
|
+
*/
|
|
5600
|
+
function isIpLiteral(host) {
|
|
5601
|
+
const h = String(host || '').trim().replace(/^\[|\]$/g, '');
|
|
5602
|
+
if (!h) return false;
|
|
5603
|
+
if (/^\d{1,3}(\.\d{1,3}){3}$/.test(h)) return true; // IPv4 dotted-quad
|
|
5604
|
+
return h.includes(':'); // any colon → IPv6 literal
|
|
5605
|
+
}
|
|
5606
|
+
|
|
5607
|
+
/**
|
|
5608
|
+
* Order resolved connect candidates routable-first (#133, fix B). Ranks a
|
|
5609
|
+
* link-local `fe80::`/`169.254.` candidate LAST (stable within a rank) so a fast
|
|
5610
|
+
* routable IPv4/global address wins the Happy-Eyeballs race instead of a slow
|
|
5611
|
+
* interface-scoped one dominating the connect budget.
|
|
5612
|
+
* @param {Array<{address:string, family?:number}>} addresses
|
|
5613
|
+
* @returns {Array<{address:string, family?:number}>} a new, rank-sorted array
|
|
5614
|
+
*/
|
|
5615
|
+
function orderProbeAddresses(addresses) {
|
|
5616
|
+
const list = Array.isArray(addresses) ? addresses.filter((a) => a && a.address) : [];
|
|
5617
|
+
return list
|
|
5618
|
+
.map((a, i) => ({ a, i, rank: isLinkLocalAddress(a.address) ? 1 : 0 }))
|
|
5619
|
+
.sort((x, y) => (x.rank - y.rank) || (x.i - y.i))
|
|
5620
|
+
.map((x) => x.a);
|
|
5621
|
+
}
|
|
5622
|
+
|
|
5623
|
+
/**
|
|
5624
|
+
* Resolve a probe host to an ordered list of candidate host strings to try,
|
|
5625
|
+
* routable-first (#133, fix B). A loopback host, an IP literal, or an
|
|
5626
|
+
* unavailable resolver yields the single host verbatim (unchanged legacy
|
|
5627
|
+
* behaviour); a real hostname is resolved via `lookupImpl(host, { all:true })`
|
|
5628
|
+
* and its addresses are ranked so a link-local `fe80::` never dominates the
|
|
5629
|
+
* connect budget. Fail-open: any resolver error or empty result falls back to
|
|
5630
|
+
* `[host]`, so a host that can't be pre-resolved still gets its one legacy probe.
|
|
5631
|
+
* @param {string} host
|
|
5632
|
+
* @param {{ lookupImpl?: Function }} [opts]
|
|
5633
|
+
* @returns {Promise<string[]>}
|
|
5634
|
+
*/
|
|
5635
|
+
async function resolveProbeCandidates(host, { lookupImpl = dnsLookup } = {}) {
|
|
5636
|
+
const h = String(host || '').trim();
|
|
5637
|
+
if (!h) return [];
|
|
5638
|
+
if (isLoopbackHost(h) || isIpLiteral(h) || typeof lookupImpl !== 'function') return [h];
|
|
5639
|
+
try {
|
|
5640
|
+
const all = await lookupImpl(h, { all: true });
|
|
5641
|
+
const hosts = orderProbeAddresses(Array.isArray(all) ? all : [])
|
|
5642
|
+
.map((a) => wsHostPart(encodeIpZoneId(String(a.address))));
|
|
5643
|
+
return hosts.length ? hosts : [h];
|
|
5644
|
+
} catch {
|
|
5645
|
+
return [h];
|
|
5646
|
+
}
|
|
5647
|
+
}
|
|
5648
|
+
|
|
5649
|
+
/**
|
|
5650
|
+
* Race a WS `/agentic` probe across ordered candidate hosts (Happy-Eyeballs,
|
|
5651
|
+
* #133 fix B) and resolve with the FIRST host whose socket opens — or `null` if
|
|
5652
|
+
* every candidate fails/times out. Candidates are tried in order (routable
|
|
5653
|
+
* first) with a small stagger so the preferred address gets a head start but a
|
|
5654
|
+
* slow-to-open one can't stall the whole probe: a fast later candidate still
|
|
5655
|
+
* wins. Each per-host probe is bounded by `timeoutMs`. A single candidate skips
|
|
5656
|
+
* the racing machinery entirely (unchanged legacy path).
|
|
5657
|
+
* @param {number} port
|
|
5658
|
+
* @param {{ hosts?: string[], token?: string, timeoutMs?: number, wsProbe?: Function, staggerMs?: number }} [opts]
|
|
5659
|
+
* @returns {Promise<string|null>} the winning host, or null
|
|
5660
|
+
*/
|
|
5661
|
+
async function raceProbeCandidates(port, {
|
|
5662
|
+
hosts = [],
|
|
5663
|
+
token = LOCAL_AGENTIC_TOKEN,
|
|
5664
|
+
timeoutMs = AGENTIC_DISCOVERY_TIMEOUT_MS,
|
|
5665
|
+
wsProbe = probeAgenticChannel,
|
|
5666
|
+
staggerMs = 250,
|
|
5667
|
+
} = {}) {
|
|
5668
|
+
const list = Array.isArray(hosts) ? hosts.filter(Boolean) : [];
|
|
5669
|
+
if (list.length === 0) return null;
|
|
5670
|
+
if (list.length === 1) {
|
|
5671
|
+
try {
|
|
5672
|
+
return (await wsProbe(port, { host: list[0], token, timeoutMs })) ? list[0] : null;
|
|
5673
|
+
} catch {
|
|
5674
|
+
return null;
|
|
5675
|
+
}
|
|
5676
|
+
}
|
|
5677
|
+
return new Promise((resolve) => {
|
|
5678
|
+
let pending = list.length;
|
|
5679
|
+
let settled = false;
|
|
5680
|
+
const timers = [];
|
|
5681
|
+
const done = (host) => {
|
|
5682
|
+
if (settled) return;
|
|
5683
|
+
settled = true;
|
|
5684
|
+
timers.forEach(clearTimeout);
|
|
5685
|
+
resolve(host);
|
|
5686
|
+
};
|
|
5687
|
+
const start = (host) => {
|
|
5688
|
+
Promise.resolve()
|
|
5689
|
+
.then(() => wsProbe(port, { host, token, timeoutMs }))
|
|
5690
|
+
.catch(() => false)
|
|
5691
|
+
.then((ok) => {
|
|
5692
|
+
if (ok) done(host);
|
|
5693
|
+
else if (--pending === 0) done(null);
|
|
5694
|
+
});
|
|
5695
|
+
};
|
|
5696
|
+
list.forEach((host, idx) => {
|
|
5697
|
+
if (idx === 0) start(host);
|
|
5698
|
+
else timers.push(setTimeout(() => start(host), staggerMs * idx));
|
|
5699
|
+
});
|
|
5700
|
+
});
|
|
5701
|
+
}
|
|
5702
|
+
|
|
5210
5703
|
/**
|
|
5211
5704
|
* Auto-discover the embedded nwf agentic hub(s) reachable from an engine base
|
|
5212
5705
|
* URL (#75, #96). Reads `GET <engine>/console/api/projects`, keeps the apps that
|
|
@@ -5216,20 +5709,25 @@ function probeAgenticChannel(port, {
|
|
|
5216
5709
|
* loopback engine probes `127.0.0.1`, a remote engine (e.g. `merlin.local`)
|
|
5217
5710
|
* probes that same host — the port is taken from the projects API but the host is
|
|
5218
5711
|
* always the engine's, so a rogue projects API can never steer a probe at the
|
|
5219
|
-
* worker's own loopback (#76).
|
|
5220
|
-
*
|
|
5221
|
-
*
|
|
5222
|
-
*
|
|
5712
|
+
* worker's own loopback (#76). Gives the projects fetch and each WS probe
|
|
5713
|
+
* INDEPENDENT deadlines (#133) so a slow fetch can't starve the probe, prefers a
|
|
5714
|
+
* routable address over a link-local `fe80::` one (Happy-Eyeballs), and is
|
|
5715
|
+
* fail-open: any error — not a nano engine (Camunda), network failure, malformed
|
|
5716
|
+
* body, or a timeout — degrades to `[]` so the worker's real job is never
|
|
5717
|
+
* blocked.
|
|
5223
5718
|
*
|
|
5224
5719
|
* @param {string} engineBaseUrl the engine base URL (e.g. `http://merlin.local:8080`)
|
|
5225
|
-
* @param {{ token?: string, fetchImpl?: Function, wsProbe?: Function, timeoutMs?: number }} [opts]
|
|
5720
|
+
* @param {{ token?: string, fetchImpl?: Function, wsProbe?: Function, lookupImpl?: Function, timeoutMs?: number, fetchTimeoutMs?: number, probeTimeoutMs?: number }} [opts]
|
|
5226
5721
|
* @returns {Promise<Array<{ project: string, port: number, label?: string, host: string }>>}
|
|
5227
5722
|
*/
|
|
5228
5723
|
async function discoverAgenticHubs(engineBaseUrl, {
|
|
5229
5724
|
token = LOCAL_AGENTIC_TOKEN,
|
|
5230
5725
|
fetchImpl = globalThis.fetch,
|
|
5231
5726
|
wsProbe = probeAgenticChannel,
|
|
5727
|
+
lookupImpl = dnsLookup,
|
|
5232
5728
|
timeoutMs = AGENTIC_DISCOVERY_TIMEOUT_MS,
|
|
5729
|
+
fetchTimeoutMs = timeoutMs,
|
|
5730
|
+
probeTimeoutMs = timeoutMs,
|
|
5233
5731
|
} = {}) {
|
|
5234
5732
|
if (typeof fetchImpl !== 'function' || typeof engineBaseUrl !== 'string' || !engineBaseUrl.trim()) {
|
|
5235
5733
|
return [];
|
|
@@ -5250,14 +5748,15 @@ async function discoverAgenticHubs(engineBaseUrl, {
|
|
|
5250
5748
|
return [];
|
|
5251
5749
|
}
|
|
5252
5750
|
const probeHost = isLoopbackHost(host) ? '127.0.0.1' : host;
|
|
5253
|
-
//
|
|
5254
|
-
// deadline
|
|
5255
|
-
//
|
|
5256
|
-
//
|
|
5257
|
-
|
|
5751
|
+
// (C) Decoupled budgets (#133): the projects fetch and each WS probe get their
|
|
5752
|
+
// OWN independent deadline. Previously they shared one 2s budget, so a fetch
|
|
5753
|
+
// that consumed most of the window starved the probe to ~0ms remaining and
|
|
5754
|
+
// stranded an otherwise-reachable worker in `advisory`. They no longer subtract
|
|
5755
|
+
// from each other — the fetch is bounded by `fetchTimeoutMs`, each probe by
|
|
5756
|
+
// `probeTimeoutMs`.
|
|
5258
5757
|
let projects;
|
|
5259
5758
|
const controller = new AbortController();
|
|
5260
|
-
const timer = setTimeout(() => controller.abort(),
|
|
5759
|
+
const timer = setTimeout(() => controller.abort(), fetchTimeoutMs);
|
|
5261
5760
|
try {
|
|
5262
5761
|
const res = await fetchImpl(`${base}/console/api/projects`, { signal: controller.signal });
|
|
5263
5762
|
if (!res || !res.ok) return [];
|
|
@@ -5269,15 +5768,23 @@ async function discoverAgenticHubs(engineBaseUrl, {
|
|
|
5269
5768
|
}
|
|
5270
5769
|
const apps = normalizeProjectApps(projects);
|
|
5271
5770
|
if (apps.length === 0) return [];
|
|
5272
|
-
|
|
5273
|
-
|
|
5274
|
-
//
|
|
5275
|
-
//
|
|
5771
|
+
// (B) Prefer a routable address over a link-local one and race families
|
|
5772
|
+
// (Happy-Eyeballs, #133): resolve the engine host to ordered candidate
|
|
5773
|
+
// addresses (routable first) so a fast IPv4/global address wins over a slow
|
|
5774
|
+
// `fe80::` link-local one that Node's default dns.lookup may return first. Each
|
|
5775
|
+
// surviving hub carries the WINNING host so the caller (and the real channel
|
|
5776
|
+
// connect) reuse that fast, routable address rather than re-resolving to the
|
|
5777
|
+
// slow link-local path.
|
|
5778
|
+
const candidates = await resolveProbeCandidates(probeHost, { lookupImpl });
|
|
5276
5779
|
const settled = await Promise.all(apps.map(async (app) => {
|
|
5277
5780
|
try {
|
|
5278
|
-
|
|
5279
|
-
|
|
5280
|
-
|
|
5781
|
+
const winner = await raceProbeCandidates(app.port, {
|
|
5782
|
+
hosts: candidates,
|
|
5783
|
+
token,
|
|
5784
|
+
timeoutMs: probeTimeoutMs,
|
|
5785
|
+
wsProbe,
|
|
5786
|
+
});
|
|
5787
|
+
return winner ? { ...app, host: winner } : null;
|
|
5281
5788
|
} catch {
|
|
5282
5789
|
return null;
|
|
5283
5790
|
}
|
|
@@ -5302,10 +5809,16 @@ async function discoverAgenticHubs(engineBaseUrl, {
|
|
|
5302
5809
|
* no projects API / not a nano engine, or discovery error/timeout). The
|
|
5303
5810
|
* worker continues doing real work with no channel.
|
|
5304
5811
|
*
|
|
5305
|
-
*
|
|
5812
|
+
* (C) When a `cache` Map is supplied, a successfully-resolved hub URL is
|
|
5813
|
+
* remembered against the engine base URL, and a later discovery that comes up
|
|
5814
|
+
* empty for that same engine reuses the cached known-good hub instead of dropping
|
|
5815
|
+
* to `advisory` — so a brief blip doesn't strip visibility from an
|
|
5816
|
+
* already-connected worker (#133).
|
|
5817
|
+
*
|
|
5818
|
+
* @param {{ camunda?: object, fetchImpl?: Function, wsProbe?: Function, lookupImpl?: Function, cache?: Map, timeoutMs?: number }} [opts]
|
|
5306
5819
|
* @returns {Promise<{ status: string, config?: object, message?: string, candidates?: Array }>}
|
|
5307
5820
|
*/
|
|
5308
|
-
async function resolveAgenticTarget({ camunda, ...opts } = {}) {
|
|
5821
|
+
async function resolveAgenticTarget({ camunda, cache, ...opts } = {}) {
|
|
5309
5822
|
const base = resolveAgenticConfig(camunda);
|
|
5310
5823
|
if (!base) return { status: 'off' };
|
|
5311
5824
|
// Explicit target wins verbatim and skips discovery entirely.
|
|
@@ -5324,14 +5837,14 @@ async function resolveAgenticTarget({ camunda, ...opts } = {}) {
|
|
|
5324
5837
|
|
|
5325
5838
|
if (hubs.length === 1) {
|
|
5326
5839
|
const { project, port, host } = hubs[0];
|
|
5327
|
-
|
|
5328
|
-
|
|
5329
|
-
|
|
5330
|
-
|
|
5331
|
-
url: `http://${wsHostPart(host)}:${port}`,
|
|
5332
|
-
discovered: { project, port, host },
|
|
5333
|
-
},
|
|
5840
|
+
const config = {
|
|
5841
|
+
...base,
|
|
5842
|
+
url: `http://${wsHostPart(host)}:${port}`,
|
|
5843
|
+
discovered: { project, port, host },
|
|
5334
5844
|
};
|
|
5845
|
+
// Cache the known-good hub so a later blip self-heals from cache (#133-C).
|
|
5846
|
+
if (cache && typeof cache.set === 'function') cache.set(base.url, config);
|
|
5847
|
+
return { status: 'connect', config };
|
|
5335
5848
|
}
|
|
5336
5849
|
if (hubs.length > 1) {
|
|
5337
5850
|
const list = hubs.map((h) => `${h.project} → :${h.port}`).join(', ');
|
|
@@ -5342,6 +5855,13 @@ async function resolveAgenticTarget({ camunda, ...opts } = {}) {
|
|
|
5342
5855
|
+ `Disambiguate by setting NANO_AGENTIC_URL=http://${suggestHost}:<port> (or persisted agenticUrl) to the one you want.`,
|
|
5343
5856
|
};
|
|
5344
5857
|
}
|
|
5858
|
+
// Zero matches: reuse a cached known-good hub for this engine if we have one, so
|
|
5859
|
+
// a transient discovery miss doesn't drop an already-known-good worker to
|
|
5860
|
+
// advisory (#133-C).
|
|
5861
|
+
if (cache && typeof cache.get === 'function') {
|
|
5862
|
+
const cached = cache.get(base.url);
|
|
5863
|
+
if (cached) return { status: 'connect', config: { ...cached, fromCache: true } };
|
|
5864
|
+
}
|
|
5345
5865
|
return {
|
|
5346
5866
|
status: 'advisory',
|
|
5347
5867
|
message: `agentic visibility was not discoverable at ${base.url} — the embedded app port could `
|
|
@@ -5350,6 +5870,94 @@ async function resolveAgenticTarget({ camunda, ...opts } = {}) {
|
|
|
5350
5870
|
};
|
|
5351
5871
|
}
|
|
5352
5872
|
|
|
5873
|
+
/**
|
|
5874
|
+
* A jittered backoff schedule (ms) for background agentic re-discovery (#133-A):
|
|
5875
|
+
* ~2s → 4s → 8s → 16s → 30s (capped), each waited value randomised ±20%, spanning
|
|
5876
|
+
* roughly `ceilingMs` (default ~5 minutes) of total elapsed retry time before the
|
|
5877
|
+
* loop gives up. A worker that lost the cold-start discovery race walks this
|
|
5878
|
+
* schedule to upgrade `advisory → connected` without a restart.
|
|
5879
|
+
* @param {{ base?: number, cap?: number, ceilingMs?: number, rng?: () => number }} [opts]
|
|
5880
|
+
* @returns {number[]} the ordered per-attempt wait durations
|
|
5881
|
+
*/
|
|
5882
|
+
function defaultAgenticRediscoveryDelays({
|
|
5883
|
+
base = 2_000,
|
|
5884
|
+
cap = 30_000,
|
|
5885
|
+
ceilingMs = 5 * 60 * 1_000,
|
|
5886
|
+
rng = Math.random,
|
|
5887
|
+
} = {}) {
|
|
5888
|
+
const delays = [];
|
|
5889
|
+
let d = base;
|
|
5890
|
+
let total = 0;
|
|
5891
|
+
while (total < ceilingMs) {
|
|
5892
|
+
const jitter = Math.round((rng() - 0.5) * 0.4 * d); // ±20%
|
|
5893
|
+
const wait = Math.max(500, d + jitter);
|
|
5894
|
+
delays.push(wait);
|
|
5895
|
+
total += wait;
|
|
5896
|
+
d = Math.min(cap, d * 2);
|
|
5897
|
+
}
|
|
5898
|
+
return delays;
|
|
5899
|
+
}
|
|
5900
|
+
|
|
5901
|
+
/**
|
|
5902
|
+
* Background self-heal loop for agentic discovery (#133-A). After an initial
|
|
5903
|
+
* cold-start miss leaves a worker in `advisory`, re-run `resolveTarget` on a
|
|
5904
|
+
* jittered backoff schedule and, on the FIRST attempt that yields a
|
|
5905
|
+
* `status:'connect'` target, invoke `onConnect(target)` and stop — so the worker
|
|
5906
|
+
* upgrades to `connected` without a restart. Fail-open: an attempt whose
|
|
5907
|
+
* `resolveTarget` throws is swallowed and the loop continues; likewise, if
|
|
5908
|
+
* `onConnect` itself throws (a transient channel-open failure), the loop keeps
|
|
5909
|
+
* re-discovering rather than stopping, so retries proceed until a connect
|
|
5910
|
+
* callback actually succeeds. The loop also stops early whenever
|
|
5911
|
+
* `shouldContinue()` returns false (e.g. a channel already came up, or the worker
|
|
5912
|
+
* is shutting down). Returns the connecting target, or `null` if the schedule was
|
|
5913
|
+
* exhausted / cancelled without a hit. Timers and the resolver are injectable so
|
|
5914
|
+
* this is unit-testable without real waits or sockets.
|
|
5915
|
+
*
|
|
5916
|
+
* @param {{
|
|
5917
|
+
* resolveTarget: () => Promise<{status:string, config?:object}>,
|
|
5918
|
+
* onConnect?: (target: {status:string, config?:object}) => (void|Promise<void>),
|
|
5919
|
+
* delaysMs?: number[],
|
|
5920
|
+
* sleep?: (ms:number) => Promise<void>,
|
|
5921
|
+
* shouldContinue?: () => boolean,
|
|
5922
|
+
* logger?: object,
|
|
5923
|
+
* }} opts
|
|
5924
|
+
* @returns {Promise<object|null>}
|
|
5925
|
+
*/
|
|
5926
|
+
async function rediscoverAgenticUntilConnected({
|
|
5927
|
+
resolveTarget,
|
|
5928
|
+
onConnect,
|
|
5929
|
+
delaysMs = defaultAgenticRediscoveryDelays(),
|
|
5930
|
+
sleep = (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
5931
|
+
shouldContinue = () => true,
|
|
5932
|
+
logger = null,
|
|
5933
|
+
} = {}) {
|
|
5934
|
+
if (typeof resolveTarget !== 'function') return null;
|
|
5935
|
+
for (const delay of delaysMs) {
|
|
5936
|
+
if (!shouldContinue()) return null;
|
|
5937
|
+
try { await sleep(delay); } catch { return null; }
|
|
5938
|
+
if (!shouldContinue()) return null;
|
|
5939
|
+
let target;
|
|
5940
|
+
try {
|
|
5941
|
+
target = await resolveTarget();
|
|
5942
|
+
} catch (err) {
|
|
5943
|
+
logger?.debug?.(`agentic re-discovery attempt failed: ${err?.message || err}`);
|
|
5944
|
+
continue;
|
|
5945
|
+
}
|
|
5946
|
+
if (target && target.status === 'connect') {
|
|
5947
|
+
try {
|
|
5948
|
+
await onConnect?.(target);
|
|
5949
|
+
} catch (err) {
|
|
5950
|
+
// A transient channel-open failure must not prematurely stop self-heal:
|
|
5951
|
+
// keep re-discovering until an onConnect callback actually succeeds.
|
|
5952
|
+
logger?.debug?.(`agentic re-discovery onConnect failed, will retry: ${err?.message || err}`);
|
|
5953
|
+
continue;
|
|
5954
|
+
}
|
|
5955
|
+
return target;
|
|
5956
|
+
}
|
|
5957
|
+
}
|
|
5958
|
+
return null;
|
|
5959
|
+
}
|
|
5960
|
+
|
|
5353
5961
|
/**
|
|
5354
5962
|
* Collapse an agentic disconnect/failure detail into the single short string the
|
|
5355
5963
|
* marker's `agentic.message` field carries (#99 contract). Accepts the close
|
|
@@ -5852,7 +6460,17 @@ async function workAgent(req, flags) {
|
|
|
5852
6460
|
// `supervisor status` reflects connecting/advisory/off immediately, before
|
|
5853
6461
|
// the socket opens (or without a channel at all).
|
|
5854
6462
|
writeActivity();
|
|
5855
|
-
|
|
6463
|
+
// Track the live connection state on the activity marker so the supervisor
|
|
6464
|
+
// shows connected↔disconnected transitions (#99). A close carries a normalized
|
|
6465
|
+
// diagnostic under the contract `agentic.message` field (not `reason`) so a hub
|
|
6466
|
+
// drop explains WHY; a fresh (re)connect clears any stale message.
|
|
6467
|
+
const markAgentic = (status, message = null) => { agenticState = { ...agenticState, status, message }; writeActivity(); };
|
|
6468
|
+
// Open (or re-open) the visibility channel for a resolved connect config. This
|
|
6469
|
+
// is the SINGLE place the connected+authenticated client is instantiated — the
|
|
6470
|
+
// initial connect path and the background self-heal loop (#133) both call it,
|
|
6471
|
+
// (re)assigning the shared `workChannel`/`bufferMonitor` closures the job
|
|
6472
|
+
// recorders and shutdown path already track.
|
|
6473
|
+
const openAgenticChannel = async (cfg) => {
|
|
5856
6474
|
try {
|
|
5857
6475
|
workChannel = await createWorkChannel({
|
|
5858
6476
|
instance: workerName,
|
|
@@ -5863,30 +6481,25 @@ async function workAgent(req, flags) {
|
|
|
5863
6481
|
host: hostname(),
|
|
5864
6482
|
},
|
|
5865
6483
|
listJobKeys: () => [...activeJobs.keys()],
|
|
5866
|
-
url:
|
|
5867
|
-
token:
|
|
5868
|
-
credential:
|
|
5869
|
-
bufferCapacity:
|
|
6484
|
+
url: cfg.url,
|
|
6485
|
+
token: cfg.token,
|
|
6486
|
+
credential: cfg.credential,
|
|
6487
|
+
bufferCapacity: cfg.bufferCapacity,
|
|
5870
6488
|
logger,
|
|
5871
6489
|
});
|
|
5872
|
-
const shown = redactAgenticUrl(buildAgenticUrl(
|
|
5873
|
-
const mode =
|
|
5874
|
-
if (
|
|
5875
|
-
const d =
|
|
6490
|
+
const shown = redactAgenticUrl(buildAgenticUrl(cfg.url, {}));
|
|
6491
|
+
const mode = cfg.secure ? 'secure' : 'local';
|
|
6492
|
+
if (cfg.discovered) {
|
|
6493
|
+
const d = cfg.discovered;
|
|
5876
6494
|
logger.info(` agentic channel: auto-discovered ${d.project} on the app's /agentic port ${wsHostPart(d.host)}:${d.port} (bypassing the WS-incapable console proxy).`);
|
|
5877
6495
|
}
|
|
5878
6496
|
logger.info(` agentic channel (${mode}): announcing presence as ${workerName} on ${shown}`);
|
|
5879
|
-
//
|
|
5880
|
-
//
|
|
5881
|
-
//
|
|
5882
|
-
//
|
|
5883
|
-
//
|
|
5884
|
-
//
|
|
5885
|
-
// true — record that as `disconnected` rather than leaving it stuck at
|
|
5886
|
-
// `connecting`. A close carries a normalized diagnostic under the contract
|
|
5887
|
-
// `agentic.message` field (not `reason`) so a hub drop explains WHY; a
|
|
5888
|
-
// fresh (re)connect clears any stale message.
|
|
5889
|
-
const markAgentic = (status, message = null) => { agenticState = { ...agenticState, status, message }; writeActivity(); };
|
|
6497
|
+
// onConnect fires only for listeners present at first open, so also
|
|
6498
|
+
// reconcile the already-open case synchronously via connected(). If the
|
|
6499
|
+
// socket opened and then dropped inside the createWorkChannel() await window
|
|
6500
|
+
// (before these listeners existed), connected() is false but everConnected()
|
|
6501
|
+
// is true — record that as `disconnected` rather than leaving it stuck at
|
|
6502
|
+
// `connecting`.
|
|
5890
6503
|
workChannel.onConnect(() => markAgentic('connected'));
|
|
5891
6504
|
workChannel.onReconnect(() => markAgentic('connected'));
|
|
5892
6505
|
workChannel.onDisconnect((info) => markAgentic('disconnected', normalizeAgenticMessage(info)));
|
|
@@ -5902,6 +6515,7 @@ async function workAgent(req, flags) {
|
|
|
5902
6515
|
agenticState = { ...agenticState, status: 'disconnected', message: normalizeAgenticMessage(err) };
|
|
5903
6516
|
writeActivity();
|
|
5904
6517
|
logger.warn(` agentic channel unavailable (${err?.message || err}); continuing without visibility.`);
|
|
6518
|
+
return;
|
|
5905
6519
|
}
|
|
5906
6520
|
// C4 (#43): observe the client's built-in outbound buffer across the
|
|
5907
6521
|
// channel lifecycle — surface a high-water mark and warn when the bound
|
|
@@ -5912,7 +6526,7 @@ async function workAgent(req, flags) {
|
|
|
5912
6526
|
if (workChannel) {
|
|
5913
6527
|
try {
|
|
5914
6528
|
bufferMonitor = createBufferMonitor(workChannel, {
|
|
5915
|
-
capacity:
|
|
6529
|
+
capacity: cfg.bufferCapacity,
|
|
5916
6530
|
logger,
|
|
5917
6531
|
});
|
|
5918
6532
|
} catch (err) {
|
|
@@ -5920,6 +6534,39 @@ async function workAgent(req, flags) {
|
|
|
5920
6534
|
logger.warn(` agentic buffer monitor unavailable (${err?.message || err}); channel presence still active.`);
|
|
5921
6535
|
}
|
|
5922
6536
|
}
|
|
6537
|
+
};
|
|
6538
|
+
|
|
6539
|
+
if (agenticCfg) {
|
|
6540
|
+
await openAgenticChannel(agenticCfg);
|
|
6541
|
+
} else if (agenticTarget.status === 'advisory') {
|
|
6542
|
+
// (A) Self-heal a cold-start discovery miss (#133): discovery is one-shot at
|
|
6543
|
+
// enrolment, so a worker that merely lost the cold-start race (e.g. a slow
|
|
6544
|
+
// link-local candidate blew the budget) would otherwise run `advisory` for
|
|
6545
|
+
// its whole lifetime — the only recovery being a restart. Keep re-discovering
|
|
6546
|
+
// in the background on a jittered backoff and, on a later success, upgrade
|
|
6547
|
+
// advisory→connected WITHOUT a restart, flipping the AGENTIC status surface.
|
|
6548
|
+
// A shared cache lets a brief blip reuse the last known-good hub (#133-C).
|
|
6549
|
+
const hubCache = new Map();
|
|
6550
|
+
rediscoverAgenticUntilConnected({
|
|
6551
|
+
resolveTarget: () => resolveAgenticTarget({ camunda, logger, cache: hubCache }),
|
|
6552
|
+
onConnect: async (target) => {
|
|
6553
|
+
agenticCfg = target.config;
|
|
6554
|
+
agenticState = agenticStateForTarget(target, safeAgenticDisplayUrl);
|
|
6555
|
+
writeActivity();
|
|
6556
|
+
logger.info(' agentic channel: background re-discovery succeeded — upgrading advisory → connecting.');
|
|
6557
|
+
await openAgenticChannel(agenticCfg);
|
|
6558
|
+
// openAgenticChannel swallows its own open failures (it nulls
|
|
6559
|
+
// workChannel and returns rather than throwing), so a failed open must
|
|
6560
|
+
// be re-thrown here — otherwise the self-heal loop treats this attempt
|
|
6561
|
+
// as success and stops retrying with workChannel still null (#133).
|
|
6562
|
+
if (workChannel === null) {
|
|
6563
|
+
throw new Error('agentic channel failed to open after background re-discovery');
|
|
6564
|
+
}
|
|
6565
|
+
},
|
|
6566
|
+
// Stop as soon as a channel exists (loop won this or a prior attempt did).
|
|
6567
|
+
shouldContinue: () => workChannel === null,
|
|
6568
|
+
logger,
|
|
6569
|
+
}).catch(() => { /* best-effort self-heal — never surfaces an error */ });
|
|
5923
6570
|
}
|
|
5924
6571
|
|
|
5925
6572
|
// C3 (#42): the role's live-terminal mode — a full PTY (streamed on the relay
|
|
@@ -6025,9 +6672,57 @@ async function workAgent(req, flags) {
|
|
|
6025
6672
|
return job.fail({ errorMessage: msg, retries });
|
|
6026
6673
|
}
|
|
6027
6674
|
|
|
6675
|
+
// #130: per-task liveness overrides. Precedence: envelope override →
|
|
6676
|
+
// worker-flag default → built-in default, each clamped to a sane max so
|
|
6677
|
+
// a task can't request an unbounded window. Absent envelope fields leave
|
|
6678
|
+
// the worker-flag behaviour unchanged. These drive BOTH the harness idle
|
|
6679
|
+
// liveness (idle/recovery) AND the broker lock recovery window, so a
|
|
6680
|
+
// JVM-heavy task can widen its own window without a global flag change.
|
|
6681
|
+
const {
|
|
6682
|
+
idleTimeoutMs: effectiveIdleTimeoutMs,
|
|
6683
|
+
recoveryWindowMs: effectiveRecoveryWindowMs,
|
|
6684
|
+
hardCapMs: effectiveHardCapMs,
|
|
6685
|
+
} = resolveLivenessOverrides(envelope.task, { idleTimeoutMs, recoveryWindowMs, hardCapMs });
|
|
6686
|
+
// Recompute the lock-extend cadence from the effective recovery window
|
|
6687
|
+
// (same ~1/3-of-window rule as at startup) so a widened window still
|
|
6688
|
+
// renews comfortably before it lapses.
|
|
6689
|
+
const effectiveLockExtendIntervalMs = Math.min(
|
|
6690
|
+
Math.max(5_000, Math.floor(effectiveRecoveryWindowMs / 3)),
|
|
6691
|
+
Math.max(1, Math.floor(effectiveRecoveryWindowMs * 0.75)),
|
|
6692
|
+
);
|
|
6693
|
+
if (
|
|
6694
|
+
effectiveIdleTimeoutMs !== idleTimeoutMs ||
|
|
6695
|
+
effectiveRecoveryWindowMs !== recoveryWindowMs ||
|
|
6696
|
+
effectiveHardCapMs !== hardCapMs
|
|
6697
|
+
) {
|
|
6698
|
+
logger.info(`[${jobType}] job ${job.jobKey} liveness (envelope override) → idle timeout: ${effectiveIdleTimeoutMs}ms; recovery window: ${effectiveRecoveryWindowMs}ms; hard cap: ${effectiveHardCapMs > 0 ? `${effectiveHardCapMs}ms` : 'off'}`);
|
|
6699
|
+
}
|
|
6700
|
+
|
|
6028
6701
|
const runId = randomUUID();
|
|
6029
6702
|
if (isContainer) liveRunIds.add(runId);
|
|
6030
6703
|
|
|
6704
|
+
// Fail-closed on a half-specified repository envelope (issue #129,
|
|
6705
|
+
// hardening 2): a `repository` block that declares intent (any field set)
|
|
6706
|
+
// but whose `url` is absent or not a usable clone target almost always
|
|
6707
|
+
// means an orchestrator bug. Refuse the job LOUDLY (retryable) instead of
|
|
6708
|
+
// silently degrading to a launch/temp cwd and running an agent with no
|
|
6709
|
+
// repo where one was expected. A task with NO `repository` block at all
|
|
6710
|
+
// is legitimately repo-less and takes the temp-cwd default below.
|
|
6711
|
+
// Container runs don't provision a host repo (cloning is a later
|
|
6712
|
+
// increment), so this gate is host-only, mirroring `hasRepo`.
|
|
6713
|
+
if (!isContainer) {
|
|
6714
|
+
const repoIntent = classifyRepoEnvelope(job.customHeaders ?? {}, job.variables ?? {});
|
|
6715
|
+
if (repoIntent.malformed) {
|
|
6716
|
+
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
6717
|
+
const why = repoIntent.url
|
|
6718
|
+
? `repository.url ${JSON.stringify(repoIntent.url)} is not a usable clone target`
|
|
6719
|
+
: 'repository.url is missing';
|
|
6720
|
+
const msg = `incomplete repository envelope — ${why}; refusing to run in the launch/temp cwd (likely an orchestrator bug emitting a half-specified repository block)`;
|
|
6721
|
+
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
6722
|
+
return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
6723
|
+
}
|
|
6724
|
+
}
|
|
6725
|
+
|
|
6031
6726
|
// Host git provisioning (increment 2a): sandbox=none + a repository →
|
|
6032
6727
|
// clone into a throwaway workspace, run the harness there, then push +
|
|
6033
6728
|
// reconcile the agent PR. Container-side cloning is a later increment.
|
|
@@ -6039,7 +6734,7 @@ async function workAgent(req, flags) {
|
|
|
6039
6734
|
// here — ahead of provisionRepo — guarantees the first renewal is queued
|
|
6040
6735
|
// before the clone, so the lock can't lapse mid-provision and trigger the
|
|
6041
6736
|
// duplicate-activation / stale-409 race. The `finally` below stops it.
|
|
6042
|
-
stopLockExtender = startLockExtender(job,
|
|
6737
|
+
stopLockExtender = startLockExtender(job, effectiveRecoveryWindowMs, effectiveLockExtendIntervalMs, `[${jobType}] job ${job.jobKey}`, logger);
|
|
6043
6738
|
let cwd;
|
|
6044
6739
|
let extraEnv;
|
|
6045
6740
|
let repoToken = null;
|
|
@@ -6083,6 +6778,32 @@ async function workAgent(req, flags) {
|
|
|
6083
6778
|
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
6084
6779
|
return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
6085
6780
|
}
|
|
6781
|
+
} else if (!isContainer) {
|
|
6782
|
+
// Repo-less host job (issue #129, hardening 1): nothing is provisioned,
|
|
6783
|
+
// so default the agent's cwd to a fresh, empty `run-*` dir under the
|
|
6784
|
+
// runs root instead of leaving it undefined (which inherits the
|
|
6785
|
+
// worker's launch dir and lets stray scratch files / a `git init`
|
|
6786
|
+
// pollute it — or worse, an agent `cd` into a shared checkout). Track
|
|
6787
|
+
// it in `liveRunDirs` and let the SAME `finally` below reap it, reusing
|
|
6788
|
+
// the existing run-dir lifecycle — the only difference from the
|
|
6789
|
+
// provisioned path is an EMPTY workspace.
|
|
6790
|
+
//
|
|
6791
|
+
// Hygiene, not a sandbox: the agent runs as the user with full
|
|
6792
|
+
// filesystem access, so an empty cwd does not CONFINE it (it can still
|
|
6793
|
+
// `cd` to a known absolute path). True confinement is the container
|
|
6794
|
+
// increment; a provisioned repository envelope stays the preferred path.
|
|
6795
|
+
try {
|
|
6796
|
+
mkdirSync(agentRunsRoot(), { recursive: true });
|
|
6797
|
+
runDir = mkdtempSync(join(agentRunsRoot(), 'run-'));
|
|
6798
|
+
liveRunDirs.add(runDir);
|
|
6799
|
+
cwd = runDir;
|
|
6800
|
+
} catch (err) {
|
|
6801
|
+
if (runDir) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(runDir); runDir = null; }
|
|
6802
|
+
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
6803
|
+
const msg = `could not create a temp workspace under the runs root: ${err.message}`;
|
|
6804
|
+
logger.warn(`[${jobType}] job ${job.jobKey} not provisioned — ${msg}; retries left ${retries}`);
|
|
6805
|
+
return job.fail({ errorMessage: msg.slice(0, 2000), retries, retryBackOff: 15_000 });
|
|
6806
|
+
}
|
|
6086
6807
|
}
|
|
6087
6808
|
|
|
6088
6809
|
let result;
|
|
@@ -6115,8 +6836,9 @@ async function workAgent(req, flags) {
|
|
|
6115
6836
|
} catch { resultDir = null; resultFile = null; }
|
|
6116
6837
|
|
|
6117
6838
|
result = await runAgentJob(profile, job, {
|
|
6118
|
-
timeoutMs:
|
|
6119
|
-
idleTimeoutMs,
|
|
6839
|
+
timeoutMs: effectiveHardCapMs,
|
|
6840
|
+
idleTimeoutMs: effectiveIdleTimeoutMs,
|
|
6841
|
+
recoveryWindowMs: effectiveRecoveryWindowMs,
|
|
6120
6842
|
envelope,
|
|
6121
6843
|
sandbox,
|
|
6122
6844
|
image,
|
|
@@ -10801,6 +11523,14 @@ export { buildNpmInvocation };
|
|
|
10801
11523
|
export { resolveAgenticConfig, LOCAL_AGENTIC_TOKEN };
|
|
10802
11524
|
export { resolveAgenticSetting, PROTOCOLS, PERMISSION_MODES };
|
|
10803
11525
|
export { resolveAgenticTarget, discoverAgenticHubs, probeAgenticChannel, normalizeProjectApps, isLoopbackHost };
|
|
11526
|
+
export {
|
|
11527
|
+
orderProbeAddresses,
|
|
11528
|
+
resolveProbeCandidates,
|
|
11529
|
+
raceProbeCandidates,
|
|
11530
|
+
isLinkLocalAddress,
|
|
11531
|
+
rediscoverAgenticUntilConnected,
|
|
11532
|
+
defaultAgenticRediscoveryDelays,
|
|
11533
|
+
};
|
|
10804
11534
|
export { compareSemver, githubRepoSlug, filterReleasesSince, renderReleaseBody };
|
|
10805
11535
|
export {
|
|
10806
11536
|
webConsoleUrl,
|
|
@@ -10811,6 +11541,8 @@ export {
|
|
|
10811
11541
|
export {
|
|
10812
11542
|
normalizeTaskEnvelope,
|
|
10813
11543
|
collectEnvelopeFrom,
|
|
11544
|
+
classifyRepoEnvelope,
|
|
11545
|
+
isPlausibleRepoUrl,
|
|
10814
11546
|
parseLinkedResources,
|
|
10815
11547
|
pickLinkedResource,
|
|
10816
11548
|
resolveBrokerRestConfig,
|
|
@@ -10847,6 +11579,10 @@ export {
|
|
|
10847
11579
|
runAgentJob,
|
|
10848
11580
|
spawnCapturePty,
|
|
10849
11581
|
spawnCaptureAcp,
|
|
11582
|
+
sampleSubtreeCpu,
|
|
11583
|
+
createIdleLivenessMonitor,
|
|
11584
|
+
resolveLivenessOverrides,
|
|
11585
|
+
parsePsTime,
|
|
10850
11586
|
ensureAcpFlag,
|
|
10851
11587
|
startLockExtender,
|
|
10852
11588
|
provisionRepo,
|