@adhdev/daemon-core 0.9.82-rc.351 → 0.9.82-rc.352
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/upgrade-helper.d.ts +1 -1
- package/dist/index.js +424 -189
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +424 -189
- package/dist/index.mjs.map +1 -1
- package/dist/logging/logger.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-runtime-store.d.ts +9 -0
- package/dist/mesh/mesh-work-queue.d.ts +28 -0
- package/dist/providers/approval-utils.d.ts +7 -0
- package/dist/providers/cli-provider-instance.d.ts +15 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +1 -0
- package/package.json +2 -2
- package/src/commands/upgrade-helper.ts +57 -14
- package/src/logging/command-log.ts +7 -5
- package/src/logging/logger.ts +12 -6
- package/src/mesh/mesh-events-coordinator.ts +165 -84
- package/src/mesh/mesh-events-pending.ts +14 -15
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-reconcile-loop.ts +67 -7
- package/src/mesh/mesh-runtime-store.ts +17 -0
- package/src/mesh/mesh-work-queue.ts +89 -0
- package/src/providers/approval-utils.d.ts +1 -0
- package/src/providers/approval-utils.ts +10 -0
- package/src/providers/cli-provider-instance.ts +73 -19
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +51 -0
package/dist/logging/logger.d.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* 1. daemonLog(category, msg, level) — explicit per-category logging
|
|
8
8
|
* 2. installGlobalInterceptor() — Auto-intercept console.log (once on daemon start)
|
|
9
9
|
* 3. Recent log ring buffer — for remote transmission via P2P/WS
|
|
10
|
-
* 4. File logging —
|
|
10
|
+
* 4. File logging — ~/.adhdev/logs/daemon-YYYY-MM-DD.log (date-based rolling)
|
|
11
11
|
*
|
|
12
12
|
* use:
|
|
13
13
|
* import { daemonLog, LOG } from './daemon-logger';
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { EventEmitter } from 'events';
|
|
16
16
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
17
|
-
export type MeshLedgerKind = 'task_dispatched' | 'task_completed' | 'task_failed' | 'task_stalled' | 'task_approval_needed' | 'p2p_dispatch_failed' | 'session_launched' | 'session_auto_launch' | 'session_stopped' | 'checkpoint_created' | 'node_cloned' | 'node_joined' | 'node_removed' | 'coordinator_started' | 'recovery_attempted' | 'ledger_replicated' | 'ledger_reconciled' | 'direct_fast_forward' | 'delivery_unroutable' | 'direct_dispatch_pruned' | 'event_held';
|
|
17
|
+
export type MeshLedgerKind = 'task_dispatched' | 'task_completed' | 'task_failed' | 'task_stalled' | 'task_approval_needed' | 'p2p_dispatch_failed' | 'session_launched' | 'session_auto_launch' | 'session_stopped' | 'checkpoint_created' | 'node_cloned' | 'node_joined' | 'node_removed' | 'coordinator_started' | 'recovery_attempted' | 'ledger_replicated' | 'ledger_reconciled' | 'direct_fast_forward' | 'delivery_unroutable' | 'direct_dispatch_pruned' | 'event_held' | 'task_reclaimed';
|
|
18
18
|
export interface MeshLedgerEntry {
|
|
19
19
|
id: string;
|
|
20
20
|
meshId: string;
|
|
@@ -191,6 +191,15 @@ export declare class MeshRuntimeStore {
|
|
|
191
191
|
createdAt: string;
|
|
192
192
|
updatedAt: string;
|
|
193
193
|
}>;
|
|
194
|
+
/**
|
|
195
|
+
* Bug B watchdog support: true when at least one delivery record for the task has
|
|
196
|
+
* reached a confirmed-handed-off status (delivered / acked / completed). The
|
|
197
|
+
* assigned-stranded watchdog uses this to distinguish a dispatch that was never
|
|
198
|
+
* confirmed (reclaimable) from one that WAS handed to the worker (a genuinely
|
|
199
|
+
* in-flight or completion-lost task, which is PHASE 4's responsibility, not this
|
|
200
|
+
* watchdog's). Indexed by (mesh_id, task_id).
|
|
201
|
+
*/
|
|
202
|
+
taskHasConfirmedDelivery(meshId: string, taskId: string): boolean;
|
|
194
203
|
expireStaleSessionDeliveries(meshId: string): void;
|
|
195
204
|
deleteSessionDeliveries(meshId: string): void;
|
|
196
205
|
recordCompletionConflict(entry: {
|
|
@@ -60,6 +60,14 @@ export interface MeshWorkQueueEntry {
|
|
|
60
60
|
requeueCount?: number;
|
|
61
61
|
/** Max automatic requeue attempts. When requeueCount reaches this, task is auto-failed. */
|
|
62
62
|
maxRetries?: number;
|
|
63
|
+
/**
|
|
64
|
+
* Bug B: number of times the reconcile assigned-stranded watchdog has reclaimed this
|
|
65
|
+
* row from 'assigned' back to 'pending' because its dispatch was never confirmed
|
|
66
|
+
* delivered. Separate from requeueCount (operator/execution retries) and bounded by
|
|
67
|
+
* MAX_STRANDED_RECLAIMS so a permanently-undeliverable target auto-fails rather than
|
|
68
|
+
* cycling reclaim→re-dispatch→strand forever.
|
|
69
|
+
*/
|
|
70
|
+
strandedReclaimCount?: number;
|
|
63
71
|
/** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
|
|
64
72
|
autoLaunch?: {
|
|
65
73
|
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
@@ -230,6 +238,26 @@ export declare function requeueTask(meshId: string, taskId: string, opts?: {
|
|
|
230
238
|
/** Per-task retry cap override. Falls back to mesh policy maxTaskRetries (default 1). */
|
|
231
239
|
maxRetries?: number;
|
|
232
240
|
} & MeshQueueMutationOptions): MeshWorkQueueEntry | null;
|
|
241
|
+
/**
|
|
242
|
+
* Bug B: reclaim a task stuck in 'assigned' because its dispatch was never confirmed.
|
|
243
|
+
*
|
|
244
|
+
* claimNextTask atomically marks a row 'assigned' BEFORE the fire-and-forget dispatch
|
|
245
|
+
* runs. If that dispatch neither rejects (→ no .catch requeue) nor is confirmed
|
|
246
|
+
* delivered — a relay that hangs without acking, or a confirm timer lost across a
|
|
247
|
+
* daemon restart — the row stays 'assigned' forever, contributing 0 pending so PHASE 3
|
|
248
|
+
* reconcile never re-examines it. This returns such a row to 'pending' and clears its
|
|
249
|
+
* dead assignment ownership (node / session / provider / dispatchTimestamp) — the same
|
|
250
|
+
* ownership-clear requeueTask applies — so PHASE 3 can re-dispatch it onto a fresh idle
|
|
251
|
+
* session.
|
|
252
|
+
*
|
|
253
|
+
* Guarded to 'assigned' rows only (a completion/cancel that already moved the row off
|
|
254
|
+
* 'assigned' must never be resurrected) and bounded by MAX_STRANDED_RECLAIMS (beyond
|
|
255
|
+
* which the task is failed so dependents unblock).
|
|
256
|
+
*/
|
|
257
|
+
export declare function reclaimStrandedAssignedTask(meshId: string, taskId: string, opts?: {
|
|
258
|
+
reason?: string;
|
|
259
|
+
ageMs?: number;
|
|
260
|
+
} & MeshQueueMutationOptions): MeshWorkQueueEntry | null;
|
|
233
261
|
/**
|
|
234
262
|
* Update the status of the task currently assigned to a specific session.
|
|
235
263
|
*/
|
|
@@ -1,4 +1,11 @@
|
|
|
1
1
|
import type { ProviderModule } from './contracts.js';
|
|
2
|
+
/**
|
|
3
|
+
* True when any of the given button labels reads as a decline/negative option
|
|
4
|
+
* (No / Deny / Cancel / Skip / …). Used as the second half of an approval-modal
|
|
5
|
+
* structural anchor: a real approval modal offers BOTH an affirmative and a
|
|
6
|
+
* decline, which distinguishes it from a generic numbered menu or prose list.
|
|
7
|
+
*/
|
|
8
|
+
export declare function hasNegativeApprovalOption(buttons: string[] | null | undefined): boolean;
|
|
2
9
|
export declare function getApprovalPositiveHints(provider?: Pick<ProviderModule, 'approvalPositiveHints'> | null): string[];
|
|
3
10
|
export declare function pickApprovalButton(buttons: string[] | null | undefined, provider?: Pick<ProviderModule, 'approvalPositiveHints'> | null): {
|
|
4
11
|
index: number;
|
|
@@ -48,6 +48,20 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
48
48
|
* keystroke until the modal *content* has settled.
|
|
49
49
|
*/
|
|
50
50
|
private static readonly AUTO_APPROVE_SETTLE_MS;
|
|
51
|
+
/**
|
|
52
|
+
* Busy-side hysteresis for the settle gate. A momentary `generating` flip
|
|
53
|
+
* while the SAME approval modal's button block is still on screen (its
|
|
54
|
+
* question line scrolled out of the captured frame, only the buttons + a
|
|
55
|
+
* residual `esc to interrupt` spinner remain) briefly reports
|
|
56
|
+
* status!=waiting_approval. Without hysteresis that flip wipes the settle
|
|
57
|
+
* clock, and the modal→generating→modal flap restarts the 600ms window
|
|
58
|
+
* every time so auto-approve never fires. We keep the in-progress settle
|
|
59
|
+
* gate warm across an inactive blip up to this bound; only once the modal
|
|
60
|
+
* has genuinely stayed gone this long (a real resolution → idle) is the
|
|
61
|
+
* gate cleared. Bounded so a genuinely new, later approval still re-settles
|
|
62
|
+
* from scratch rather than firing on a stale timestamp.
|
|
63
|
+
*/
|
|
64
|
+
private static readonly AUTO_APPROVE_GATE_HYSTERESIS_MS;
|
|
51
65
|
private adapter;
|
|
52
66
|
private context;
|
|
53
67
|
private events;
|
|
@@ -64,6 +78,7 @@ export declare class CliProviderInstance implements ProviderInstance {
|
|
|
64
78
|
private pendingAutoApprovalSignature;
|
|
65
79
|
private pendingAutoApprovalSince;
|
|
66
80
|
private autoApproveSettleTimer;
|
|
81
|
+
private autoApproveInactiveSince;
|
|
67
82
|
private controlValues;
|
|
68
83
|
private summaryMetadata;
|
|
69
84
|
private appliedEffectKeys;
|
|
@@ -57,6 +57,7 @@ interface ModalSpec {
|
|
|
57
57
|
}>;
|
|
58
58
|
buttonPattern: string;
|
|
59
59
|
buttonFlags?: string;
|
|
60
|
+
buttonLabelGroup?: number;
|
|
60
61
|
}
|
|
61
62
|
export type DispatchGroup = 'spinner' | 'modal' | 'settled-prompt' | 'cue-ordering' | 'error-detection' | 'approval-stitching';
|
|
62
63
|
export interface DispatchOrderSpec {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adhdev/daemon-core",
|
|
3
|
-
"version": "0.9.82-rc.
|
|
3
|
+
"version": "0.9.82-rc.352",
|
|
4
4
|
"description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
"author": "vilmire",
|
|
47
47
|
"license": "AGPL-3.0-or-later",
|
|
48
48
|
"dependencies": {
|
|
49
|
-
"@adhdev/mesh-shared": "0.9.82-rc.
|
|
49
|
+
"@adhdev/mesh-shared": "0.9.82-rc.352",
|
|
50
50
|
"@adhdev/session-host-core": "*",
|
|
51
51
|
"@agentclientprotocol/sdk": "^0.16.1",
|
|
52
52
|
"ajv": "^8.20.0",
|
|
@@ -292,13 +292,14 @@ async function waitForPidExit(pid: number, timeoutMs: number): Promise<void> {
|
|
|
292
292
|
}
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
-
export function stopSessionHostProcesses(appName: string): void {
|
|
295
|
+
export async function stopSessionHostProcesses(appName: string): Promise<void> {
|
|
296
296
|
const pidFile = path.join(os.homedir(), '.adhdev', `${appName}-session-host.pid`);
|
|
297
|
+
let killedPid: number | null = null;
|
|
297
298
|
try {
|
|
298
299
|
if (fs.existsSync(pidFile)) {
|
|
299
300
|
const pid = Number.parseInt(fs.readFileSync(pidFile, 'utf8').trim(), 10);
|
|
300
301
|
if (Number.isFinite(pid) && pid !== process.pid && isManagedSessionHostPid(pid)) {
|
|
301
|
-
killPid(pid);
|
|
302
|
+
if (killPid(pid)) killedPid = pid;
|
|
302
303
|
}
|
|
303
304
|
}
|
|
304
305
|
} catch {
|
|
@@ -310,6 +311,31 @@ export function stopSessionHostProcesses(appName: string): void {
|
|
|
310
311
|
// noop
|
|
311
312
|
}
|
|
312
313
|
}
|
|
314
|
+
|
|
315
|
+
// The session-host process keeps node-pty's `conpty.node` memory-mapped. On
|
|
316
|
+
// Windows a mapped native addon stays EXCLUSIVELY locked until the process
|
|
317
|
+
// fully exits and tears down the mapping — and that teardown lags `taskkill`
|
|
318
|
+
// by an indeterminate interval. `taskkill` only *requests* termination, so
|
|
319
|
+
// returning immediately lets the caller run `npm install` while conpty.node
|
|
320
|
+
// is still locked, which makes npm's copy-to-staging fail with EBUSY (the
|
|
321
|
+
// intermittent Windows upgrade failure). Wait for the killed process to
|
|
322
|
+
// actually disappear — like we already do for the parent daemon pid — so the
|
|
323
|
+
// file handle is released before the install runs. (POSIX can replace an open
|
|
324
|
+
// file freely, so the wait is harmless there.)
|
|
325
|
+
if (killedPid !== null) {
|
|
326
|
+
await waitForPidExit(killedPid, 15000);
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// npm copies the current install's files into a staging dir before swapping in
|
|
331
|
+
// the new version. On Windows that copy of `conpty.node` can still race a
|
|
332
|
+
// just-killed session-host whose mapping hasn't been released yet, surfacing as
|
|
333
|
+
// EBUSY/EPERM. Treat those as transient and retry with backoff.
|
|
334
|
+
function isRetriableInstallLockError(error: any): boolean {
|
|
335
|
+
const code = error?.code;
|
|
336
|
+
if (code === 'EBUSY' || code === 'EPERM') return true;
|
|
337
|
+
const text = `${error?.message || ''} ${error?.stderr || ''}`;
|
|
338
|
+
return /\bEBUSY\b|\bEPERM\b|resource busy or locked/i.test(text);
|
|
313
339
|
}
|
|
314
340
|
|
|
315
341
|
function removeDaemonPidFile(): void {
|
|
@@ -412,23 +438,40 @@ async function runDaemonUpgradeHelper(payload: DaemonUpgradeHelperPayload): Prom
|
|
|
412
438
|
await waitForPidExit(payload.parentPid, 15000);
|
|
413
439
|
}
|
|
414
440
|
|
|
415
|
-
stopSessionHostProcesses(sessionHostAppName);
|
|
441
|
+
await stopSessionHostProcesses(sessionHostAppName);
|
|
416
442
|
removeDaemonPidFile();
|
|
417
443
|
cleanupStaleGlobalInstallDirs(payload.packageName, installCommand.surface);
|
|
418
444
|
|
|
419
445
|
const spec = `${payload.packageName}@${payload.targetVersion || 'latest'}`;
|
|
420
446
|
appendUpgradeLog(`Installing ${spec}`);
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
447
|
+
// Windows can still race a lingering conpty.node mapping even after the
|
|
448
|
+
// session-host exits, so retry the install on transient lock errors there.
|
|
449
|
+
const maxInstallAttempts = process.platform === 'win32' ? 3 : 1;
|
|
450
|
+
let installOutput = '';
|
|
451
|
+
for (let attempt = 1; attempt <= maxInstallAttempts; attempt++) {
|
|
452
|
+
try {
|
|
453
|
+
installOutput = String(execFileSync(
|
|
454
|
+
installCommand.command,
|
|
455
|
+
installCommand.args,
|
|
456
|
+
{
|
|
457
|
+
encoding: 'utf8',
|
|
458
|
+
stdio: 'pipe',
|
|
459
|
+
maxBuffer: 20 * 1024 * 1024,
|
|
460
|
+
env: buildInstallEnvWithNodeOnPath(),
|
|
461
|
+
...installCommand.execOptions,
|
|
462
|
+
},
|
|
463
|
+
));
|
|
464
|
+
break;
|
|
465
|
+
} catch (error: any) {
|
|
466
|
+
if (attempt < maxInstallAttempts && isRetriableInstallLockError(error)) {
|
|
467
|
+
appendUpgradeLog(`Install attempt ${attempt} hit a file lock (${error?.code || 'lock'}); cleaning staging and retrying after backoff`);
|
|
468
|
+
cleanupStaleGlobalInstallDirs(payload.packageName, installCommand.surface);
|
|
469
|
+
await new Promise((resolve) => setTimeout(resolve, attempt * 1500));
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
throw error;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
432
475
|
if (installOutput.trim()) {
|
|
433
476
|
appendUpgradeLog(installOutput.trim());
|
|
434
477
|
}
|
|
@@ -15,11 +15,13 @@ import * as path from 'path';
|
|
|
15
15
|
import * as os from 'os';
|
|
16
16
|
|
|
17
17
|
// ─── Config ──────────────────────────────────
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
// Command history lives under the unified ADHDev home (~/.adhdev/logs/) next to
|
|
19
|
+
// the daemon log, on every platform. Honor ADHDEV_CONFIG_DIR for isolated homes.
|
|
20
|
+
// Keep this in sync with logger.ts LOG_DIR.
|
|
21
|
+
const ADHDEV_HOME = process.env.ADHDEV_CONFIG_DIR && process.env.ADHDEV_CONFIG_DIR.trim()
|
|
22
|
+
? process.env.ADHDEV_CONFIG_DIR.trim()
|
|
23
|
+
: path.join(os.homedir(), '.adhdev');
|
|
24
|
+
const LOG_DIR = path.join(ADHDEV_HOME, 'logs');
|
|
23
25
|
|
|
24
26
|
const MAX_FILE_SIZE = 5 * 1024 * 1024; // 5MB
|
|
25
27
|
const MAX_DAYS = 7;
|
package/src/logging/logger.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* 1. daemonLog(category, msg, level) — explicit per-category logging
|
|
8
8
|
* 2. installGlobalInterceptor() — Auto-intercept console.log (once on daemon start)
|
|
9
9
|
* 3. Recent log ring buffer — for remote transmission via P2P/WS
|
|
10
|
-
* 4. File logging —
|
|
10
|
+
* 4. File logging — ~/.adhdev/logs/daemon-YYYY-MM-DD.log (date-based rolling)
|
|
11
11
|
*
|
|
12
12
|
* use:
|
|
13
13
|
* import { daemonLog, LOG } from './daemon-logger';
|
|
@@ -37,11 +37,17 @@ export function setLogLevel(level: LogLevel): void {
|
|
|
37
37
|
|
|
38
38
|
export function getLogLevel(): LogLevel { return currentLevel; }
|
|
39
39
|
// ─── File logging (date-based rolling) ──────────────────────────────
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
40
|
+
// Logs live under the unified ADHDev home (~/.adhdev/logs/) on every platform,
|
|
41
|
+
// alongside config.json, providers/, history/, daemon.pid and session-host.log.
|
|
42
|
+
// Earlier builds wrote to OS-specific dirs (~/Library/Logs/adhdev on macOS,
|
|
43
|
+
// ~/.local/share/adhdev/logs on Linux, %LOCALAPPDATA%/adhdev/logs on Windows),
|
|
44
|
+
// which made the daemon log undiscoverable next to everything else under
|
|
45
|
+
// ~/.adhdev and inconsistent with session-host.log. Honor ADHDEV_CONFIG_DIR so
|
|
46
|
+
// isolated/standalone namespaces keep their logs in their own home.
|
|
47
|
+
const ADHDEV_HOME = process.env.ADHDEV_CONFIG_DIR && process.env.ADHDEV_CONFIG_DIR.trim()
|
|
48
|
+
? process.env.ADHDEV_CONFIG_DIR.trim()
|
|
49
|
+
: path.join(os.homedir(), '.adhdev');
|
|
50
|
+
const LOG_DIR = path.join(ADHDEV_HOME, 'logs');
|
|
45
51
|
|
|
46
52
|
const MAX_LOG_SIZE = 5 * 1024 * 1024; // 5MB per day
|
|
47
53
|
const MAX_LOG_DAYS = 7; // 7-day retention
|
|
@@ -7,6 +7,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
7
7
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
8
8
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
9
9
|
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
|
|
10
|
+
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
10
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
12
|
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
@@ -17,7 +18,7 @@ import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUn
|
|
|
17
18
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
18
19
|
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
19
20
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
20
|
-
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
22
|
import {
|
|
22
23
|
findRecentTerminalLedgerEvidence,
|
|
23
24
|
hasDispatchAfterTerminal,
|
|
@@ -35,16 +36,16 @@ import {
|
|
|
35
36
|
} from './mesh-events-utils.js';
|
|
36
37
|
|
|
37
38
|
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
38
|
-
// pending-events queue
|
|
39
|
-
//
|
|
40
|
-
//
|
|
39
|
+
// pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
|
|
40
|
+
// a unicast event may be stamped with the status id, the bare machineId, OR the
|
|
41
|
+
// config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
|
|
42
|
+
// created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
|
|
43
|
+
// completion matches a coordinator that knows itself as bare `<machineId>` (the
|
|
44
|
+
// base-node completion-surface bug) and vice versa.
|
|
41
45
|
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
42
|
-
const ids = new Set<string>();
|
|
43
46
|
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
44
|
-
if (statusInstanceId) ids.add(statusInstanceId);
|
|
45
47
|
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
46
|
-
|
|
47
|
-
return [...ids];
|
|
48
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
// ---------------------------------------------------------------------------
|
|
@@ -304,6 +305,95 @@ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): s
|
|
|
304
305
|
// Queue assignment
|
|
305
306
|
// ---------------------------------------------------------------------------
|
|
306
307
|
|
|
308
|
+
// Per-dispatch confirmation timeout (Bug B). A dispatch promise that never settles —
|
|
309
|
+
// a saturated remote P2P relay that hangs, or a transport that resolves only after
|
|
310
|
+
// the worker acks — would otherwise leave the just-claimed queue row 'assigned' with
|
|
311
|
+
// its delivery stuck 'delivering' forever: the .catch that requeues never fires, and
|
|
312
|
+
// PHASE 3 reconcile skips the row (it counts 0 pending). Racing the dispatch against
|
|
313
|
+
// this timeout guarantees a hung dispatch deterministically returns the task to
|
|
314
|
+
// 'pending' for re-dispatch. Generous so a merely-slow-but-live dispatch (a cold
|
|
315
|
+
// remote relay) is never reclaimed early; the reconcile assigned-stranded watchdog is
|
|
316
|
+
// the durable cross-restart backstop for a timer lost to a daemon restart.
|
|
317
|
+
const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
|
|
318
|
+
|
|
319
|
+
interface DeliverTaskContext {
|
|
320
|
+
meshId: string;
|
|
321
|
+
nodeId: string;
|
|
322
|
+
sessionId: string;
|
|
323
|
+
providerType: string;
|
|
324
|
+
task: MeshWorkQueueEntry;
|
|
325
|
+
transport: 'remote' | 'local';
|
|
326
|
+
sourceCoordinatorSessionId?: string;
|
|
327
|
+
sourceCoordinatorDaemonId?: string;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// CONS scope 3: the SINGLE source of truth for dispatching a claimed task to its
|
|
331
|
+
// session. The remote (P2P dispatchMeshCommand) and local (cliManager.handleCliCommand)
|
|
332
|
+
// branches differ ONLY in the transport call — the delivery record, the delivered/failed
|
|
333
|
+
// transitions, the pending-requeue-on-failure, the dispatch_failed ledger entry, AND the
|
|
334
|
+
// Bug B hang timeout are identical and live here once so a future change to the dispatch
|
|
335
|
+
// lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
|
|
336
|
+
// performs only the transport-specific send and returns its promise.
|
|
337
|
+
function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: DeliverTaskContext): void {
|
|
338
|
+
const delivery = createSessionDelivery({
|
|
339
|
+
meshId: ctx.meshId,
|
|
340
|
+
nodeId: ctx.nodeId,
|
|
341
|
+
sessionId: ctx.sessionId,
|
|
342
|
+
providerType: ctx.providerType,
|
|
343
|
+
taskId: ctx.task.id,
|
|
344
|
+
kind: 'task',
|
|
345
|
+
message: ctx.task.message,
|
|
346
|
+
status: 'delivering',
|
|
347
|
+
...(ctx.sourceCoordinatorSessionId ? { sourceCoordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
|
|
348
|
+
...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
// Invoke the transport synchronously (preserves the prior fire-and-forget timing,
|
|
352
|
+
// and lets a synchronous throw fall into the same failure path as a rejection).
|
|
353
|
+
let dispatchPromise: Promise<unknown>;
|
|
354
|
+
try {
|
|
355
|
+
dispatchPromise = Promise.resolve(dispatchThunk());
|
|
356
|
+
} catch (e) {
|
|
357
|
+
dispatchPromise = Promise.reject(e);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
361
|
+
const guarded = Promise.race([
|
|
362
|
+
dispatchPromise,
|
|
363
|
+
new Promise<never>((_, reject) => {
|
|
364
|
+
timer = setTimeout(
|
|
365
|
+
() => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
|
|
366
|
+
DISPATCH_CONFIRM_TIMEOUT_MS,
|
|
367
|
+
);
|
|
368
|
+
// Never keep the process alive solely for this confirm-timeout timer.
|
|
369
|
+
if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
|
|
370
|
+
}),
|
|
371
|
+
]);
|
|
372
|
+
|
|
373
|
+
guarded.then(() => {
|
|
374
|
+
if (timer) clearTimeout(timer);
|
|
375
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
376
|
+
}).catch((e: any) => {
|
|
377
|
+
if (timer) clearTimeout(timer);
|
|
378
|
+
// A dispatch failure (transport reject OR hang timeout) is most often transient —
|
|
379
|
+
// a busy/refusing adapter, or a relay that never acked — not a permanent task
|
|
380
|
+
// failure. Marking the task terminal here would permanently kill tasks a later
|
|
381
|
+
// tick delivers fine. Return it to 'pending' and record a retryable dispatch_failed
|
|
382
|
+
// ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
|
|
383
|
+
LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
|
|
384
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
385
|
+
updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
|
|
386
|
+
try {
|
|
387
|
+
appendLedgerEntry(ctx.meshId, {
|
|
388
|
+
kind: 'dispatch_failed' as any,
|
|
389
|
+
nodeId: ctx.nodeId,
|
|
390
|
+
sessionId: ctx.sessionId,
|
|
391
|
+
payload: { taskId: ctx.task.id, deliveryId: delivery.id, error: e?.message, retryable: true, transport: ctx.transport },
|
|
392
|
+
});
|
|
393
|
+
} catch { /* ledger write is best-effort */ }
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
|
|
307
397
|
export function tryAssignQueueTask(
|
|
308
398
|
components: DaemonComponents,
|
|
309
399
|
meshId: string,
|
|
@@ -337,45 +427,36 @@ export function tryAssignQueueTask(
|
|
|
337
427
|
// completion back to that exact session (multi-coordinator). Carried over P2P
|
|
338
428
|
// to the remote worker, which echoes it on its completion event.
|
|
339
429
|
const sourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId) || undefined;
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
430
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
431
|
+
const remoteDaemonId = node.daemonId;
|
|
432
|
+
// CONS3: only the transport call differs — everything else (delivery record,
|
|
433
|
+
// status transitions, requeue-on-failure, ledger, Bug B hang timeout) is in
|
|
434
|
+
// the shared deliverTaskToSession helper.
|
|
435
|
+
deliverTaskToSession(
|
|
436
|
+
() => dispatchMeshCommand(remoteDaemonId, 'agent_command', {
|
|
437
|
+
targetSessionId: sessionId,
|
|
438
|
+
cliType: providerType,
|
|
439
|
+
action: 'send_chat',
|
|
440
|
+
message: task.message,
|
|
441
|
+
meshContext: {
|
|
442
|
+
meshId,
|
|
443
|
+
nodeId,
|
|
444
|
+
taskId: task.id,
|
|
445
|
+
...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
446
|
+
...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
|
|
447
|
+
},
|
|
448
|
+
}),
|
|
449
|
+
{
|
|
358
450
|
meshId,
|
|
359
451
|
nodeId,
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
452
|
+
sessionId,
|
|
453
|
+
providerType,
|
|
454
|
+
task,
|
|
455
|
+
transport: 'remote',
|
|
456
|
+
...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
|
|
457
|
+
...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
363
458
|
},
|
|
364
|
-
|
|
365
|
-
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
366
|
-
}).catch((e: any) => {
|
|
367
|
-
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
368
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
369
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
370
|
-
try {
|
|
371
|
-
appendLedgerEntry(meshId, {
|
|
372
|
-
kind: 'dispatch_failed' as any,
|
|
373
|
-
nodeId,
|
|
374
|
-
sessionId,
|
|
375
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
376
|
-
});
|
|
377
|
-
} catch { /* ledger write is best-effort */ }
|
|
378
|
-
});
|
|
459
|
+
);
|
|
379
460
|
return true;
|
|
380
461
|
}
|
|
381
462
|
}
|
|
@@ -412,44 +493,26 @@ export function tryAssignQueueTask(
|
|
|
412
493
|
}
|
|
413
494
|
} catch { /* best-effort — dispatch still proceeds */ }
|
|
414
495
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
// Mirror the remote-dispatch catch above: a local dispatch failure is most often a
|
|
436
|
-
// transient busy/refusal (e.g. the adapter rejected send_chat while mid-generation),
|
|
437
|
-
// not a permanent task failure. Marking the task terminal 'failed' here with no ledger
|
|
438
|
-
// and no retry permanently killed tasks that a later tick would have delivered fine.
|
|
439
|
-
// Return the task to 'pending' and record a retryable dispatch_failed ledger entry so
|
|
440
|
-
// the reconcile loop re-dispatches it, exactly as the remote branch does.
|
|
441
|
-
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
442
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
443
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
444
|
-
try {
|
|
445
|
-
appendLedgerEntry(meshId, {
|
|
446
|
-
kind: 'dispatch_failed' as any,
|
|
447
|
-
nodeId,
|
|
448
|
-
sessionId,
|
|
449
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
450
|
-
});
|
|
451
|
-
} catch { /* ledger write is best-effort */ }
|
|
452
|
-
});
|
|
496
|
+
// CONS3: same shared dispatch lifecycle as the remote branch — only the transport
|
|
497
|
+
// (cliManager.handleCliCommand) differs.
|
|
498
|
+
deliverTaskToSession(
|
|
499
|
+
() => components.cliManager.handleCliCommand('agent_command', {
|
|
500
|
+
targetSessionId: sessionId,
|
|
501
|
+
cliType: providerType,
|
|
502
|
+
action: 'send_chat',
|
|
503
|
+
message: task.message,
|
|
504
|
+
}),
|
|
505
|
+
{
|
|
506
|
+
meshId,
|
|
507
|
+
nodeId,
|
|
508
|
+
sessionId,
|
|
509
|
+
providerType,
|
|
510
|
+
task,
|
|
511
|
+
transport: 'local',
|
|
512
|
+
...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
|
|
513
|
+
...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
|
|
514
|
+
},
|
|
515
|
+
);
|
|
453
516
|
|
|
454
517
|
return true;
|
|
455
518
|
}
|
|
@@ -929,7 +992,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
929
992
|
|
|
930
993
|
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
931
994
|
? mesh.nodes.filter((node: any) => {
|
|
932
|
-
|
|
995
|
+
// Bug A: match the target pin with the shared 3-form (id / nodeId / node_id)
|
|
996
|
+
// normalizer, mirroring the remote-idle drain (meshNodeIdMatches at the
|
|
997
|
+
// getRemoteIdleSessions filter). A strict `readMeshNodeId(node) !== targetNodeId`
|
|
998
|
+
// dropped a target node whose identity arrived under a different form (a freshly
|
|
999
|
+
// mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
|
|
1000
|
+
if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
|
|
933
1001
|
// Skip nodes that can never satisfy requiredTags regardless of which provider
|
|
934
1002
|
// from providerPriority is selected. A node satisfies tags if at least one
|
|
935
1003
|
// provider in its priority list would produce matching capability tags.
|
|
@@ -944,7 +1012,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
944
1012
|
})
|
|
945
1013
|
: [];
|
|
946
1014
|
if (!candidateNodes.length) {
|
|
947
|
-
|
|
1015
|
+
// Bug A: distinguish the two ways the candidate set empties. A task pinned to a
|
|
1016
|
+
// targetNodeId whose node is absent from the mesh (or whose id arrived under a
|
|
1017
|
+
// different form) is a ROUTING miss — report it as `target_node_id_unmatched`, not
|
|
1018
|
+
// the hard-coded `no_node_satisfies_required_tags`, which mislabelled a 3-form
|
|
1019
|
+
// node-id mismatch as a capability failure and sent diagnosis down the wrong path.
|
|
1020
|
+
// Only fall back to the tag reason when no target pin is in play, or the pin DID
|
|
1021
|
+
// match a node but its tags excluded it (a genuine capability miss).
|
|
1022
|
+
const targetPinUnmatched = !!task.targetNodeId
|
|
1023
|
+
&& !(Array.isArray(mesh?.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, task.targetNodeId)));
|
|
1024
|
+
markAutoLaunch(meshId, task.id, {
|
|
1025
|
+
status: 'skipped',
|
|
1026
|
+
reason: targetPinUnmatched ? 'target_node_id_unmatched' : 'no_node_satisfies_required_tags',
|
|
1027
|
+
nodeId: task.targetNodeId,
|
|
1028
|
+
});
|
|
948
1029
|
continue;
|
|
949
1030
|
}
|
|
950
1031
|
|