@mindexec/cli 0.2.396 → 0.2.398
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/remote-hub.js +17 -5
- package/scripts/remote-agent-managed-smoke.mjs +23 -0
- package/scripts/remote-hub-smoke.mjs +1 -1
- package/scripts/remote-registry-follower-smoke.mjs +31 -0
- package/server.js +327 -15
- package/wwwroot/_content/MindExecution.Shared/css/mind-map-overrides.css +30 -5
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-core.js +24 -21
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-css3d-manager.js +120 -12
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-interactions.js +32 -5
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-nodes.js +11 -5
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-text-overlay-v2.js +61 -1
- package/wwwroot/_content/MindExecution.Shared/js/mind-map-texture-factory.js +6 -1
- package/wwwroot/index.html +4 -5
- package/wwwroot/service-worker-assets.js +9 -9
- package/wwwroot/service-worker.js +1 -1
package/package.json
CHANGED
package/remote-hub.js
CHANGED
|
@@ -90,7 +90,8 @@ const REMOTE_FRAME_MODE_ALIASES = new Map([
|
|
|
90
90
|
const MAX_SYNTHETIC_DEVICES = 1000;
|
|
91
91
|
const DEFAULT_HOST_TARGET_LEASE_MS = 30000;
|
|
92
92
|
const DUPLICATE_DEVICE_ACTIVE_REJECT_MS = 15000;
|
|
93
|
-
const DUPLICATE_DEVICE_LOG_THROTTLE_MS =
|
|
93
|
+
const DUPLICATE_DEVICE_LOG_THROTTLE_MS = 60000;
|
|
94
|
+
const DUPLICATE_DEVICE_RETRY_AFTER_MS = 30000;
|
|
94
95
|
const SYNTHETIC_FRAME_DATA_URL = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAADElEQVR42mP8z8AAAAMBAQDJ/pLvAAAAAElFTkSuQmCC';
|
|
95
96
|
const SYNTHETIC_FRAME_PAYLOAD = Buffer.from(SYNTHETIC_FRAME_DATA_URL.split(',')[1], 'base64');
|
|
96
97
|
const SYNTHETIC_FRAME_HASH = crypto.createHash('sha256').update(SYNTHETIC_FRAME_PAYLOAD).digest('hex').slice(0, 16);
|
|
@@ -1060,6 +1061,16 @@ export function createRemoteHub(options = {}) {
|
|
|
1060
1061
|
const pairToken = safeString(
|
|
1061
1062
|
options.pairToken || env.REMOTE_HUB_PAIR_TOKEN || env.MINDEXEC_REMOTE_PAIR_TOKEN || crypto.randomBytes(6).toString('hex'),
|
|
1062
1063
|
256);
|
|
1064
|
+
const duplicateDeviceLogThrottleMs = clampNumber(
|
|
1065
|
+
env.MINDEXEC_REMOTE_DUPLICATE_DEVICE_LOG_MS || env.REMOTE_HUB_DUPLICATE_DEVICE_LOG_MS,
|
|
1066
|
+
5000,
|
|
1067
|
+
10 * 60 * 1000,
|
|
1068
|
+
DUPLICATE_DEVICE_LOG_THROTTLE_MS);
|
|
1069
|
+
const duplicateDeviceRetryAfterMs = clampNumber(
|
|
1070
|
+
env.MINDEXEC_REMOTE_DUPLICATE_DEVICE_RETRY_MS || env.REMOTE_HUB_DUPLICATE_DEVICE_RETRY_MS,
|
|
1071
|
+
5000,
|
|
1072
|
+
10 * 60 * 1000,
|
|
1073
|
+
DUPLICATE_DEVICE_RETRY_AFTER_MS);
|
|
1063
1074
|
|
|
1064
1075
|
const devices = new Map();
|
|
1065
1076
|
const taskBatches = new Map();
|
|
@@ -1500,7 +1511,7 @@ export function createRemoteHub(options = {}) {
|
|
|
1500
1511
|
}
|
|
1501
1512
|
|
|
1502
1513
|
function rejectDuplicateActiveDevice(socket, existing, attemptedSessionId) {
|
|
1503
|
-
const retryAfterMs = Math.max(
|
|
1514
|
+
const retryAfterMs = Math.max(heartbeatMs, duplicateDeviceRetryAfterMs);
|
|
1504
1515
|
writeJsonLine(socket, {
|
|
1505
1516
|
type: 'disconnect',
|
|
1506
1517
|
reason: 'duplicate-device-active',
|
|
@@ -1512,11 +1523,12 @@ export function createRemoteHub(options = {}) {
|
|
|
1512
1523
|
existing.counters.duplicateConnectionsRejected = (existing.counters.duplicateConnectionsRejected || 0) + 1;
|
|
1513
1524
|
const nowMs = Date.now();
|
|
1514
1525
|
const lastLoggedAt = duplicateDeviceLogAt.get(existing.deviceId) || 0;
|
|
1515
|
-
if (nowMs - lastLoggedAt >=
|
|
1526
|
+
if (nowMs - lastLoggedAt >= duplicateDeviceLogThrottleMs) {
|
|
1516
1527
|
duplicateDeviceLogAt.set(existing.deviceId, nowMs);
|
|
1517
|
-
|
|
1528
|
+
logEvent(
|
|
1518
1529
|
'remote',
|
|
1519
|
-
`duplicate device connection suppressed ${existing.deviceName} (${existing.deviceId}); keeping active session ${existing.sessionId}
|
|
1530
|
+
`duplicate device connection suppressed ${existing.deviceName} (${existing.deviceId}); keeping active session ${existing.sessionId}`,
|
|
1531
|
+
'remote');
|
|
1520
1532
|
}
|
|
1521
1533
|
|
|
1522
1534
|
try {
|
|
@@ -251,6 +251,29 @@ async function main() {
|
|
|
251
251
|
assert.equal(secondAgent.managerCandidates?.[0], managerEndpoint, JSON.stringify(secondAgent.managerCandidates));
|
|
252
252
|
assert.equal(secondAgent.managerCandidates?.length, 1, JSON.stringify(secondAgent.managerCandidates));
|
|
253
253
|
|
|
254
|
+
const duplicateReport = await fetchJson(`${clientBridge.baseUrl}/api/remote/agent/sync-report`, {
|
|
255
|
+
method: 'POST',
|
|
256
|
+
body: JSON.stringify({
|
|
257
|
+
ok: false,
|
|
258
|
+
attempted: true,
|
|
259
|
+
disconnected: true,
|
|
260
|
+
authenticated: true,
|
|
261
|
+
reason: 'connect-failed',
|
|
262
|
+
error: 'duplicate-device-active',
|
|
263
|
+
targetActive: true,
|
|
264
|
+
targetEndpoint: managerEndpoint,
|
|
265
|
+
targetEndpointCandidates: [managerEndpoint, staleEndpoint],
|
|
266
|
+
targetLeaseId: LEASE_ID,
|
|
267
|
+
targetNodeId: 'remote-agent-managed-smoke-node'
|
|
268
|
+
})
|
|
269
|
+
});
|
|
270
|
+
assert.equal(duplicateReport.ok, true, JSON.stringify(duplicateReport.payload));
|
|
271
|
+
assert.equal(duplicateReport.payload?.wake?.reason, 'duplicate-device-active', JSON.stringify(duplicateReport.payload));
|
|
272
|
+
|
|
273
|
+
const duplicateState = await fetchJson(`${clientBridge.baseUrl}/api/remote/agent/status`);
|
|
274
|
+
assert.equal(duplicateState.payload?.ready, true, JSON.stringify(duplicateState.payload));
|
|
275
|
+
assert.equal(duplicateState.payload?.needsReconnect, false, JSON.stringify(duplicateState.payload));
|
|
276
|
+
|
|
254
277
|
const staleReport = await fetchJson(`${clientBridge.baseUrl}/api/remote/agent/sync-report`, {
|
|
255
278
|
method: 'POST',
|
|
256
279
|
body: JSON.stringify({
|
|
@@ -602,7 +602,7 @@ try {
|
|
|
602
602
|
), 1000);
|
|
603
603
|
assert.equal(duplicateDisconnect.reason, 'duplicate-device-active');
|
|
604
604
|
assert.equal(duplicateDisconnect.activeSessionId, oldSessionId);
|
|
605
|
-
assert.equal(duplicateDisconnect.retryAfterMs >=
|
|
605
|
+
assert.equal(duplicateDisconnect.retryAfterMs >= 30000, true);
|
|
606
606
|
await wait(100);
|
|
607
607
|
const duplicateRejectedDevice = hub.listDevices()[0];
|
|
608
608
|
assert.equal(duplicateRejectedDevice.sessionId, oldSessionId);
|
|
@@ -533,6 +533,37 @@ async function main() {
|
|
|
533
533
|
const renewStatus = await fetchJson(`${renewHost.baseUrl}/api/status`);
|
|
534
534
|
assert.equal(renewStatus.payload?.remoteHostTargetRenew?.status, 'active', JSON.stringify(renewStatus.payload?.remoteHostTargetRenew));
|
|
535
535
|
assert.equal(renewStatus.payload?.remoteHostTargetRenew?.endpoint, renewPublicEndpoint, JSON.stringify(renewStatus.payload?.remoteHostTargetRenew));
|
|
536
|
+
const currentLeaseId = currentTarget?.lease_id;
|
|
537
|
+
const currentHostInstanceId = currentTarget?.host_instance_id;
|
|
538
|
+
currentTarget = {
|
|
539
|
+
...currentTarget,
|
|
540
|
+
pair_token: 'stale-local-pair-token',
|
|
541
|
+
lease_id: 'stale-local-lease',
|
|
542
|
+
host_instance_id: 'stale-local-host-instance',
|
|
543
|
+
expires_at: new Date(Date.now() + 60_000).toISOString()
|
|
544
|
+
};
|
|
545
|
+
const staleLocalResume = await fetchJson(`${renewHost.baseUrl}/api/remote/system/resume`, {
|
|
546
|
+
method: 'POST',
|
|
547
|
+
token: BRIDGE_TOKEN,
|
|
548
|
+
body: JSON.stringify({
|
|
549
|
+
driftMs: 65000
|
|
550
|
+
})
|
|
551
|
+
});
|
|
552
|
+
assert.equal(staleLocalResume.ok, true, JSON.stringify(staleLocalResume.payload));
|
|
553
|
+
assert.equal(staleLocalResume.payload?.ok, true, JSON.stringify(staleLocalResume.payload));
|
|
554
|
+
await waitFor(() => {
|
|
555
|
+
return currentTarget?.pair_token === PAIR_TOKEN
|
|
556
|
+
&& currentTarget?.lease_id === currentLeaseId
|
|
557
|
+
&& currentTarget?.host_instance_id === currentHostInstanceId
|
|
558
|
+
? currentTarget
|
|
559
|
+
: null;
|
|
560
|
+
}, 8000, `stale local host target republish\n${renewHost.details()}`);
|
|
561
|
+
const republishedStatus = await fetchJson(`${renewHost.baseUrl}/api/status`);
|
|
562
|
+
assert.equal(republishedStatus.payload?.remoteHub?.hostTargetActive, true, JSON.stringify(republishedStatus.payload?.remoteHub));
|
|
563
|
+
assert.equal(republishedStatus.payload?.remoteHostTargetRenew?.status, 'active', JSON.stringify(republishedStatus.payload?.remoteHostTargetRenew));
|
|
564
|
+
assert.equal(republishedStatus.payload?.remoteHostTargetRenew?.reason, 'local-monitor-host-republished', JSON.stringify(republishedStatus.payload?.remoteHostTargetRenew));
|
|
565
|
+
const renewAgent = await fetchJson(`${renewHost.baseUrl}/api/remote/agent/status`);
|
|
566
|
+
assert.equal(renewAgent.payload?.running, false, JSON.stringify(renewAgent.payload));
|
|
536
567
|
|
|
537
568
|
const renewClear = await fetchJson(`${renewHost.baseUrl}/api/remote/host-target`, {
|
|
538
569
|
method: 'DELETE',
|
package/server.js
CHANGED
|
@@ -42,6 +42,8 @@ const BRIDGE_INSTANCE_ID = String(process.env.MINDEXEC_BRIDGE_INSTANCE_ID || REM
|
|
|
42
42
|
const REMOTE_HUB_PAIR_TOKEN = String(process.env.REMOTE_HUB_PAIR_TOKEN || process.env.MINDEXEC_REMOTE_PAIR_TOKEN || REMOTE_HUB_IDENTITY.pairToken || crypto.randomBytes(18).toString('hex')).trim() || crypto.randomBytes(18).toString('hex');
|
|
43
43
|
const TREE_SITTER_GRAMMAR_DIR = path.join(BRIDGE_ROOT, 'tree-sitter-grammars');
|
|
44
44
|
const VERBOSE_CODEX_TRACE = /^(1|true|yes|on)$/i.test(String(process.env.BRIDGE_VERBOSE_CODEX || ''));
|
|
45
|
+
const VERBOSE_REMOTE_HTTP_LOGS = /^(1|true|yes|on)$/i.test(String(process.env.MINDEXEC_VERBOSE_REMOTE_HTTP || process.env.BRIDGE_VERBOSE_REMOTE_HTTP || ''));
|
|
46
|
+
const VERBOSE_REMOTE_AGENT_LOGS = /^(1|true|yes|on)$/i.test(String(process.env.MINDEXEC_VERBOSE_REMOTE_AGENT || process.env.BRIDGE_VERBOSE_REMOTE_AGENT || ''));
|
|
45
47
|
const COLOR_LOGS_ENABLED = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
|
|
46
48
|
const DEFAULT_WEB_APP_ROOT = path.join(BRIDGE_ROOT, 'wwwroot');
|
|
47
49
|
|
|
@@ -331,7 +333,7 @@ function logSection(title, lines = []) {
|
|
|
331
333
|
}
|
|
332
334
|
}
|
|
333
335
|
|
|
334
|
-
function logHttpRequest(method, requestPath) {
|
|
336
|
+
function logHttpRequest(method, requestPath) {
|
|
335
337
|
let methodTone = 'bridge';
|
|
336
338
|
switch (String(method || '').toUpperCase()) {
|
|
337
339
|
case 'POST':
|
|
@@ -353,10 +355,32 @@ function logHttpRequest(method, requestPath) {
|
|
|
353
355
|
}
|
|
354
356
|
|
|
355
357
|
const methodLabel = tone(String(method || 'REQ').padEnd(6, ' '), methodTone);
|
|
356
|
-
console.log(`${formatLogTime()} ${formatScope('http', 'muted')} ${methodLabel} ${requestPath}`);
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
function
|
|
358
|
+
console.log(`${formatLogTime()} ${formatScope('http', 'muted')} ${methodLabel} ${requestPath}`);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
function shouldLogHttpRequest(method, requestPath) {
|
|
362
|
+
if (VERBOSE_REMOTE_HTTP_LOGS) {
|
|
363
|
+
return true;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const normalizedPath = String(requestPath || '').toLowerCase();
|
|
367
|
+
if (normalizedPath === '/api/remote/status'
|
|
368
|
+
|| normalizedPath === '/api/remote/devices'
|
|
369
|
+
|| normalizedPath === '/api/remote/frames'
|
|
370
|
+
|| normalizedPath === '/api/remote/agent/status'
|
|
371
|
+
|| normalizedPath === '/api/remote/agent/connect'
|
|
372
|
+
|| normalizedPath === '/api/remote/agent/sync-report') {
|
|
373
|
+
return false;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
if (!normalizedPath.startsWith('/api/remote/devices/')) {
|
|
377
|
+
return true;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
return !(/\/(thumbnail|thumbnail\/request|live\/frame|live\/start|live\/stop)$/i.test(normalizedPath));
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function normalizeWorkspaceStylePath(inputPath) {
|
|
360
384
|
return String(inputPath || '').replaceAll('\\', '/').trim();
|
|
361
385
|
}
|
|
362
386
|
|
|
@@ -2117,11 +2141,13 @@ app.use('/assets/thumbs', async (req, res) => {
|
|
|
2117
2141
|
});
|
|
2118
2142
|
});
|
|
2119
2143
|
|
|
2120
|
-
// Logging middleware
|
|
2121
|
-
app.use((req, res, next) => {
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
}
|
|
2144
|
+
// Logging middleware
|
|
2145
|
+
app.use((req, res, next) => {
|
|
2146
|
+
if (shouldLogHttpRequest(req.method, req.path)) {
|
|
2147
|
+
logHttpRequest(req.method, req.path);
|
|
2148
|
+
}
|
|
2149
|
+
next();
|
|
2150
|
+
});
|
|
2125
2151
|
|
|
2126
2152
|
const PROTECTED_BRIDGE_ROUTES = [
|
|
2127
2153
|
{ method: 'POST', exact: '/api/workspace/browse' },
|
|
@@ -3633,6 +3659,10 @@ const REMOTE_AGENT_OUTPUT_RECONNECT_GRACE_MS = Math.max(
|
|
|
3633
3659
|
Number(process.env.MINDEXEC_REMOTE_AGENT_OUTPUT_RECONNECT_GRACE_MS || 6000) || 6000);
|
|
3634
3660
|
const REMOTE_AGENT_DEFAULT_ENGINE = 'auto';
|
|
3635
3661
|
const REMOTE_AGENT_SYNC_REPORT_LOG_REPEAT_MS = 30000;
|
|
3662
|
+
const REMOTE_AGENT_DUPLICATE_ACTIVE_LOG_REPEAT_MS = 60000;
|
|
3663
|
+
const REMOTE_AGENT_SOFT_SYNC_REPORT_LOG_REPEAT_MS = Math.max(
|
|
3664
|
+
REMOTE_AGENT_SYNC_REPORT_LOG_REPEAT_MS,
|
|
3665
|
+
Number(process.env.MINDEXEC_REMOTE_AGENT_SOFT_SYNC_REPORT_LOG_MS || 300000) || 300000);
|
|
3636
3666
|
const REMOTE_AGENT_SYNC_REPORT_WAKE_MS = Math.max(500, Number(process.env.MINDEXEC_REMOTE_AGENT_SYNC_REPORT_WAKE_MS || 1500) || 1500);
|
|
3637
3667
|
const REMOTE_AGENT_RACE_START_STAGGER_MS = Math.max(0, Number(process.env.MINDEXEC_REMOTE_AGENT_RACE_STAGGER_MS ?? 0) || 0);
|
|
3638
3668
|
const REMOTE_AGENT_MAX_PARALLEL_CANDIDATES = Math.max(
|
|
@@ -3680,6 +3710,8 @@ let remoteAgentSyncReportState = null;
|
|
|
3680
3710
|
let remoteAgentSyncReportLogKey = '';
|
|
3681
3711
|
let remoteAgentSyncReportLogAt = 0;
|
|
3682
3712
|
let remoteAgentSyncReportWakeAt = 0;
|
|
3713
|
+
let remoteAgentConnectFailureLogKey = '';
|
|
3714
|
+
let remoteAgentConnectFailureLogAt = 0;
|
|
3683
3715
|
let remoteAgentConnectPromise = null;
|
|
3684
3716
|
const remoteAgentIntentionalStopKeys = new Set();
|
|
3685
3717
|
const remoteAgentRecentSuccessfulManagers = new Map();
|
|
@@ -3846,6 +3878,66 @@ function createRemoteAgentSyncReport(body = {}) {
|
|
|
3846
3878
|
};
|
|
3847
3879
|
}
|
|
3848
3880
|
|
|
3881
|
+
function isRemoteAgentDuplicateActiveText(value) {
|
|
3882
|
+
return /duplicate-device-active/i.test(String(value || ''));
|
|
3883
|
+
}
|
|
3884
|
+
|
|
3885
|
+
function isRemoteAgentDuplicateActiveReport(report) {
|
|
3886
|
+
return !!report
|
|
3887
|
+
&& (isRemoteAgentDuplicateActiveText(report.reason)
|
|
3888
|
+
|| isRemoteAgentDuplicateActiveText(report.error));
|
|
3889
|
+
}
|
|
3890
|
+
|
|
3891
|
+
function isRemoteAgentSoftSyncReportReason(reason) {
|
|
3892
|
+
return /^(local-monitor-is-host|local-monitor-host-revived|local-monitor-host-republished|local-monitor-host-republish-[\w-]+|same-host|local-host-superseded)$/i
|
|
3893
|
+
.test(String(reason || '').trim());
|
|
3894
|
+
}
|
|
3895
|
+
|
|
3896
|
+
function isRemoteAgentDuplicateActiveState(state = remoteAgentState) {
|
|
3897
|
+
return isRemoteAgentDuplicateActiveText(state?.lastError)
|
|
3898
|
+
|| isRemoteAgentDuplicateActiveText(state?.reconnectReason)
|
|
3899
|
+
|| isRemoteAgentDuplicateActiveText(state?.stderrTail)
|
|
3900
|
+
|| isRemoteAgentDuplicateActiveText(state?.stdoutTail);
|
|
3901
|
+
}
|
|
3902
|
+
|
|
3903
|
+
function findRemoteAgentDuplicateActiveAttempt(attempts = []) {
|
|
3904
|
+
return attempts.find(attempt =>
|
|
3905
|
+
attempt?.child
|
|
3906
|
+
&& attempt.child.exitCode === null
|
|
3907
|
+
&& !attempt.child.killed
|
|
3908
|
+
&& isRemoteAgentDuplicateActiveState(attempt.state));
|
|
3909
|
+
}
|
|
3910
|
+
|
|
3911
|
+
function logRemoteAgentConnectFailure(manager, error = 'unknown') {
|
|
3912
|
+
const duplicateActive = isRemoteAgentDuplicateActiveText(error);
|
|
3913
|
+
const safeManager = normalizeRemoteManagerEndpoint(manager) || 'invalid';
|
|
3914
|
+
const safeError = duplicateActive
|
|
3915
|
+
? 'duplicate-device-active'
|
|
3916
|
+
: safeRemoteAgentField(error || 'unknown', 700);
|
|
3917
|
+
const key = `${duplicateActive ? 'duplicate-active' : 'connect-failed'}|${safeManager}|${safeError}`;
|
|
3918
|
+
const intervalMs = duplicateActive
|
|
3919
|
+
? REMOTE_AGENT_DUPLICATE_ACTIVE_LOG_REPEAT_MS
|
|
3920
|
+
: REMOTE_AGENT_SYNC_REPORT_LOG_REPEAT_MS;
|
|
3921
|
+
const now = Date.now();
|
|
3922
|
+
if (key === remoteAgentConnectFailureLogKey
|
|
3923
|
+
&& now - remoteAgentConnectFailureLogAt < intervalMs) {
|
|
3924
|
+
return;
|
|
3925
|
+
}
|
|
3926
|
+
|
|
3927
|
+
remoteAgentConnectFailureLogKey = key;
|
|
3928
|
+
remoteAgentConnectFailureLogAt = now;
|
|
3929
|
+
|
|
3930
|
+
if (duplicateActive) {
|
|
3931
|
+
logEvent(
|
|
3932
|
+
'remote',
|
|
3933
|
+
`managed RemoteAgent duplicate active; keeping existing session ${formatKeyValue('manager', safeManager)}`,
|
|
3934
|
+
'remote');
|
|
3935
|
+
return;
|
|
3936
|
+
}
|
|
3937
|
+
|
|
3938
|
+
logWarn('remote', `managed RemoteAgent connect failed ${formatKeyValue('manager', safeManager)} ${formatKeyValue('error', safeError)}`);
|
|
3939
|
+
}
|
|
3940
|
+
|
|
3849
3941
|
function isRemoteAgentSyncReportReconnectSignal(report) {
|
|
3850
3942
|
if (!report || report.connected === true) {
|
|
3851
3943
|
return false;
|
|
@@ -3858,6 +3950,10 @@ function isRemoteAgentSyncReportReconnectSignal(report) {
|
|
|
3858
3950
|
return false;
|
|
3859
3951
|
}
|
|
3860
3952
|
|
|
3953
|
+
if (isRemoteAgentDuplicateActiveText(text)) {
|
|
3954
|
+
return false;
|
|
3955
|
+
}
|
|
3956
|
+
|
|
3861
3957
|
if (/(throttled|registry-not-authenticated|registry-auth-pending|session-expired|no-active-target|registry-inactive|local-monitor-is-host|local-host-superseded|same-host|target-missing-endpoint-or-pair)/i.test(text)) {
|
|
3862
3958
|
return false;
|
|
3863
3959
|
}
|
|
@@ -3996,6 +4092,10 @@ function isRemoteAgentOutputReconnectSignal(text) {
|
|
|
3996
4092
|
return false;
|
|
3997
4093
|
}
|
|
3998
4094
|
|
|
4095
|
+
if (isRemoteAgentDuplicateActiveText(normalized)) {
|
|
4096
|
+
return false;
|
|
4097
|
+
}
|
|
4098
|
+
|
|
3999
4099
|
if (/connected to remotehub/i.test(normalized) && !/(disconnect|closed|failed|econnreset|etimedout|epipe|socket|websocket|remotehub.*error)/i.test(normalized)) {
|
|
4000
4100
|
return false;
|
|
4001
4101
|
}
|
|
@@ -4039,6 +4139,21 @@ function rememberRemoteAgentSyncReport(report) {
|
|
|
4039
4139
|
}
|
|
4040
4140
|
|
|
4041
4141
|
remoteAgentSyncReportState = report;
|
|
4142
|
+
if (isRemoteAgentDuplicateActiveReport(report)) {
|
|
4143
|
+
const now = Date.now();
|
|
4144
|
+
const key = `duplicate-active|${report.targetEndpoint || ''}|${report.targetLeaseId || ''}`;
|
|
4145
|
+
if (key !== remoteAgentSyncReportLogKey
|
|
4146
|
+
|| now - remoteAgentSyncReportLogAt >= REMOTE_AGENT_DUPLICATE_ACTIVE_LOG_REPEAT_MS) {
|
|
4147
|
+
remoteAgentSyncReportLogKey = key;
|
|
4148
|
+
remoteAgentSyncReportLogAt = now;
|
|
4149
|
+
logEvent(
|
|
4150
|
+
'remote',
|
|
4151
|
+
`registry sync duplicate active ${formatKeyValue('target', report.targetEndpoint || '-')} ${formatKeyValue('auth', report.authenticated ? 'yes' : 'no')} ${formatKeyValue('note', 'existing-session-kept')}`,
|
|
4152
|
+
'remote');
|
|
4153
|
+
}
|
|
4154
|
+
return true;
|
|
4155
|
+
}
|
|
4156
|
+
|
|
4042
4157
|
const reason = report.reason || report.error || (report.ok ? 'ok' : 'failed');
|
|
4043
4158
|
const status = report.connected
|
|
4044
4159
|
? 'connected'
|
|
@@ -4051,6 +4166,34 @@ function rememberRemoteAgentSyncReport(report) {
|
|
|
4051
4166
|
: report.ok
|
|
4052
4167
|
? 'ok'
|
|
4053
4168
|
: 'failed';
|
|
4169
|
+
|
|
4170
|
+
if (isRemoteAgentSoftSyncReportReason(reason)) {
|
|
4171
|
+
const key = [
|
|
4172
|
+
'soft',
|
|
4173
|
+
status,
|
|
4174
|
+
reason,
|
|
4175
|
+
report.targetEndpoint || '',
|
|
4176
|
+
report.targetLeaseId || '',
|
|
4177
|
+
report.localHostTargetActive ? 'local-host' : ''
|
|
4178
|
+
].join('|');
|
|
4179
|
+
const now = Date.now();
|
|
4180
|
+
const repeatMs = VERBOSE_REMOTE_AGENT_LOGS
|
|
4181
|
+
? REMOTE_AGENT_SYNC_REPORT_LOG_REPEAT_MS
|
|
4182
|
+
: REMOTE_AGENT_SOFT_SYNC_REPORT_LOG_REPEAT_MS;
|
|
4183
|
+
if (key === remoteAgentSyncReportLogKey
|
|
4184
|
+
&& now - remoteAgentSyncReportLogAt < repeatMs) {
|
|
4185
|
+
return true;
|
|
4186
|
+
}
|
|
4187
|
+
|
|
4188
|
+
remoteAgentSyncReportLogKey = key;
|
|
4189
|
+
remoteAgentSyncReportLogAt = now;
|
|
4190
|
+
logEvent(
|
|
4191
|
+
'remote',
|
|
4192
|
+
`registry sync ${status} ${formatKeyValue('reason', reason)} ${formatKeyValue('target', report.targetEndpoint || '-')} ${formatKeyValue('auth', report.authenticated ? 'yes' : 'no')}`,
|
|
4193
|
+
'remote');
|
|
4194
|
+
return true;
|
|
4195
|
+
}
|
|
4196
|
+
|
|
4054
4197
|
markRemoteAgentReconnectNeededFromReport(report);
|
|
4055
4198
|
const key = [
|
|
4056
4199
|
status,
|
|
@@ -4091,6 +4234,14 @@ function appendRemoteAgentOutput(stream, chunk, stateConnectionKey = '') {
|
|
|
4091
4234
|
remoteAgentState.needsReconnect = false;
|
|
4092
4235
|
remoteAgentState.reconnectReason = '';
|
|
4093
4236
|
remoteAgentState.reconnectRequestedAt = '';
|
|
4237
|
+
remoteAgentState.lastError = '';
|
|
4238
|
+
}
|
|
4239
|
+
if (isRemoteAgentDuplicateActiveText(text)) {
|
|
4240
|
+
remoteAgentState.ready = false;
|
|
4241
|
+
remoteAgentState.needsReconnect = false;
|
|
4242
|
+
remoteAgentState.reconnectReason = '';
|
|
4243
|
+
remoteAgentState.reconnectRequestedAt = '';
|
|
4244
|
+
remoteAgentState.lastError = 'duplicate-device-active';
|
|
4094
4245
|
}
|
|
4095
4246
|
if (isRemoteAgentOutputReconnectSignal(text)) {
|
|
4096
4247
|
markRemoteAgentReconnectNeeded('process-output-disconnected', 'agent-process-output-disconnected');
|
|
@@ -4436,6 +4587,7 @@ function getRemoteAgentRunningAgeMs(agent = remoteAgentState) {
|
|
|
4436
4587
|
function isRemoteAgentRunningButNotReadyStale() {
|
|
4437
4588
|
return isRemoteAgentRegistryOwned()
|
|
4438
4589
|
&& remoteAgentState.ready !== true
|
|
4590
|
+
&& !isRemoteAgentDuplicateActiveState(remoteAgentState)
|
|
4439
4591
|
&& getRemoteAgentRunningAgeMs(remoteAgentState) >= REMOTE_AGENT_READY_STALE_MS;
|
|
4440
4592
|
}
|
|
4441
4593
|
|
|
@@ -4947,6 +5099,15 @@ function appendRemoteAgentAttemptOutput(attempt, stream, chunk) {
|
|
|
4947
5099
|
attempt.state.connectedAt = new Date().toISOString();
|
|
4948
5100
|
attempt.state.needsReconnect = false;
|
|
4949
5101
|
attempt.state.reconnectReason = '';
|
|
5102
|
+
attempt.state.reconnectRequestedAt = '';
|
|
5103
|
+
attempt.state.lastError = '';
|
|
5104
|
+
}
|
|
5105
|
+
if (isRemoteAgentDuplicateActiveText(text)) {
|
|
5106
|
+
attempt.state.ready = false;
|
|
5107
|
+
attempt.state.needsReconnect = false;
|
|
5108
|
+
attempt.state.reconnectReason = '';
|
|
5109
|
+
attempt.state.reconnectRequestedAt = '';
|
|
5110
|
+
attempt.state.lastError = 'duplicate-device-active';
|
|
4950
5111
|
}
|
|
4951
5112
|
if (isRemoteAgentOutputReconnectSignal(text)) {
|
|
4952
5113
|
attempt.state.ready = false;
|
|
@@ -5151,10 +5312,12 @@ async function startRemoteAgentConnectionRace(options = {}) {
|
|
|
5151
5312
|
const attempts = [];
|
|
5152
5313
|
for (let index = 0; index < managers.length; index += 1) {
|
|
5153
5314
|
const manager = managers[index];
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5315
|
+
if (VERBOSE_REMOTE_AGENT_LOGS || managers.length > 1) {
|
|
5316
|
+
logEvent(
|
|
5317
|
+
'remote',
|
|
5318
|
+
`managed RemoteAgent race ${index + 1}/${managers.length} ${formatKeyValue('manager', manager)}`,
|
|
5319
|
+
'remote');
|
|
5320
|
+
}
|
|
5158
5321
|
|
|
5159
5322
|
const attempt = createRemoteAgentAttempt({
|
|
5160
5323
|
manager,
|
|
@@ -5219,6 +5382,33 @@ async function startRemoteAgentConnectionRace(options = {}) {
|
|
|
5219
5382
|
return { ok: true, alreadyRunning: false, agent: serializeRemoteAgentState() };
|
|
5220
5383
|
}
|
|
5221
5384
|
|
|
5385
|
+
const duplicateActiveAttempt = findRemoteAgentDuplicateActiveAttempt(attempts);
|
|
5386
|
+
if (duplicateActiveAttempt) {
|
|
5387
|
+
for (const attempt of attempts) {
|
|
5388
|
+
if (attempt !== duplicateActiveAttempt) {
|
|
5389
|
+
await stopRemoteAgentAttempt(attempt, 'race-lost-duplicate-active');
|
|
5390
|
+
}
|
|
5391
|
+
}
|
|
5392
|
+
|
|
5393
|
+
remoteAgentState = duplicateActiveAttempt.state;
|
|
5394
|
+
remoteAgentState.status = 'running';
|
|
5395
|
+
remoteAgentState.proc = duplicateActiveAttempt.child;
|
|
5396
|
+
remoteAgentState.ready = false;
|
|
5397
|
+
remoteAgentState.needsReconnect = false;
|
|
5398
|
+
remoteAgentState.reconnectReason = '';
|
|
5399
|
+
remoteAgentState.reconnectRequestedAt = '';
|
|
5400
|
+
remoteAgentState.lastError = 'duplicate-device-active';
|
|
5401
|
+
remoteAgentState.updatedAt = new Date().toISOString();
|
|
5402
|
+
emitBridgeEvent('RemoteAgentWaiting', serializeRemoteAgentState());
|
|
5403
|
+
logRemoteAgentConnectFailure(duplicateActiveAttempt.manager, 'duplicate-device-active');
|
|
5404
|
+
return {
|
|
5405
|
+
ok: true,
|
|
5406
|
+
waiting: true,
|
|
5407
|
+
reason: 'duplicate-device-active',
|
|
5408
|
+
agent: serializeRemoteAgentState()
|
|
5409
|
+
};
|
|
5410
|
+
}
|
|
5411
|
+
|
|
5222
5412
|
const lastAttempt = attempts.findLast?.(attempt => attempt?.error || attempt?.state?.lastError)
|
|
5223
5413
|
|| attempts[attempts.length - 1]
|
|
5224
5414
|
|| null;
|
|
@@ -6328,6 +6518,14 @@ function isRemoteRegistryTargetEndpointLocal(localHub, target) {
|
|
|
6328
6518
|
return targetEndpoints.some(endpoint => localSet.has(endpoint.toLowerCase()));
|
|
6329
6519
|
}
|
|
6330
6520
|
|
|
6521
|
+
function shouldRepublishLocalHostForRegistryTarget(localHub, target) {
|
|
6522
|
+
return localHub?.hostTargetActive === true
|
|
6523
|
+
&& target?.active === true
|
|
6524
|
+
&& !isRemoteRegistryTargetExpired(target)
|
|
6525
|
+
&& !isRemoteRegistryTargetSameAsLocalHost(localHub, target)
|
|
6526
|
+
&& isRemoteRegistryTargetEndpointLocal(localHub, target);
|
|
6527
|
+
}
|
|
6528
|
+
|
|
6331
6529
|
async function fetchRemoteRegistryTarget(config, session) {
|
|
6332
6530
|
const url = new URL('/rest/v1/remote_host_targets', config.url);
|
|
6333
6531
|
url.searchParams.set('select', '*');
|
|
@@ -6401,6 +6599,32 @@ async function readRemoteRegistryContext() {
|
|
|
6401
6599
|
return { ok: true, config, session };
|
|
6402
6600
|
}
|
|
6403
6601
|
|
|
6602
|
+
async function readCurrentRemoteRegistryTarget() {
|
|
6603
|
+
const context = await readRemoteRegistryContext();
|
|
6604
|
+
if (!context.ok) {
|
|
6605
|
+
return {
|
|
6606
|
+
ok: false,
|
|
6607
|
+
reason: context.reason,
|
|
6608
|
+
target: null
|
|
6609
|
+
};
|
|
6610
|
+
}
|
|
6611
|
+
|
|
6612
|
+
try {
|
|
6613
|
+
return {
|
|
6614
|
+
ok: true,
|
|
6615
|
+
reason: 'ok',
|
|
6616
|
+
target: await fetchRemoteRegistryTarget(context.config, context.session)
|
|
6617
|
+
};
|
|
6618
|
+
} catch (error) {
|
|
6619
|
+
return {
|
|
6620
|
+
ok: false,
|
|
6621
|
+
reason: getRemoteHostTargetRegistryErrorReason(error),
|
|
6622
|
+
error: error?.message || String(error || 'registry-fetch-failed'),
|
|
6623
|
+
target: null
|
|
6624
|
+
};
|
|
6625
|
+
}
|
|
6626
|
+
}
|
|
6627
|
+
|
|
6404
6628
|
function getRemoteHostTargetEndpointReason(endpoint) {
|
|
6405
6629
|
const normalized = normalizeRemoteManagerEndpoint(endpoint);
|
|
6406
6630
|
if (!normalized) {
|
|
@@ -6890,6 +7114,37 @@ async function runRemoteHostTargetRenewOnce(trigger = 'timer', options = {}) {
|
|
|
6890
7114
|
});
|
|
6891
7115
|
|
|
6892
7116
|
if (registry.stale) {
|
|
7117
|
+
const registryTarget = await readCurrentRemoteRegistryTarget();
|
|
7118
|
+
if (shouldRepublishLocalHostForRegistryTarget(hub, registryTarget.target)) {
|
|
7119
|
+
const takeoverRegistry = await publishLocalRemoteHostTargetToRegistry(hub, { takeover: true });
|
|
7120
|
+
const reason = takeoverRegistry.ok
|
|
7121
|
+
? 'local-monitor-host-republished'
|
|
7122
|
+
: `local-monitor-host-republish-${takeoverRegistry.reason || 'registry-publish-failed'}`;
|
|
7123
|
+
updateRemoteHostTargetRenewState({
|
|
7124
|
+
status: takeoverRegistry.ok ? 'active' : 'skipped',
|
|
7125
|
+
reason,
|
|
7126
|
+
nodeId: hub.hostTargetNodeId,
|
|
7127
|
+
leaseId: hub.hostTargetLeaseId,
|
|
7128
|
+
hostInstanceId: hub.hostTargetHostInstanceId || hub.hostInstanceId,
|
|
7129
|
+
endpoint: takeoverRegistry.endpoint || hub.hostTargetEndpoint,
|
|
7130
|
+
endpointCandidates: takeoverRegistry.endpointCandidates || hub.hostTargetEndpointCandidates || [],
|
|
7131
|
+
expiresAt: hub.hostTargetExpiresAt,
|
|
7132
|
+
lastAttemptAt: attemptedAt,
|
|
7133
|
+
lastSuccessAt: takeoverRegistry.ok ? new Date().toISOString() : remoteHostTargetRenewState.lastSuccessAt,
|
|
7134
|
+
lastError: takeoverRegistry.ok || isRemoteHostTargetRenewSoftSkipReason(takeoverRegistry.reason)
|
|
7135
|
+
? ''
|
|
7136
|
+
: (takeoverRegistry.reason || 'registry-publish-failed')
|
|
7137
|
+
});
|
|
7138
|
+
logRemoteHostTargetRenew(takeoverRegistry.ok ? 'ok' : 'skipped', reason, takeoverRegistry.endpoint || hub.hostTargetEndpoint);
|
|
7139
|
+
scheduleRemoteHostTargetRenew(
|
|
7140
|
+
takeoverRegistry.ok || isRemoteHostTargetRenewSoftSkipReason(takeoverRegistry.reason)
|
|
7141
|
+
? REMOTE_HOST_TARGET_RENEW_MS
|
|
7142
|
+
: REMOTE_REGISTRY_FOLLOWER_FAST_RETRY_MS,
|
|
7143
|
+
takeoverRegistry.ok ? 'local-host-republished' : 'local-host-republish-retry');
|
|
7144
|
+
wakeRemoteRegistryFollower('local-host-republished').catch(() => {});
|
|
7145
|
+
return serializeRemoteHostTargetRenewState();
|
|
7146
|
+
}
|
|
7147
|
+
|
|
6893
7148
|
remoteHub.setHostTarget({
|
|
6894
7149
|
enabled: false,
|
|
6895
7150
|
nodeId: hub.hostTargetNodeId
|
|
@@ -7021,6 +7276,14 @@ function scheduleRemoteRegistryWakeFromSyncReport(report) {
|
|
|
7021
7276
|
};
|
|
7022
7277
|
}
|
|
7023
7278
|
|
|
7279
|
+
if (isRemoteAgentDuplicateActiveReport(report)) {
|
|
7280
|
+
return {
|
|
7281
|
+
follower: false,
|
|
7282
|
+
hostRenew: false,
|
|
7283
|
+
reason: 'duplicate-device-active'
|
|
7284
|
+
};
|
|
7285
|
+
}
|
|
7286
|
+
|
|
7024
7287
|
const now = Date.now();
|
|
7025
7288
|
let follower = false;
|
|
7026
7289
|
if (now - remoteAgentSyncReportWakeAt >= REMOTE_AGENT_SYNC_REPORT_WAKE_MS) {
|
|
@@ -7208,6 +7471,55 @@ async function runRemoteRegistryFollowerOnce(trigger = 'timer') {
|
|
|
7208
7471
|
|
|
7209
7472
|
if (localHub?.hostTargetActive === true) {
|
|
7210
7473
|
resetRemoteRegistryInactiveTargetGrace();
|
|
7474
|
+
if (shouldRepublishLocalHostForRegistryTarget(localHub, target)) {
|
|
7475
|
+
const renewed = remoteHub.setHostTarget({
|
|
7476
|
+
enabled: true,
|
|
7477
|
+
nodeId: localHub.hostTargetNodeId || target.nodeId,
|
|
7478
|
+
leaseMs: REMOTE_HOST_TARGET_LEASE_MS
|
|
7479
|
+
});
|
|
7480
|
+
const republishedHub = remoteHub.getStatus({ includeSecrets: true });
|
|
7481
|
+
const registry = renewed?.ok === true
|
|
7482
|
+
? await publishLocalRemoteHostTargetToRegistry(republishedHub, { takeover: true })
|
|
7483
|
+
: {
|
|
7484
|
+
ok: false,
|
|
7485
|
+
reason: renewed?.error || 'local-host-renew-failed',
|
|
7486
|
+
endpoint: target.endpoint,
|
|
7487
|
+
endpointCandidates: target.endpointCandidates
|
|
7488
|
+
};
|
|
7489
|
+
const reason = registry.ok
|
|
7490
|
+
? 'local-monitor-host-republished'
|
|
7491
|
+
: `local-monitor-host-republish-${registry.reason || 'registry-publish-failed'}`;
|
|
7492
|
+
updateRemoteRegistryFollowerState({
|
|
7493
|
+
status: 'skipped',
|
|
7494
|
+
reason,
|
|
7495
|
+
authenticated: true,
|
|
7496
|
+
lastAttemptAt: attemptedAt,
|
|
7497
|
+
lastSuccessAt: attemptedAt,
|
|
7498
|
+
targetEndpoint: registry.endpoint || target.endpoint,
|
|
7499
|
+
targetEndpointCandidates: registry.endpointCandidates?.length ? registry.endpointCandidates : target.endpointCandidates,
|
|
7500
|
+
targetLeaseId: republishedHub.hostTargetLeaseId || localHub.hostTargetLeaseId,
|
|
7501
|
+
targetNodeId: republishedHub.hostTargetNodeId || localHub.hostTargetNodeId,
|
|
7502
|
+
lastError: registry.ok || isRemoteHostTargetRenewSoftSkipReason(registry.reason) ? '' : (registry.reason || 'host-target-republish-failed')
|
|
7503
|
+
});
|
|
7504
|
+
await reportRemoteRegistryFollowerSync({
|
|
7505
|
+
ok: registry.ok !== false,
|
|
7506
|
+
skipped: true,
|
|
7507
|
+
reason,
|
|
7508
|
+
trigger,
|
|
7509
|
+
authenticated: true,
|
|
7510
|
+
localHostTargetActive: true,
|
|
7511
|
+
targetActive: true,
|
|
7512
|
+
targetEndpoint: registry.endpoint || target.endpoint,
|
|
7513
|
+
targetEndpointCandidates: registry.endpointCandidates?.length ? registry.endpointCandidates : target.endpointCandidates,
|
|
7514
|
+
targetLeaseId: republishedHub.hostTargetLeaseId || localHub.hostTargetLeaseId,
|
|
7515
|
+
targetNodeId: republishedHub.hostTargetNodeId || localHub.hostTargetNodeId,
|
|
7516
|
+
targetExpiresAt: republishedHub.hostTargetExpiresAt || localHub.hostTargetExpiresAt
|
|
7517
|
+
});
|
|
7518
|
+
scheduleRemoteHostTargetRenewWake('local-host-republished', { takeover: true });
|
|
7519
|
+
scheduleRemoteRegistryFollower(REMOTE_REGISTRY_FOLLOWER_POLL_MS, 'local-host-republished');
|
|
7520
|
+
return serializeRemoteRegistryFollowerState();
|
|
7521
|
+
}
|
|
7522
|
+
|
|
7211
7523
|
if (target?.active === true
|
|
7212
7524
|
&& !isRemoteRegistryTargetExpired(target)
|
|
7213
7525
|
&& !isRemoteRegistryTargetSameAsLocalHost(localHub, target)) {
|
|
@@ -12180,7 +12492,7 @@ app.post('/api/remote/agent/connect', async (req, res) => {
|
|
|
12180
12492
|
source: req.body?.source || 'registry'
|
|
12181
12493
|
});
|
|
12182
12494
|
if (!result.ok) {
|
|
12183
|
-
|
|
12495
|
+
logRemoteAgentConnectFailure(requestedManager || 'invalid', result.error || 'unknown');
|
|
12184
12496
|
}
|
|
12185
12497
|
res.status(result.ok ? 200 : 400).json(result);
|
|
12186
12498
|
} catch (err) {
|