@minionry/minion 0.7.36 → 0.7.39

package/dist/server/services/platform.js

@@ -1,5 +1,6 @@
 // Copyright (c) 2025-present Minionry, Inc.
 // SPDX-License-Identifier: LicenseRef-Minionry-Software
+import { randomBytes } from 'node:crypto';
 import { arch, hostname, type } from 'node:os';
 import { basename } from 'node:path';
 import { AnalyticsEvents, trackEvent } from './analytics.js';
@@ -75,13 +76,40 @@ const CONNECT_TIMEOUT_MS = 10_000;
 const HANDSHAKE_ERROR_CLOSE_GRACE_MS = 500;
 const HEARTBEAT_TIMEOUT_CLOSE_CODE = 4006;
 const DIAL_TIMEOUT_CLOSE_CODE = 4011;
+const HANDOFF_DIAL_TIMEOUT_MS = 10_000;
+const MIGRATE_OLD_CLOSE_DELAY_MS = 2_000;
+const SOCKET_SCOPED_TYPES = new Set(['paired', 'relayHandoff', 'ping', 'pong']);
+function handoffDeadlineMs(delayMs, handoffBy, now) {
+    const budget = delayMs + HANDOFF_DIAL_TIMEOUT_MS;
+    const untilHandoffBy = handoffBy - now;
+    return untilHandoffBy > delayMs ? Math.min(untilHandoffBy, budget) : budget;
+}
+function retireSocket(ws) {
+    try {
+        ws.onopen = null;
+        ws.onmessage = null;
+        ws.onclose = null;
+        ws.onerror = null;
+    }
+    catch {   }
+    try {
+        if (ws.readyState === WebSocketImpl.CONNECTING || ws.readyState === WebSocketImpl.OPEN) {
+            ws.close();
+        }
+    }
+    catch {   }
+}
 const CREDENTIALS_POLL_MS = 15_000;
 const OUTBOUND_QUEUE_LIMIT = 100;
+const USAGE_RETRY_QUEUE_LIMIT = 1_000;
 const MAX_OUTBOUND_FRAME_BYTES = 16 * 1024 * 1024;
 const NEVER_QUEUED_TYPES = new Set(['pong', 'terminalOutput', 'sdkGrantState', 'shuttingDown']);
 export class PlatformConnection {
     ws = null;
     socketEpoch = 0;
+    epochSeq = 0;
+    handoff = null;
+    leaving = new Map();
     reconnectTimeout = null;
     reconnectPolicy = new ReconnectPolicy();
     isIntentionallyClosed = false;
@@ -99,9 +127,13 @@ export class PlatformConnection {
     spaceVisibility;
     e2ee;
     startedAt;
+    epochNonce;
     tokens;
     outboundQueue = [];
     outboundDropped = 0;
+    usageRetryQueue = new Map();
+    usageDropped = 0;
+    usageKeySeq = 0;
     browserFrameRefusalWarned = false;
     browserFrameRefusals = 0;
     constructor(workingDirectory, callbacks = {}, platformUrl) {
@@ -109,6 +141,7 @@ export class PlatformConnection {
         this.platformUrl = platformUrl || DEFAULT_PLATFORM_URL;
         this.callbacks = callbacks;
         this.startedAt = new Date().toISOString();
+        this.epochNonce = randomBytes(16).toString('base64url');
         this.tokens = new TokenLifecycle({
             platformUrl: this.platformUrl,
             send: (msg) => this.send(msg),
@@ -163,8 +196,9 @@ export class PlatformConnection {
         }
         this.stopCredentialsWatch();
         this.presentedToken = authToken;
+        this.dropHandoff();
         this.detachSocket();
-        const epoch = ++this.socketEpoch;
+        const epoch = this.nextLiveEpoch();
         const wsUrl = this.buildConnectionUrl();
         let ws;
         try {
@@ -197,6 +231,7 @@ export class PlatformConnection {
                 ...E2EE_CAPABILITIES,
                 fileTransfer: FILE_TRANSFER_CAPABILITY,
                 maxUploadBytes: getMaxUploadBytes(),
+                relayHandoff: true,
             }),
             startedAt: this.startedAt,
             protocolVersion: String(PROTOCOL_VERSION),
@@ -217,18 +252,7 @@ export class PlatformConnection {
         }, CONNECT_TIMEOUT_MS);
     }
     attachSocketHandlers(ws, authToken, connectionTimeout, epoch) {
-        let closeSettled = false;
-        const settleClose = (event) => {
-            if (closeSettled)
-                return;
-            closeSettled = true;
-            clearTimeout(connectionTimeout);
-            if (epoch !== this.socketEpoch) {
-                console.log(`[Platform] Ignoring close on a retired socket (code ${event.code})`);
-                return;
-            }
-            this.handleSocketClose(event);
-        };
+        this.bindLiveHandlers(ws, epoch, () => clearTimeout(connectionTimeout));
         ws.onopen = () => {
             if (epoch !== this.socketEpoch) {
                 try {
@@ -238,11 +262,36 @@ export class PlatformConnection {
                 return;
             }
             clearTimeout(connectionTimeout);
-            ws.send(JSON.stringify({ type: 'auth', token: authToken }));
+            ws.send(JSON.stringify(this.authFrame(authToken)));
             this.tokens.maybeRefresh();
             this.tokens.startPeriodicCheck();
             trackEvent(AnalyticsEvents.PLATFORM_CONNECTED);
         };
+        const wsWithEmitter = ws;
+        if (typeof wsWithEmitter.on === 'function') {
+            wsWithEmitter.on('unexpected-response', (...args) => {
+                if (epoch !== this.socketEpoch)
+                    return;
+                const response = args[1];
+                if (response?.statusCode === 426) {
+                    this.protocolUpgradeRequired = true;
+                }
+            });
+        }
+    }
+    bindLiveHandlers(ws, epoch, onSettle = () => { }) {
+        let closeSettled = false;
+        const settleClose = (event) => {
+            if (closeSettled)
+                return;
+            closeSettled = true;
+            onSettle();
+            if (epoch !== this.socketEpoch) {
+                console.log(`[Platform] Ignoring close on a retired socket (code ${event.code})`);
+                return;
+            }
+            this.handleSocketClose(event);
+        };
         ws.onmessage = (event) => {
             if (epoch !== this.socketEpoch)
                 return;
@@ -255,46 +304,36 @@ export class PlatformConnection {
         };
         ws.onclose = (event) => settleClose(event);
         ws.onerror = () => {
-            clearTimeout(connectionTimeout);
+            onSettle();
             setTimeout(() => {
                 settleClose({ code: 1012, reason: 'handshake failed' });
             }, HANDSHAKE_ERROR_CLOSE_GRACE_MS);
         };
-        const wsWithEmitter = ws;
-        if (typeof wsWithEmitter.on === 'function') {
-            wsWithEmitter.on('unexpected-response', (...args) => {
-                if (epoch !== this.socketEpoch)
-                    return;
-                const response = args[1];
-                if (response?.statusCode === 426) {
-                    this.protocolUpgradeRequired = true;
-                }
-            });
-        }
+    }
+    authFrame(token, migrateFrom) {
+        return {
+            type: 'auth',
+            token,
+            startedAt: this.startedAt,
+            epochNonce: this.epochNonce,
+            ...(migrateFrom ? { migrateFrom } : {}),
+        };
+    }
+    nextLiveEpoch() {
+        this.socketEpoch = ++this.epochSeq;
+        return this.socketEpoch;
     }
     detachSocket() {
         const ws = this.ws;
         if (!ws)
             return;
         this.ws = null;
-        try {
-            ws.onopen = null;
-            ws.onmessage = null;
-            ws.onclose = null;
-            ws.onerror = null;
-        }
-        catch {   }
-        try {
-            if (ws.readyState === WebSocketImpl.CONNECTING || ws.readyState === WebSocketImpl.OPEN) {
-                ws.close();
-            }
-        }
-        catch {   }
+        retireSocket(ws);
     }
     forceClose(ws, epoch, code, reason) {
         if (epoch !== this.socketEpoch)
             return;
-        this.socketEpoch++;
+        this.nextLiveEpoch();
         this.ws = null;
         try {
             ws.onopen = null;
@@ -320,9 +359,24 @@ export class PlatformConnection {
         this.reconnectPolicy.noteDisconnected();
         if (this.isIntentionallyClosed)
             return;
-        const disposition = classifyClose(event.code, event.reason, {
+        const handoff = this.handoff;
+        if (handoff?.ws) {
+            handoff.fromClosed = event;
+            console.log(`[Platform] Socket closed mid-handoff (code ${event.code}) — waiting on the new one`);
+            return;
+        }
+        if (handoff)
+            this.clearHandoff(handoff);
+        this.recoverFromClose(event, handoff !== null);
+    }
+    recoverFromClose(event, migratingAway) {
+        let disposition = classifyClose(event.code, event.reason, {
             protocolUpgradeRequired: this.protocolUpgradeRequired,
         });
+        if (disposition === 'superseded' && migratingAway) {
+            console.log('[Platform] Ignoring 4015 on the socket this process is migrating away from — reconnecting');
+            disposition = 'reconnect';
+        }
         switch (disposition) {
             case 'protocol_too_old':
                 this.handleProtocolUpgradeRequired();
@@ -330,11 +384,18 @@ export class PlatformConnection {
             case 'superseded':
                 this.handleSuperseded(event);
                 return;
+            case 'handoff':
+                console.log(`Relay handed this socket off (code ${event.code}${event.reason ? `: ${event.reason}` : ''}), reconnecting...`);
+                this.callbacks.onDisconnected?.();
+                trackEvent(AnalyticsEvents.PLATFORM_DISCONNECTED);
+                this.scheduleReconnect(this.reconnectPolicy.handoffDelayMs());
+                return;
             case 'auth_failed':
                 console.error('\n❌ Authentication failed. Your device token may be invalid or expired.');
                 console.error('   Run `minion login --force` to re-authenticate — this server reconnects as soon as you do.\n');
                 this.tokens.notifyAuthExpired();
                 this.callbacks.onError?.('Authentication failed - run `minion login --force`');
+                this.callbacks.onDisconnected?.();
                 this.awaitCredentials(this.presentedToken);
                 return;
             default:
@@ -367,8 +428,151 @@ export class PlatformConnection {
             this.connect();
         }, PROTOCOL_UPGRADE_RECONNECT_DELAY_MS);
     }
-    handleMessage(message) {
+    beginHandoff(message) {
+        const { v, delayMs, handoffBy } = message;
+        if (v !== 1 || typeof delayMs !== 'number' || typeof handoffBy !== 'number')
+            return;
+        if (!Number.isFinite(delayMs) || !Number.isFinite(handoffBy))
+            return;
+        const from = this.ws;
+        if (this.handoff || !from || !this.isConnected || !this.connectionId)
+            return;
+        const delay = Math.max(0, delayMs);
+        const handoff = { from, migrateFrom: this.connectionId, ws: null, fromClosed: null, timers: [] };
+        this.handoff = handoff;
+        const reason = typeof message.reason === 'string' ? message.reason : 'unspecified';
+        console.log(`[Platform] Relay handoff (${reason}) — moving to a new socket in ${delay}ms`);
+        handoff.timers.push(setTimeout(() => this.dialHandoff(handoff), delay), setTimeout(() => this.abandonHandoff(handoff, 'handoffBy passed'), handoffDeadlineMs(delay, handoffBy, Date.now())));
+    }
+    dialHandoff(handoff) {
+        if (this.handoff !== handoff)
+            return;
+        const token = getCredentials()?.token;
+        if (!token) {
+            this.abandonHandoff(handoff, 'not logged in');
+            return;
+        }
+        let ws;
+        try {
+            ws = new WebSocketImpl(this.buildConnectionUrl());
+        }
+        catch (err) {
+            this.abandonHandoff(handoff, err instanceof Error ? err.message : String(err));
+            return;
+        }
+        handoff.ws = ws;
+        ws.onopen = () => {
+            if (this.handoff !== handoff)
+                return;
+            ws.send(JSON.stringify(this.authFrame(token, handoff.migrateFrom)));
+        };
+        ws.onmessage = (event) => {
+            if (this.handoff !== handoff)
+                return;
+            try {
+                const message = JSON.parse(event.data.toString());
+                if (message.type === 'paired')
+                    this.completeHandoff(handoff, ws, message);
+                else
+                    this.handleMessage(message, false);
+            }
+            catch (err) {
+                console.error('Failed to parse platform message:', err);
+            }
+        };
+        ws.onclose = (event) => this.abandonHandoff(handoff, `new socket closed with code ${event.code}`);
+        ws.onerror = () => this.abandonHandoff(handoff, 'new socket failed');
+    }
+    completeHandoff(handoff, ws, paired) {
+        this.clearHandoff(handoff);
+        const old = handoff.from;
+        if (old.readyState === WebSocketImpl.OPEN) {
+            try {
+                old.send(JSON.stringify({ type: 'handoffMark' }));
+            }
+            catch {   }
+        }
+        this.ws = ws;
+        ws.onopen = null;
+        this.bindLiveHandlers(ws, this.nextLiveEpoch());
+        this.retireLeaving(old);
+        if (handoff.fromClosed) {
+            this.tokens.maybeRefresh();
+            this.tokens.startPeriodicCheck();
+        }
+        console.log('[Platform] Relay handoff complete — moved to the new socket');
+        this.handleMessage(paired);
+    }
+    retireLeaving(old) {
+        if (old.readyState !== WebSocketImpl.OPEN && old.readyState !== WebSocketImpl.CONNECTING) {
+            retireSocket(old);
+            return;
+        }
+        old.onopen = null;
+        old.onerror = null;
+        old.onmessage = (event) => {
+            try {
+                this.handleMessage(JSON.parse(event.data.toString()), false);
+            }
+            catch (err) {
+                console.error('Failed to parse platform message:', err);
+            }
+        };
+        old.onclose = (event) => {
+            console.log(`[Platform] Previous socket closed after the handoff (code ${event.code})`);
+            this.dropLeaving(old);
+        };
+        this.leaving.set(old, setTimeout(() => {
+            this.dropLeaving(old);
+            retireSocket(old);
+        }, MIGRATE_OLD_CLOSE_DELAY_MS));
+    }
+    dropLeaving(old) {
+        const timer = this.leaving.get(old);
+        if (timer === undefined)
+            return;
+        clearTimeout(timer);
+        this.leaving.delete(old);
+        old.onmessage = null;
+        old.onclose = null;
+    }
+    abandonHandoff(handoff, why) {
+        if (this.handoff !== handoff)
+            return;
+        this.clearHandoff(handoff);
+        if (handoff.ws)
+            retireSocket(handoff.ws);
+        const closed = handoff.fromClosed;
+        console.warn(`[Platform] Relay handoff abandoned (${why}) — ${closed ? 'reconnecting' : 'staying on the current socket'}`);
+        if (closed && !this.isIntentionallyClosed)
+            this.recoverFromClose(closed, true);
+    }
+    clearHandoff(handoff) {
+        for (const timer of handoff.timers)
+            clearTimeout(timer);
+        if (this.handoff === handoff)
+            this.handoff = null;
+    }
+    dropHandoff() {
+        const handoff = this.handoff;
+        if (!handoff)
+            return;
+        this.clearHandoff(handoff);
+        if (handoff.ws)
+            retireSocket(handoff.ws);
+    }
+    settleBeforeEnvelope(message, fromLive) {
+        if (!fromLive)
+            return SOCKET_SCOPED_TYPES.has(message.type);
         this.heartbeat.noteInbound();
+        if (message.type !== 'relayHandoff')
+            return false;
+        this.beginHandoff(message);
+        return true;
+    }
+    handleMessage(message, fromLive = true) {
+        if (this.settleBeforeEnvelope(message, fromLive))
+            return;
         if (isV1Envelope(message)) {
             if (message.enc !== false) {
                 this.e2ee.handleSealedInbound(message, (legacy) => {
@@ -390,6 +594,7 @@ export class PlatformConnection {
                 replaceFileShareRegistry(message.fileShares, this.workingDirectory);
                 this.heartbeat.start();
                 this.callbacks.onConnected?.(message.connectionId);
+                this.flushOutboundQueue();
                 break;
             }
             case 'web_connected':
@@ -488,12 +693,12 @@ export class PlatformConnection {
             this.credentialsWatch = null;
         }
     }
-    scheduleReconnect() {
+    scheduleReconnect(delayMs) {
         if (this.reconnectTimeout)
             return;
         if (this.superseded)
             return;
-        const delay = this.reconnectPolicy.nextDelayMs();
+        const delay = delayMs ?? this.reconnectPolicy.nextDelayMs();
         this.reconnectTimeout = setTimeout(() => {
             this.reconnectTimeout = null;
             this.connect();
@@ -502,6 +707,10 @@ export class PlatformConnection {
     send(message) {
         if (this.refuseCleartextBrowserFrame(message))
             return;
+        if (message?.type === 'reportUsage' && !this.isConnectedToPlatform()) {
+            this.holdUsageReport(message);
+            return;
+        }
         if (!this.ws || this.ws.readyState !== WebSocketImpl.OPEN) {
             this.queueWhileDisconnected(message);
             return;
@@ -601,17 +810,44 @@ export class PlatformConnection {
         }
         this.outboundQueue.push(message);
     }
+    holdUsageReport(message) {
+        const reportId = message.data?.reportId;
+        const key = typeof reportId === 'string' && reportId !== '' ? reportId : `unkeyed-${++this.usageKeySeq}`;
+        if (this.usageRetryQueue.has(key))
+            return;
+        if (this.usageRetryQueue.size >= USAGE_RETRY_QUEUE_LIMIT) {
+            const oldest = this.usageRetryQueue.keys().next().value;
+            this.usageRetryQueue.delete(oldest);
+            this.usageDropped++;
+        }
+        this.usageRetryQueue.set(key, message);
+    }
     flushOutboundQueue() {
         if (this.outboundDropped > 0) {
             console.log(`[Platform] ${this.outboundDropped} queued outbound frame${this.outboundDropped === 1 ? '' : 's'} dropped over the ${OUTBOUND_QUEUE_LIMIT}-frame cap`);
             this.outboundDropped = 0;
         }
-        if (this.outboundQueue.length === 0)
+        if (this.outboundQueue.length > 0) {
+            const queued = this.outboundQueue;
+            this.outboundQueue = [];
+            console.log(`[Platform] Flushing ${queued.length} outbound message${queued.length === 1 ? '' : 's'} queued during the outage`);
+            for (const message of queued) {
+                this.send(message);
+            }
+        }
+        this.flushUsageRetryQueue();
+    }
+    flushUsageRetryQueue() {
+        if (this.usageDropped > 0) {
+            console.log(`[Platform] ${this.usageDropped} held usage report${this.usageDropped === 1 ? '' : 's'} dropped over the ${USAGE_RETRY_QUEUE_LIMIT}-report cap`);
+            this.usageDropped = 0;
+        }
+        if (this.usageRetryQueue.size === 0 || !this.isConnectedToPlatform())
             return;
-        const queued = this.outboundQueue;
-        this.outboundQueue = [];
-        console.log(`[Platform] Flushing ${queued.length} outbound message${queued.length === 1 ? '' : 's'} queued during the outage`);
-        for (const message of queued) {
+        const held = [...this.usageRetryQueue.values()];
+        this.usageRetryQueue.clear();
+        console.log(`[Platform] Sending ${held.length} usage report${held.length === 1 ? '' : 's'} held while unpaired`);
+        for (const message of held) {
             this.send(message);
         }
     }
@@ -643,12 +879,19 @@ export class PlatformConnection {
         this.stopCredentialsWatch();
         this.outboundQueue = [];
         this.outboundDropped = 0;
+        this.usageRetryQueue.clear();
+        this.usageDropped = 0;
         if (this.reconnectTimeout) {
             clearTimeout(this.reconnectTimeout);
             this.reconnectTimeout = null;
         }
-        this.socketEpoch++;
+        this.nextLiveEpoch();
+        this.dropHandoff();
         this.detachSocket();
+        for (const old of [...this.leaving.keys()]) {
+            this.dropLeaving(old);
+            retireSocket(old);
+        }
         this.isConnected = false;
         this.connectionId = null;
         this.envelopeNegotiated = false;

package/dist/server/services/relay-http-client.js

@@ -0,0 +1,107 @@
+// Copyright (c) 2025-present Minionry, Inc.
+// SPDX-License-Identifier: LicenseRef-Minionry-Software
+import { ReconnectPolicy } from './platform-reconnect.js';
+export const RELAY_HTTP_RETRY_BUDGET_MS = 60_000;
+const RETRYABLE_STATUSES = new Set([502, 503, 504]);
+const RETRYABLE_ERROR_CODES = new Set(['ECONNRESET', 'ECONNREFUSED', 'UND_ERR_SOCKET']);
+export class RelayHttpClient {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    async fetch(input, init, request = {}) {
+        const now = this.options.now ?? Date.now;
+        const startedAt = now();
+        const budgetMs = request.budgetMs ?? this.options.budgetMs ?? RELAY_HTTP_RETRY_BUDGET_MS;
+        const backoff = new ReconnectPolicy({
+            initialMs: this.options.initialDelayMs ?? 500,
+            maxMs: this.options.maxDelayMs ?? 8_000,
+            random: this.options.random,
+        });
+        const replayable = isReplayableBody(init?.body);
+        for (;;) {
+            const attempt = await this.attempt(input, init);
+            const waitMs = replayable ? retryWaitMs(attempt, backoff, now()) : null;
+            if (waitMs === null || now() - startedAt + waitMs > budgetMs)
+                return settle(attempt);
+            if (attempt.ok)
+                await discardBody(attempt.response);
+            await sleep(waitMs, init?.signal ?? undefined);
+        }
+    }
+    async attempt(input, init) {
+        const fetchImpl = this.options.fetchImpl ?? globalThis.fetch;
+        try {
+            return { ok: true, response: await fetchImpl(input, init) };
+        }
+        catch (error) {
+            return { ok: false, error };
+        }
+    }
+}
+function retryWaitMs(attempt, backoff, nowMs) {
+    if (!attempt.ok)
+        return isRetryableNetworkError(attempt.error) ? backoff.nextDelayMs() : null;
+    if (!RETRYABLE_STATUSES.has(attempt.response.status))
+        return null;
+    return Math.max(retryAfterMs(attempt.response, nowMs) ?? 0, backoff.nextDelayMs());
+}
+function settle(attempt) {
+    if (attempt.ok)
+        return attempt.response;
+    throw attempt.error;
+}
+export const relayHttp = new RelayHttpClient();
+function retryAfterMs(response, nowMs) {
+    const raw = response.headers?.get?.('retry-after')?.trim();
+    if (!raw)
+        return null;
+    if (/^\d+$/.test(raw))
+        return Number(raw) * 1_000;
+    const at = Date.parse(raw);
+    return Number.isNaN(at) ? null : Math.max(0, at - nowMs);
+}
+function isRetryableNetworkError(err, depth = 0) {
+    if (depth > 4 || typeof err !== 'object' || err === null)
+        return false;
+    const { code, cause, errors } = err;
+    if (typeof code === 'string' && RETRYABLE_ERROR_CODES.has(code))
+        return true;
+    if (Array.isArray(errors) && errors.some((inner) => isRetryableNetworkError(inner, depth + 1)))
+        return true;
+    return isRetryableNetworkError(cause, depth + 1);
+}
+function isReplayableBody(body) {
+    return (body === undefined ||
+        body === null ||
+        typeof body === 'string' ||
+        body instanceof ArrayBuffer ||
+        ArrayBuffer.isView(body) ||
+        body instanceof URLSearchParams ||
+        body instanceof Blob ||
+        body instanceof FormData);
+}
+async function discardBody(response) {
+    try {
+        await response.body?.cancel();
+    }
+    catch {
+    }
+}
+function sleep(ms, signal) {
+    return new Promise((resolve, reject) => {
+        if (signal?.aborted) {
+            reject(signal.reason);
+            return;
+        }
+        const onAbort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason);
+        };
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort);
+            resolve();
+        }, ms);
+        signal?.addEventListener('abort', onAbort, { once: true });
+    });
+}