@otto-assistant/otto 0.1.2 → 0.7.16

package/dist/gateway-proxy-reconnect.e2e.test.js

@@ -0,0 +1,394 @@
+// Gateway-proxy reconnection test.
+//
+// Parameterized: can test against local digital-twin OR a real production gateway.
+//
+// Local mode (default):
+//   Starts a digital-twin + local gateway-proxy binary, kills and restarts the proxy.
+//
+// Production mode (env vars):
+//   GATEWAY_TEST_URL        - production gateway WS+REST URL (e.g. wss://discord-gateway.otto.dev)
+//   GATEWAY_TEST_TOKEN      - client token (clientId:secret)
+//   GATEWAY_TEST_REDEPLOY   - if "1", runs `fly deploy` between kill/restart instead of local binary
+//
+// Usage:
+//   # Local (needs gateway-proxy binary built):
+//   pnpm test --run src/gateway-proxy-reconnect.e2e.test.ts
+//
+//   # Against production (just connect + kill WS + wait for reconnect):
+//   GATEWAY_TEST_URL=wss://discord-gateway.otto.dev \
+//   GATEWAY_TEST_TOKEN=myclientid:mysecret \
+//   OTTO_TEST_LOGS=1 \
+//   pnpm test --run src/gateway-proxy-reconnect.e2e.test.ts -t "production"
+import fs from 'node:fs';
+import path from 'node:path';
+import net from 'node:net';
+import { spawn } from 'node:child_process';
+import { describe, test, expect, afterAll, afterEach } from 'vitest';
+import { ChannelType, Client, Events, GatewayIntentBits, Partials, } from 'discord.js';
+import { DigitalDiscord } from 'discord-digital-twin/src';
+import { setGlobalDispatcher, Agent } from 'undici';
+// Match production discord-bot.ts settings: increase connection pool and
+// disable timeouts so REST calls don't hang during gateway redeploys.
+setGlobalDispatcher(new Agent({ headersTimeout: 0, bodyTimeout: 0, connections: 500 }));
+// --- Config from env ---
+const PROD_GATEWAY_URL = process.env['GATEWAY_TEST_URL'] || '';
+const PROD_TOKEN = process.env['GATEWAY_TEST_TOKEN'] || '';
+const isProdTest = !!(PROD_GATEWAY_URL && PROD_TOKEN);
+// --- Constants ---
+const DEBUG_BINARY = path.resolve(process.cwd(), '..', 'gateway-proxy', 'target', 'debug', 'gateway-proxy');
+const RELEASE_BINARY = path.resolve(process.cwd(), '..', 'gateway-proxy', 'target', 'release', 'gateway-proxy');
+const BINARY_PATH = fs.existsSync(DEBUG_BINARY) ? DEBUG_BINARY : RELEASE_BINARY;
+const binaryExists = fs.existsSync(BINARY_PATH);
+const GUILD_ID = '800000000000000001';
+const CHANNEL_ID = '800000000000000010';
+const USER_ID = '800000000000000099';
+// --- Helpers ---
+function getAvailablePort() {
+    return new Promise((resolve, reject) => {
+        const srv = net.createServer();
+        srv.listen(0, () => {
+            const addr = srv.address();
+            if (!addr || typeof addr === 'string') {
+                srv.close();
+                reject(new Error('Failed to get port'));
+                return;
+            }
+            const port = addr.port;
+            srv.close(() => {
+                resolve(port);
+            });
+        });
+    });
+}
+async function waitForProxyReady({ port, timeoutMs = 30_000, }) {
+    const start = Date.now();
+    while (Date.now() - start < timeoutMs) {
+        try {
+            const res = await fetch(`http://127.0.0.1:${port}/shard-count`);
+            if (res.ok) {
+                return;
+            }
+        }
+        catch {
+            // not ready yet
+        }
+        await new Promise((r) => {
+            setTimeout(r, 200);
+        });
+    }
+    throw new Error(`gateway-proxy not ready after ${timeoutMs}ms`);
+}
+function startProxy({ configDir, port, twinPort, botToken, gatewayUrl, clients, }) {
+    const config = {
+        log_level: 'debug',
+        token: botToken,
+        intents: 32511,
+        shards: 1,
+        port,
+        validate_token: !clients,
+        gateway_url: gatewayUrl,
+        twilight_http_proxy: `127.0.0.1:${twinPort}`,
+        externally_accessible_url: `ws://127.0.0.1:${port}`,
+        cache: {
+            channels: true,
+            presences: false,
+            emojis: false,
+            current_member: true,
+            members: false,
+            roles: true,
+            scheduled_events: false,
+            stage_instances: false,
+            stickers: false,
+            users: false,
+            voice_states: false,
+        },
+    };
+    if (clients) {
+        config.clients = clients;
+    }
+    const configPath = path.join(configDir, 'config.json');
+    fs.writeFileSync(configPath, JSON.stringify(config, null, 2));
+    const child = spawn(BINARY_PATH, [], {
+        cwd: configDir,
+        stdio: ['ignore', 'pipe', 'pipe'],
+        env: { ...process.env, RUST_LOG: 'debug' },
+    });
+    const showLogs = !!process.env['OTTO_TEST_LOGS'];
+    const logLines = [];
+    child.stdout?.on('data', (data) => {
+        const line = data.toString().trim();
+        if (line) {
+            logLines.push(`[proxy-stdout] ${line}`);
+            if (showLogs) {
+                console.log(`[proxy-stdout] ${line}`);
+            }
+        }
+    });
+    child.stderr?.on('data', (data) => {
+        const line = data.toString().trim();
+        if (line) {
+            logLines.push(`[proxy-stderr] ${line}`);
+            if (showLogs) {
+                console.log(`[proxy-stderr] ${line}`);
+            }
+        }
+    });
+    child._logLines = logLines;
+    return child;
+}
+function dumpProxyLogs(child) {
+    const logLines = child._logLines;
+    if (logLines?.length) {
+        console.log('\n--- proxy logs ---');
+        for (const line of logLines) {
+            console.log(line);
+        }
+        console.log('--- end proxy logs ---\n');
+    }
+}
+function killProxy(child) {
+    return new Promise((resolve) => {
+        if (child.killed || child.exitCode !== null) {
+            resolve();
+            return;
+        }
+        child.once('exit', () => {
+            resolve();
+        });
+        child.kill('SIGTERM');
+    });
+}
+function createDiscordJsClient({ restUrl }) {
+    return new Client({
+        intents: [
+            GatewayIntentBits.Guilds,
+            GatewayIntentBits.GuildMessages,
+            GatewayIntentBits.MessageContent,
+        ],
+        partials: [Partials.Channel, Partials.Message],
+        rest: { api: restUrl },
+    });
+}
+/** Attach all shard event listeners and collect events into an array for diagnosis. */
+function attachEventCollector({ client, label, }) {
+    const events = [];
+    client.on(Events.ShardReady, (shardId) => {
+        events.push(`ShardReady:${shardId}`);
+        console.log(`[${label}] ShardReady shard=${shardId}`);
+    });
+    client.on(Events.ShardReconnecting, (shardId) => {
+        events.push(`ShardReconnecting:${shardId}`);
+        console.log(`[${label}] ShardReconnecting shard=${shardId}`);
+    });
+    client.on(Events.ShardResume, (shardId, replayed) => {
+        events.push(`ShardResume:${shardId}:${replayed}`);
+        console.log(`[${label}] ShardResume shard=${shardId} replayed=${replayed}`);
+    });
+    client.on(Events.ShardDisconnect, (event, shardId) => {
+        events.push(`ShardDisconnect:${shardId}:${event.code}`);
+        console.log(`[${label}] ShardDisconnect shard=${shardId} code=${event.code}`);
+    });
+    client.on(Events.ShardError, (error, shardId) => {
+        events.push(`ShardError:${shardId}:${error.message}`);
+        console.log(`[${label}] ShardError shard=${shardId} error=${error.message}`);
+    });
+    client.on(Events.Invalidated, () => {
+        events.push('Invalidated');
+        console.log(`[${label}] Session invalidated`);
+    });
+    client.on(Events.Error, (error) => {
+        events.push(`Error:${error.message}`);
+        console.log(`[${label}] Client error: ${error.message}`);
+    });
+    client.on(Events.Debug, (info) => {
+        if (info.includes('close') ||
+            info.includes('Close') ||
+            info.includes('CLOSE') ||
+            info.includes('destroy') ||
+            info.includes('session') ||
+            info.includes('Session') ||
+            info.includes('IDENTIFY') ||
+            info.includes('RESUME') ||
+            info.includes('Identifying') ||
+            info.includes('Resuming') ||
+            info.includes('Invalid') ||
+            info.includes('Zombie') ||
+            info.includes('econnr') ||
+            info.includes('fetch') ||
+            info.includes('Fetch') ||
+            info.includes('error') ||
+            info.includes('Error') ||
+            info.includes('Gateway Information') ||
+            info.includes('fully ready')) {
+            events.push(`Debug:${info}`);
+            console.log(`[${label}] Debug: ${info}`);
+        }
+    });
+    return events;
+}
+function waitForClientReady({ client, token, timeoutMs = 30_000, }) {
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+            reject(new Error(`Client did not become ready within ${timeoutMs}ms`));
+        }, timeoutMs);
+        client.once(Events.ClientReady, () => {
+            clearTimeout(timeout);
+            resolve();
+        });
+        client.login(token).catch(reject);
+    });
+}
+function waitForReconnection({ client, events, label, timeoutMs = 30_000, }) {
+    return new Promise((resolve) => {
+        const timeout = setTimeout(() => {
+            console.log(`[${label}] TIMEOUT: client did not reconnect within ${timeoutMs}ms`);
+            console.log(`[${label}] Events so far:`, events);
+            resolve(false);
+        }, timeoutMs);
+        client.on(Events.ShardReady, () => {
+            clearTimeout(timeout);
+            resolve(true);
+        });
+        client.on(Events.ShardResume, () => {
+            clearTimeout(timeout);
+            resolve(true);
+        });
+    });
+}
+// ============================================================================
+// Local tests (digital-twin + local binary)
+// ============================================================================
+const describeLocal = binaryExists ? describe : describe.skip;
+describeLocal('gateway-proxy reconnection (local binary)', () => {
+    let discord;
+    let proxyProcess;
+    let client;
+    let proxyPort;
+    let tmpDir;
+    afterAll(async () => {
+        client?.destroy();
+        if (proxyProcess && !proxyProcess.killed) {
+            proxyProcess.kill('SIGTERM');
+        }
+        await discord?.stop().catch(() => { });
+        if (tmpDir) {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+    }, 5_000);
+    test('reconnects after local proxy restart (REST through proxy, clientId:secret)', async () => {
+        tmpDir = fs.mkdtempSync(path.join(process.cwd(), 'tmp', 'gw-reconnect-'));
+        proxyPort = await getAvailablePort();
+        const CLIENT_ID = 'test-client';
+        const CLIENT_SECRET = 'test-secret-12345';
+        const CLIENT_TOKEN = `${CLIENT_ID}:${CLIENT_SECRET}`;
+        discord = new DigitalDiscord({
+            guilds: [
+                {
+                    id: GUILD_ID,
+                    name: 'Reconnect Test Guild',
+                    ownerId: USER_ID,
+                    channels: [{ id: CHANNEL_ID, name: 'general', type: ChannelType.GuildText }],
+                },
+            ],
+            users: [{ id: USER_ID, username: 'reconnect-tester' }],
+            gatewayUrlOverride: `ws://127.0.0.1:${proxyPort}`,
+            dbUrl: `file:${path.join(tmpDir, 'twin.db')}`,
+        });
+        await discord.start();
+        console.log(`[local] twin at port ${discord.port}, proxy will be at ${proxyPort}`);
+        const proxyConfigDir = path.join(tmpDir, 'proxy-config');
+        fs.mkdirSync(proxyConfigDir, { recursive: true });
+        const proxyOpts = {
+            configDir: proxyConfigDir,
+            port: proxyPort,
+            twinPort: discord.port,
+            botToken: discord.botToken,
+            gatewayUrl: discord.gatewayUrl,
+            clients: { [CLIENT_ID]: { secret: CLIENT_SECRET, guilds: [GUILD_ID] } },
+        };
+        proxyProcess = startProxy(proxyOpts);
+        await waitForProxyReady({ port: proxyPort });
+        // REST through proxy (matches production gateway mode)
+        client = createDiscordJsClient({ restUrl: `http://127.0.0.1:${proxyPort}/api` });
+        const events = attachEventCollector({ client, label: 'local' });
+        await waitForClientReady({ client, token: CLIENT_TOKEN });
+        console.log('[local] Client ready');
+        // Kill proxy
+        const firstProxy = proxyProcess;
+        await killProxy(proxyProcess);
+        await new Promise((r) => { setTimeout(r, 1000); });
+        // Restart proxy on same port
+        proxyProcess = startProxy(proxyOpts);
+        await waitForProxyReady({ port: proxyPort });
+        console.log('[local] Proxy restarted');
+        const reconnected = await waitForReconnection({ client, events, label: 'local' });
+        console.log('[local] All events:', events);
+        if (!reconnected) {
+            dumpProxyLogs(firstProxy);
+            dumpProxyLogs(proxyProcess);
+        }
+        expect(reconnected).toBe(true);
+        expect(client.isReady()).toBe(true);
+    }, 90_000);
+});
+// ============================================================================
+// Production test (real gateway proxy on Fly.io)
+// ============================================================================
+const describeProd = isProdTest ? describe : describe.skip;
+describeProd('gateway-proxy reconnection (production)', () => {
+    let client;
+    afterEach(() => {
+        client?.destroy();
+    });
+    test('discord.js reconnects to production gateway after fly deploy', async () => {
+        // Derive REST URL from gateway WS URL (wss://host → https://host/api)
+        const parsedUrl = new URL(PROD_GATEWAY_URL);
+        parsedUrl.protocol = parsedUrl.protocol === 'wss:' ? 'https:' : 'http:';
+        const restUrl = `${parsedUrl.origin}/api`;
+        console.log(`[prod] Gateway URL: ${PROD_GATEWAY_URL}`);
+        console.log(`[prod] REST URL: ${restUrl}`);
+        console.log(`[prod] Token: ${PROD_TOKEN.slice(0, 8)}...`);
+        client = createDiscordJsClient({ restUrl });
+        const events = attachEventCollector({ client, label: 'prod' });
+        // Connect to production gateway
+        await waitForClientReady({ client, token: PROD_TOKEN, timeoutMs: 60_000 });
+        console.log(`[prod] Client ready. Guilds: ${client.guilds.cache.size}`);
+        expect(client.guilds.cache.size).toBeGreaterThanOrEqual(1);
+        // Deploy the gateway (restarts the Fly machine).
+        // Uses `pnpm run deploy` which cross-compiles Rust locally then deploys
+        // via Dockerfile.fly. Never use `fly deploy` directly.
+        console.log('[prod] Running pnpm run deploy to restart gateway...');
+        const deployResult = await new Promise((resolve) => {
+            const deployChild = spawn('pnpm', ['run', 'deploy'], {
+                cwd: path.resolve(process.cwd(), '..', 'gateway-proxy'),
+                stdio: ['ignore', 'pipe', 'pipe'],
+            });
+            let output = '';
+            deployChild.stdout?.on('data', (d) => {
+                const line = d.toString();
+                output += line;
+                console.log(`[fly] ${line.trim()}`);
+            });
+            deployChild.stderr?.on('data', (d) => {
+                const line = d.toString();
+                output += line;
+                console.log(`[fly-err] ${line.trim()}`);
+            });
+            deployChild.on('exit', (code) => {
+                resolve({ code: code ?? 1, output });
+            });
+        });
+        console.log(`[prod] Deploy exited with code ${deployResult.code}`);
+        // Wait for reconnection (longer timeout for production)
+        const reconnected = await waitForReconnection({
+            client,
+            events,
+            label: 'prod',
+            timeoutMs: 120_000,
+        });
+        console.log('[prod] All events:', events);
+        expect(reconnected).toBe(true);
+        expect(client.isReady()).toBe(true);
+        expect(client.guilds.cache.size).toBeGreaterThanOrEqual(1);
+    }, 300_000);
+});