mono-pilot 0.2.9 → 0.2.12

package/dist/src/cluster_v2/services/discord/collector.js

@@ -0,0 +1,569 @@
+import { appendFile, mkdir } from "node:fs/promises";
+import { dirname, join, parse } from "node:path";
+import { extractDiscordCollectorConfig, } from "../../../config/discord.js";
+import { loadMonoPilotConfigObject } from "../../../config/mono-pilot.js";
+import { emitClusterV2DiscordChannelBatch } from "../../events.js";
+import { logClusterEvent } from "../../observability.js";
+import { getAuthStorePath, readDiscordAuthToken, writeDiscordAuthToken, } from "./auth-store.js";
+import { exchangeDiscordAuthorizeCode, tryRefreshDiscordToken } from "./oauth.js";
+import { DiscordRpcClient } from "./rpc-client.js";
+const INITIAL_RECONNECT_DELAY_MS = 1_000;
+const TOKEN_EXPIRY_SKEW_MS = 60_000;
+function isTerminalAuthErrorMessage(message) {
+    return (message.includes("invalid_client") ||
+        message.includes("access_denied") ||
+        message.includes("invalid_grant"));
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function readString(value) {
+    return typeof value === "string" && value.length > 0 ? value : null;
+}
+function readAttachmentUrls(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    const urls = [];
+    for (const item of value) {
+        if (!isRecord(item)) {
+            continue;
+        }
+        const url = readString(item.url) ?? readString(item.proxy_url);
+        if (url) {
+            urls.push(url);
+        }
+    }
+    return urls;
+}
+function readEmbedUrls(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    const urls = [];
+    for (const item of value) {
+        if (!isRecord(item)) {
+            continue;
+        }
+        const candidates = [
+            readString(item.url),
+            isRecord(item.image) ? readString(item.image.url) : null,
+            isRecord(item.thumbnail) ? readString(item.thumbnail.url) : null,
+            isRecord(item.video) ? readString(item.video.url) : null,
+        ];
+        for (const candidate of candidates) {
+            if (candidate) {
+                urls.push(candidate);
+            }
+        }
+    }
+    return urls;
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function isTokenLikelyExpired(expiresAt) {
+    if (!expiresAt) {
+        return false;
+    }
+    const expiresTime = Date.parse(expiresAt);
+    if (!Number.isFinite(expiresTime)) {
+        return false;
+    }
+    return expiresTime - Date.now() <= TOKEN_EXPIRY_SKEW_MS;
+}
+function formatLocalDateStamp(date) {
+    const year = String(date.getFullYear());
+    const month = String(date.getMonth() + 1).padStart(2, "0");
+    const day = String(date.getDate()).padStart(2, "0");
+    return `${year}-${month}-${day}`;
+}
+class JsonlWriter {
+    outputPath;
+    logContext;
+    queue = Promise.resolve();
+    dirReady = false;
+    outputDir;
+    outputBaseName;
+    outputExtension;
+    currentDateStamp = null;
+    currentOutputPath = null;
+    constructor(outputPath, logContext) {
+        this.outputPath = outputPath;
+        this.logContext = logContext;
+        const parsed = parse(outputPath);
+        this.outputDir = parsed.dir;
+        this.outputBaseName = parsed.name || parsed.base || "messages";
+        this.outputExtension = parsed.ext || ".jsonl";
+    }
+    append(record) {
+        this.queue = this.queue
+            .then(async () => {
+            if (!this.dirReady) {
+                await mkdir(this.outputDir || dirname(this.outputPath), { recursive: true });
+                this.dirReady = true;
+            }
+            const currentPath = this.resolveCurrentOutputPath();
+            await appendFile(currentPath, `${JSON.stringify(record)}\n`, "utf-8");
+        })
+            .catch((error) => {
+            logClusterEvent("warn", "discord_collector_persist_failed", this.logContext, {
+                error: error instanceof Error ? error.message : String(error),
+                outputPath: this.outputPath,
+                currentOutputPath: this.currentOutputPath,
+            });
+        });
+    }
+    resolveCurrentOutputPath(now = new Date()) {
+        const dateStamp = formatLocalDateStamp(now);
+        if (this.currentDateStamp === dateStamp && this.currentOutputPath) {
+            return this.currentOutputPath;
+        }
+        this.currentDateStamp = dateStamp;
+        this.currentOutputPath = join(this.outputDir, `${this.outputBaseName}.${dateStamp}${this.outputExtension}`);
+        return this.currentOutputPath;
+    }
+    async flush() {
+        await this.queue;
+    }
+}
+class DiscordCollector {
+    config;
+    writer;
+    lifecycleContext;
+    loop = null;
+    closed = false;
+    client = null;
+    seq = 0;
+    reconnectDelayMs = INITIAL_RECONNECT_DELAY_MS;
+    authToken = null;
+    warnedOauthConfig = false;
+    channelContextCache = new Map();
+    channelContextInFlight = new Map();
+    guildNameCache = new Map();
+    guildNameInFlight = new Map();
+    channelAliasById = new Map();
+    channelMessageCounts = new Map();
+    channelBatchSequences = new Map();
+    descriptor = {
+        name: "discord_intel",
+        version: "v1",
+        capabilities: {
+            mode: "leader_local",
+        },
+    };
+    constructor(config, context) {
+        this.config = config;
+        this.lifecycleContext = {
+            ...context,
+            role: context.role ? `${context.role}:discord_intel` : "discord_intel",
+        };
+        for (const channel of this.config.channels) {
+            if (channel.alias) {
+                this.channelAliasById.set(channel.id, channel.alias);
+            }
+        }
+        this.writer = new JsonlWriter(this.config.outputPath, this.lifecycleContext);
+        this.descriptor = {
+            name: "discord_intel",
+            version: "v1",
+            capabilities: {
+                mode: "leader_local",
+                events: this.config.events,
+                channelCount: this.config.channels.length,
+                outputPath: this.config.outputPath,
+            },
+        };
+    }
+    start() {
+        if (this.loop) {
+            return;
+        }
+        this.loop = this.runLoop();
+    }
+    async close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        this.client?.close();
+        if (this.loop) {
+            await this.loop;
+        }
+        await this.writer.flush();
+        logClusterEvent("info", "discord_collector_stopped", this.lifecycleContext);
+    }
+    async runLoop() {
+        logClusterEvent("info", "discord_collector_started", this.lifecycleContext, {
+            outputPath: this.config.outputPath,
+            events: this.config.events,
+            channelCount: this.config.channels.length,
+        });
+        while (!this.closed) {
+            try {
+                await this.runOnce();
+                this.reconnectDelayMs = INITIAL_RECONNECT_DELAY_MS;
+            }
+            catch (error) {
+                if (this.closed) {
+                    break;
+                }
+                const message = error instanceof Error ? error.message : String(error);
+                if (isTerminalAuthErrorMessage(message)) {
+                    logClusterEvent("error", "discord_collector_terminal_auth_error", this.lifecycleContext, {
+                        error: message,
+                        action: "stop_retries_until_restart",
+                        authStorePath: getAuthStorePath(),
+                    });
+                    break;
+                }
+                logClusterEvent("warn", "discord_collector_cycle_failed", this.lifecycleContext, {
+                    error: message,
+                    retryInMs: this.reconnectDelayMs,
+                });
+                await sleep(this.reconnectDelayMs);
+                this.reconnectDelayMs = Math.min(this.reconnectDelayMs * 2, this.config.maxReconnectDelayMs);
+            }
+        }
+    }
+    async runOnce() {
+        if (this.closed) {
+            return;
+        }
+        const client = new DiscordRpcClient({
+            clientId: this.config.clientId ?? "",
+        });
+        this.client = client;
+        const unsubs = this.config.events.map((eventName) => client.onEvent(eventName, (payload) => {
+            void this.persistEvent(client, eventName, payload);
+        }));
+        try {
+            await client.connect();
+            let accessToken = await this.resolveAccessToken(client);
+            try {
+                await client.authenticate(accessToken);
+            }
+            catch (error) {
+                if (this.config.accessToken) {
+                    throw error;
+                }
+                logClusterEvent("warn", "discord_collector_auth_token_rejected", this.lifecycleContext, {
+                    error: error instanceof Error ? error.message : String(error),
+                });
+                const refreshed = await this.tryRefreshToken();
+                if (refreshed) {
+                    accessToken = refreshed.accessToken;
+                    await client.authenticate(accessToken);
+                }
+                else {
+                    const interactive = await this.authorizeAndPersist(client);
+                    accessToken = interactive.accessToken;
+                    await client.authenticate(accessToken);
+                }
+            }
+            for (const channel of this.config.channels) {
+                for (const eventName of this.config.events) {
+                    await client.subscribe(eventName, { channel_id: channel.id });
+                }
+            }
+            logClusterEvent("info", "discord_collector_connected", this.lifecycleContext, {
+                channels: this.config.channels.length,
+                events: this.config.events,
+            });
+            await client.waitForDisconnect();
+            if (!this.closed) {
+                throw new Error("discord ipc disconnected");
+            }
+        }
+        finally {
+            for (const unsub of unsubs) {
+                unsub();
+            }
+            client.close();
+            if (this.client === client) {
+                this.client = null;
+            }
+        }
+    }
+    async resolveAccessToken(client) {
+        if (this.config.accessToken) {
+            return this.config.accessToken;
+        }
+        await this.loadCachedAuthToken();
+        if (this.authToken && isTokenLikelyExpired(this.authToken.expiresAt)) {
+            const refreshed = await this.tryRefreshToken();
+            if (refreshed) {
+                return refreshed.accessToken;
+            }
+        }
+        if (this.authToken?.accessToken) {
+            return this.authToken.accessToken;
+        }
+        const interactive = await this.authorizeAndPersist(client);
+        return interactive.accessToken;
+    }
+    async loadCachedAuthToken() {
+        if (this.authToken || this.config.accessToken || !this.config.clientId) {
+            return;
+        }
+        try {
+            this.authToken = await readDiscordAuthToken(this.config.clientId);
+            if (this.authToken) {
+                logClusterEvent("info", "discord_collector_auth_cache_hit", this.lifecycleContext, {
+                    authStorePath: getAuthStorePath(),
+                });
+            }
+            else {
+                logClusterEvent("info", "discord_collector_auth_cache_miss", this.lifecycleContext, {
+                    authStorePath: getAuthStorePath(),
+                });
+            }
+        }
+        catch (error) {
+            logClusterEvent("warn", "discord_collector_auth_cache_load_failed", this.lifecycleContext, {
+                error: error instanceof Error ? error.message : String(error),
+                authStorePath: getAuthStorePath(),
+            });
+        }
+    }
+    async tryRefreshToken() {
+        if (!this.config.clientId || !this.authToken?.refreshToken) {
+            return null;
+        }
+        const refreshed = await tryRefreshDiscordToken({
+            clientId: this.config.clientId,
+            clientSecret: this.config.clientSecret,
+            redirectUri: this.config.redirectUri,
+            scopes: this.config.scopes,
+            refreshToken: this.authToken.refreshToken,
+        });
+        if (!refreshed) {
+            return null;
+        }
+        const persisted = await this.persistAuthToken(refreshed);
+        logClusterEvent("info", "discord_collector_auth_token_refreshed", this.lifecycleContext, {
+            authStorePath: getAuthStorePath(),
+        });
+        return persisted;
+    }
+    async authorizeAndPersist(client) {
+        if (!this.config.clientId) {
+            throw new Error("discord collector requires clientId");
+        }
+        if (!this.warnedOauthConfig && !this.config.clientSecret && !this.config.accessToken) {
+            this.warnedOauthConfig = true;
+            logClusterEvent("warn", "discord_collector_missing_client_secret", this.lifecycleContext, {
+                hint: "oauth2/token authorization_code usually requires clientSecret (and sometimes redirectUri)",
+            });
+        }
+        logClusterEvent("info", "discord_collector_authorize_start", this.lifecycleContext, {
+            authStorePath: getAuthStorePath(),
+            scopes: this.config.scopes,
+        });
+        const { code } = await client.authorize(this.config.scopes);
+        const exchanged = await exchangeDiscordAuthorizeCode({
+            clientId: this.config.clientId,
+            clientSecret: this.config.clientSecret,
+            redirectUri: this.config.redirectUri,
+            scopes: this.config.scopes,
+            code,
+        });
+        const persisted = await this.persistAuthToken(exchanged);
+        logClusterEvent("info", "discord_collector_authorize_success", this.lifecycleContext, {
+            authStorePath: getAuthStorePath(),
+        });
+        return persisted;
+    }
+    async persistAuthToken(token) {
+        if (!this.config.clientId) {
+            throw new Error("discord collector requires clientId");
+        }
+        try {
+            const persisted = await writeDiscordAuthToken(this.config.clientId, token);
+            this.authToken = persisted;
+            return persisted;
+        }
+        catch (error) {
+            logClusterEvent("warn", "discord_collector_auth_cache_write_failed", this.lifecycleContext, {
+                error: error instanceof Error ? error.message : String(error),
+                authStorePath: getAuthStorePath(),
+            });
+            const fallback = {
+                ...token,
+                updatedAt: new Date().toISOString(),
+            };
+            this.authToken = fallback;
+            return fallback;
+        }
+    }
+    async persistEvent(client, eventName, payload) {
+        const payloadRecord = isRecord(payload) ? payload : null;
+        const messageRecord = payloadRecord && isRecord(payloadRecord.message) ? payloadRecord.message : payloadRecord;
+        const authorRecord = messageRecord && isRecord(messageRecord.author) ? messageRecord.author : null;
+        const channelId = readString(payloadRecord?.channel_id) ?? readString(messageRecord?.channel_id) ?? null;
+        const channelAlias = channelId ? (this.channelAliasById.get(channelId) ?? null) : null;
+        let guildId = readString(payloadRecord?.guild_id) ?? readString(messageRecord?.guild_id) ?? null;
+        let channelName = null;
+        let guildName = null;
+        if (channelId) {
+            try {
+                const context = await this.resolveChannelContext(client, channelId, guildId);
+                channelName = context.channelName;
+                guildId = guildId ?? context.guildId;
+                guildName = context.guildName;
+            }
+            catch (error) {
+                logClusterEvent("warn", "discord_collector_channel_context_failed", this.lifecycleContext, {
+                    channelId,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        if (!guildName && guildId) {
+            try {
+                guildName = await this.resolveGuildName(client, guildId);
+            }
+            catch (error) {
+                logClusterEvent("warn", "discord_collector_guild_context_failed", this.lifecycleContext, {
+                    guildId,
+                    error: error instanceof Error ? error.message : String(error),
+                });
+            }
+        }
+        const record = {
+            seq: ++this.seq,
+            receivedAt: new Date().toISOString(),
+            event: eventName,
+            channelId,
+            channelAlias,
+            channelName,
+            guildId,
+            guildName,
+            messageId: readString(messageRecord?.id),
+            authorId: readString(authorRecord?.id),
+            authorUsername: readString(authorRecord?.username),
+            content: readString(messageRecord?.content),
+            attachments: readAttachmentUrls(messageRecord?.attachments),
+            embeds: readEmbedUrls(messageRecord?.embeds),
+        };
+        if (this.config.includeRawPayload) {
+            record.raw = payload;
+        }
+        this.writer.append(record);
+        this.maybeEmitChannelBatchEvent(eventName, record);
+    }
+    maybeEmitChannelBatchEvent(eventName, record) {
+        if (eventName !== "MESSAGE_CREATE") {
+            return;
+        }
+        if (!record.channelId) {
+            return;
+        }
+        const channelId = record.channelId;
+        const nextCount = (this.channelMessageCounts.get(channelId) ?? 0) + 1;
+        if (nextCount < this.config.systemEventBatchSize) {
+            this.channelMessageCounts.set(channelId, nextCount);
+            return;
+        }
+        this.channelMessageCounts.set(channelId, 0);
+        const sequence = (this.channelBatchSequences.get(channelId) ?? 0) + 1;
+        this.channelBatchSequences.set(channelId, sequence);
+        emitClusterV2DiscordChannelBatch({
+            scope: this.lifecycleContext.scope ?? "default",
+            channelId,
+            channelAlias: record.channelAlias ?? undefined,
+            channelName: record.channelName ?? undefined,
+            guildName: record.guildName ?? undefined,
+            count: this.config.systemEventBatchSize,
+            sequence,
+        });
+    }
+    async resolveChannelContext(client, channelId, fallbackGuildId) {
+        const cached = this.channelContextCache.get(channelId);
+        if (cached) {
+            return cached;
+        }
+        const inFlight = this.channelContextInFlight.get(channelId);
+        if (inFlight) {
+            return inFlight;
+        }
+        const promise = (async () => {
+            const channel = await client.getChannel(channelId);
+            const guildId = channel.guildId ?? fallbackGuildId;
+            const guildName = guildId ? await this.resolveGuildName(client, guildId) : null;
+            const context = {
+                channelName: channel.name,
+                guildId,
+                guildName,
+            };
+            this.channelContextCache.set(channelId, context);
+            return context;
+        })()
+            .finally(() => {
+            this.channelContextInFlight.delete(channelId);
+        });
+        this.channelContextInFlight.set(channelId, promise);
+        return promise;
+    }
+    async resolveGuildName(client, guildId) {
+        if (this.guildNameCache.has(guildId)) {
+            return this.guildNameCache.get(guildId) ?? null;
+        }
+        const inFlight = this.guildNameInFlight.get(guildId);
+        if (inFlight) {
+            return inFlight;
+        }
+        const promise = (async () => {
+            const guild = await client.getGuild(guildId);
+            const name = guild.name;
+            this.guildNameCache.set(guildId, name);
+            return name;
+        })()
+            .catch((error) => {
+            logClusterEvent("warn", "discord_collector_get_guild_failed", this.lifecycleContext, {
+                guildId,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            this.guildNameCache.set(guildId, null);
+            return null;
+        })
+            .finally(() => {
+            this.guildNameInFlight.delete(guildId);
+        });
+        this.guildNameInFlight.set(guildId, promise);
+        return promise;
+    }
+}
+function validateCollectorConfig(config, context) {
+    if (!config.enabled) {
+        return false;
+    }
+    if (!config.clientId) {
+        logClusterEvent("warn", "discord_collector_disabled_missing_client_id", context);
+        return false;
+    }
+    if (config.channels.length === 0) {
+        logClusterEvent("warn", "discord_collector_disabled_missing_channels", context);
+        return false;
+    }
+    return true;
+}
+export async function maybeStartDiscordCollector(context) {
+    let configObject;
+    try {
+        configObject = await loadMonoPilotConfigObject();
+    }
+    catch (error) {
+        logClusterEvent("warn", "discord_collector_config_load_failed", context, {
+            error: error instanceof Error ? error.message : String(error),
+        });
+        return null;
+    }
+    const config = extractDiscordCollectorConfig(configObject);
+    if (!validateCollectorConfig(config, context)) {
+        return null;
+    }
+    const collector = new DiscordCollector(config, context);
+    collector.start();
+    return collector;
+}

package/dist/src/cluster_v2/services/discord/index.js

@@ -0,0 +1 @@
+export { maybeStartDiscordCollector } from "./collector.js";

package/dist/src/cluster_v2/services/discord/oauth.js

@@ -0,0 +1,87 @@
+const DISCORD_OAUTH_TOKEN_URL = "https://discord.com/api/oauth2/token";
+function readString(value) {
+    if (typeof value !== "string") {
+        return undefined;
+    }
+    const trimmed = value.trim();
+    return trimmed.length > 0 ? trimmed : undefined;
+}
+function toTokenRecord(payload) {
+    const accessToken = readString(payload.access_token);
+    if (!accessToken) {
+        return null;
+    }
+    let expiresAt;
+    if (typeof payload.expires_in === "number" && Number.isFinite(payload.expires_in)) {
+        expiresAt = new Date(Date.now() + Math.max(0, payload.expires_in) * 1000).toISOString();
+    }
+    return {
+        accessToken,
+        refreshToken: readString(payload.refresh_token),
+        tokenType: readString(payload.token_type),
+        scope: readString(payload.scope),
+        expiresAt,
+    };
+}
+function buildRequestBody(grantType, params) {
+    const body = new URLSearchParams();
+    body.set("client_id", params.clientId);
+    body.set("grant_type", grantType);
+    if (params.clientSecret) {
+        body.set("client_secret", params.clientSecret);
+    }
+    if (params.redirectUri) {
+        body.set("redirect_uri", params.redirectUri);
+    }
+    if (params.scopes && params.scopes.length > 0) {
+        body.set("scope", params.scopes.join(" "));
+    }
+    if (grantType === "authorization_code") {
+        body.set("code", params.code);
+    }
+    else {
+        body.set("refresh_token", params.refreshToken);
+    }
+    return body;
+}
+async function exchangeToken(grantType, params, strict) {
+    const response = await fetch(DISCORD_OAUTH_TOKEN_URL, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/x-www-form-urlencoded",
+        },
+        body: buildRequestBody(grantType, params),
+    });
+    let payload = {};
+    try {
+        payload = (await response.json());
+    }
+    catch {
+        if (strict) {
+            throw new Error(`discord oauth token exchange failed: HTTP ${response.status}`);
+        }
+        return null;
+    }
+    const token = toTokenRecord(payload);
+    if (response.ok && token) {
+        return token;
+    }
+    const message = readString(payload.error_description) ??
+        readString(payload.message) ??
+        readString(payload.error) ??
+        `HTTP ${response.status}`;
+    if (strict) {
+        throw new Error(`discord oauth token exchange failed: ${message}`);
+    }
+    return null;
+}
+export async function exchangeDiscordAuthorizeCode(params) {
+    const token = await exchangeToken("authorization_code", params, true);
+    if (!token) {
+        throw new Error("discord oauth authorization_code exchange returned no token");
+    }
+    return token;
+}
+export async function tryRefreshDiscordToken(params) {
+    return exchangeToken("refresh_token", params, false);
+}