@otto-assistant/bridge 0.4.92

package/dist/forum-sync/sync-to-files.js

@@ -0,0 +1,190 @@
+// Discord -> filesystem sync.
+// Fetches forum threads from Discord and writes them as markdown files.
+// Handles incremental sync (skip unchanged threads) and stale file cleanup.
+import fs from 'node:fs';
+import path from 'node:path';
+import { createLogger } from '../logger.js';
+import { buildMessageSections, extractProjectChannelFromContent, formatMessageSection, getStringValue, stringifyFrontmatter, } from './markdown.js';
+import { ensureDirectory, fetchForumThreads, fetchThreadMessages, getCanonicalThreadFilePath, loadExistingForumFiles, resolveForumChannel, } from './discord-operations.js';
+import { DEFAULT_RATE_LIMIT_DELAY_MS, ForumSyncOperationError, addIgnoredPath, delay, } from './types.js';
+const forumLogger = createLogger('FORUM');
+function resolveTagNames({ thread, forumChannel, }) {
+    const availableTagsById = new Map(forumChannel.availableTags.map((tag) => [tag.id, tag.name]));
+    return thread.appliedTags
+        .map((tagId) => availableTagsById.get(tagId))
+        .filter((tagName) => Boolean(tagName));
+}
+function resolveSubfolderForThread({ existingSubfolder, thread, forumChannel, }) {
+    const hasGlobalTag = resolveTagNames({ thread, forumChannel }).some((tagName) => tagName.toLowerCase().trim() === 'global');
+    if (hasGlobalTag)
+        return 'global';
+    if (existingSubfolder)
+        return existingSubfolder;
+    return undefined;
+}
+function buildFrontmatter({ thread, forumChannel, sections, project, projectChannelId, }) {
+    const firstSection = sections[0];
+    const createdTimestamp = thread.createdTimestamp ?? Date.now();
+    const latestTimestamp = sections.reduce((latest, section) => {
+        const created = Date.parse(section.createdAt);
+        const edited = section.editedAt ? Date.parse(section.editedAt) : 0;
+        return Math.max(latest, created, edited);
+    }, createdTimestamp);
+    return {
+        title: thread.name,
+        threadId: thread.id,
+        forumChannelId: forumChannel.id,
+        tags: resolveTagNames({ thread, forumChannel }),
+        author: firstSection?.authorName || '',
+        authorId: firstSection?.authorId || '',
+        createdAt: thread.createdAt?.toISOString() ||
+            new Date(createdTimestamp).toISOString(),
+        lastUpdated: new Date(latestTimestamp).toISOString(),
+        lastMessageId: thread.lastMessageId,
+        lastSyncedAt: new Date().toISOString(),
+        messageCount: sections.length,
+        ...(project && { project }),
+        ...(projectChannelId && { projectChannelId }),
+    };
+}
+export async function syncSingleThreadToFile({ thread, forumChannel, outputDir, runtimeState, previousFilePath, subfolder, project, projectChannelId, }) {
+    const messages = await fetchThreadMessages({ thread });
+    if (messages instanceof Error)
+        return messages;
+    // Extract projectChannelId from the starter message footer if not already known.
+    // This allows Discord -> file sync to reconstruct the correct subfolder
+    // even when no local .md file exists (e.g. fresh machine, deleted files).
+    let resolvedProjectChannelId = projectChannelId;
+    let resolvedSubfolder = subfolder;
+    const sections = buildMessageSections({ messages });
+    const firstSection = sections[0];
+    if (firstSection) {
+        const { cleanContent, projectChannelId: footerChannelId } = extractProjectChannelFromContent({ content: firstSection.content });
+        firstSection.content = cleanContent;
+        if (footerChannelId && !resolvedProjectChannelId) {
+            resolvedProjectChannelId = footerChannelId;
+        }
+        if (resolvedProjectChannelId && !resolvedSubfolder) {
+            resolvedSubfolder = resolvedProjectChannelId;
+        }
+    }
+    // Ensure subfolder directory exists when writing into a nested path
+    if (resolvedSubfolder) {
+        const subDir = path.join(outputDir, resolvedSubfolder);
+        const ensureResult = await ensureDirectory({ directory: subDir });
+        if (ensureResult instanceof Error)
+            return ensureResult;
+    }
+    const body = sections
+        .map((section) => formatMessageSection({ section }))
+        .join('\n\n---\n\n');
+    const frontmatter = buildFrontmatter({
+        thread,
+        forumChannel,
+        sections,
+        project,
+        projectChannelId: resolvedProjectChannelId,
+    });
+    const markdown = stringifyFrontmatter({ frontmatter, body });
+    const targetPath = getCanonicalThreadFilePath({
+        outputDir,
+        threadId: thread.id,
+        subfolder: resolvedSubfolder,
+    });
+    addIgnoredPath({ runtimeState, filePath: targetPath });
+    const writeResult = await fs.promises
+        .writeFile(targetPath, markdown, 'utf8')
+        .catch((cause) => {
+        return new ForumSyncOperationError({
+            forumChannelId: forumChannel.id,
+            reason: `failed to write ${targetPath}`,
+            cause,
+        });
+    });
+    if (writeResult instanceof Error)
+        return writeResult;
+    // Clean up old file if thread was renamed (file path changed)
+    if (previousFilePath &&
+        previousFilePath !== targetPath &&
+        fs.existsSync(previousFilePath)) {
+        addIgnoredPath({ runtimeState, filePath: previousFilePath });
+        await fs.promises.unlink(previousFilePath).catch((cause) => {
+            forumLogger.warn(`Failed to remove old forum file ${previousFilePath}:`, cause);
+        });
+    }
+}
+export async function syncForumToFiles({ discordClient, forumChannelId, outputDir, forceFullRefresh = false, forceThreadIds, runtimeState, }) {
+    const ensureResult = await ensureDirectory({ directory: outputDir });
+    if (ensureResult instanceof Error) {
+        return new ForumSyncOperationError({
+            forumChannelId,
+            reason: `failed to create output directory ${outputDir}`,
+            cause: ensureResult,
+        });
+    }
+    const forumChannel = await resolveForumChannel({
+        discordClient,
+        forumChannelId,
+    });
+    if (forumChannel instanceof Error)
+        return forumChannel;
+    const threads = await fetchForumThreads({ forumChannel });
+    if (threads instanceof Error)
+        return threads;
+    const existingFiles = await loadExistingForumFiles({ outputDir });
+    const existingByThreadId = new Map(existingFiles.map((entry) => [entry.threadId, entry]));
+    const result = { synced: 0, skipped: 0, deleted: 0 };
+    for (const thread of threads) {
+        const existing = existingByThreadId.get(thread.id);
+        const savedLastMessageId = getStringValue({ value: existing?.frontmatter.lastMessageId }) || null;
+        const isForced = forceFullRefresh || Boolean(forceThreadIds?.has(thread.id));
+        if (!isForced &&
+            savedLastMessageId &&
+            savedLastMessageId === thread.lastMessageId) {
+            result.skipped += 1;
+            continue;
+        }
+        const syncResult = await syncSingleThreadToFile({
+            thread,
+            forumChannel,
+            outputDir,
+            runtimeState,
+            previousFilePath: existing?.filePath,
+            subfolder: resolveSubfolderForThread({
+                existingSubfolder: existing?.subfolder,
+                thread,
+                forumChannel,
+            }),
+            project: getStringValue({ value: existing?.frontmatter.project }),
+            projectChannelId: getStringValue({
+                value: existing?.frontmatter.projectChannelId,
+            }),
+        });
+        if (syncResult instanceof Error)
+            return syncResult;
+        result.synced += 1;
+        await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS });
+    }
+    // Delete files for threads that no longer exist in Discord
+    const liveThreadIds = new Set(threads.map((thread) => thread.id));
+    for (const existing of existingFiles) {
+        if (liveThreadIds.has(existing.threadId))
+            continue;
+        if (!fs.existsSync(existing.filePath))
+            continue;
+        addIgnoredPath({ runtimeState, filePath: existing.filePath });
+        const deleteResult = await fs.promises
+            .unlink(existing.filePath)
+            .catch((cause) => {
+            return new ForumSyncOperationError({
+                forumChannelId,
+                reason: `failed deleting stale file ${existing.filePath}`,
+                cause,
+            });
+        });
+        if (deleteResult instanceof Error)
+            return deleteResult;
+        result.deleted += 1;
+    }
+    return result;
+}

package/dist/forum-sync/types.js

@@ -0,0 +1,53 @@
+// Type definitions, tagged errors, and constants for forum sync.
+// All shared types and error classes live here to avoid circular dependencies
+// between the sync modules.
+import * as errore from 'errore';
+// ═══════════════════════════════════════════════════════════════════════════
+// CONSTANTS
+// ═══════════════════════════════════════════════════════════════════════════
+export const DEFAULT_DEBOUNCE_MS = 800;
+export const DEFAULT_RATE_LIMIT_DELAY_MS = 250;
+export const WRITE_IGNORE_TTL_MS = 2_000;
+// ═══════════════════════════════════════════════════════════════════════════
+// TAGGED ERRORS
+// ═══════════════════════════════════════════════════════════════════════════
+export class ForumChannelResolveError extends errore.createTaggedError({
+    name: 'ForumChannelResolveError',
+    message: 'Could not resolve forum channel $forumChannelId',
+}) {
+}
+export class ForumSyncOperationError extends errore.createTaggedError({
+    name: 'ForumSyncOperationError',
+    message: 'Forum sync operation failed for forum $forumChannelId: $reason',
+}) {
+}
+export class ForumFrontmatterParseError extends errore.createTaggedError({
+    name: 'ForumFrontmatterParseError',
+    message: 'Failed to parse frontmatter: $reason',
+}) {
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// SHARED UTILITIES
+// ═══════════════════════════════════════════════════════════════════════════
+export function delay({ ms }) {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
+/** Mark a file path as recently written so the file watcher ignores it. */
+export function addIgnoredPath({ runtimeState, filePath, }) {
+    if (!runtimeState)
+        return;
+    runtimeState.ignoredPaths.set(filePath, Date.now() + WRITE_IGNORE_TTL_MS);
+}
+/** Check if a file path was recently written by us and should be ignored. */
+export function shouldIgnorePath({ runtimeState, filePath, }) {
+    const expiresAt = runtimeState.ignoredPaths.get(filePath);
+    if (!expiresAt)
+        return false;
+    if (expiresAt < Date.now()) {
+        runtimeState.ignoredPaths.delete(filePath);
+        return false;
+    }
+    return true;
+}

package/dist/forum-sync/watchers.js

@@ -0,0 +1,307 @@
+// Runtime state management, file watchers, and Discord event listeners.
+// Manages the lifecycle of forum sync: initial sync, live Discord event handling,
+// file system watcher for bidirectional sync, and debounced sync scheduling.
+import fs from 'node:fs';
+import parcelWatcher from '@parcel/watcher';
+import { ChannelType, Events, } from 'discord.js';
+import { createLogger } from '../logger.js';
+import { readForumSyncConfig } from './config.js';
+import { ensureDirectory, getCanonicalThreadFilePath, } from './discord-operations.js';
+import { syncForumToFiles } from './sync-to-files.js';
+import { syncFilesToForum } from './sync-to-discord.js';
+import { DEFAULT_DEBOUNCE_MS, ForumSyncOperationError, addIgnoredPath, shouldIgnorePath, } from './types.js';
+const forumLogger = createLogger('FORUM');
+// ═══════════════════════════════════════════════════════════════════════════
+// MODULE STATE
+// ═══════════════════════════════════════════════════════════════════════════
+const forumStateById = new Map();
+const watcherUnsubscribeByForumId = new Map();
+let discordListenersRegistered = false;
+// ═══════════════════════════════════════════════════════════════════════════
+// RUNTIME STATE
+// ═══════════════════════════════════════════════════════════════════════════
+function buildRuntimeState({ forumChannelId, outputDir, direction, }) {
+    return {
+        forumChannelId,
+        outputDir,
+        direction,
+        dirtyThreadIds: new Set(),
+        ignoredPaths: new Map(),
+        queuedFileEvents: new Map(),
+        discordDebounceTimer: null,
+        fileDebounceTimer: null,
+    };
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// FILE WATCHER EVENT HANDLING
+// ═══════════════════════════════════════════════════════════════════════════
+async function runQueuedFileEvents({ runtimeState, discordClient, }) {
+    const queuedEntries = Array.from(runtimeState.queuedFileEvents.entries());
+    runtimeState.queuedFileEvents.clear();
+    if (queuedEntries.length === 0)
+        return;
+    const changedFilePaths = queuedEntries
+        .filter(([, eventType]) => eventType === 'create' || eventType === 'update')
+        .map(([filePath]) => filePath);
+    const deletedFilePaths = queuedEntries
+        .filter(([, eventType]) => eventType === 'delete')
+        .map(([filePath]) => filePath);
+    const fileSyncResult = await syncFilesToForum({
+        discordClient,
+        forumChannelId: runtimeState.forumChannelId,
+        outputDir: runtimeState.outputDir,
+        runtimeState,
+        changedFilePaths,
+        deletedFilePaths,
+    });
+    if (fileSyncResult instanceof Error) {
+        forumLogger.warn(`FS -> Discord sync failed for ${runtimeState.forumChannelId}: ${fileSyncResult.message}`);
+        return;
+    }
+    if (fileSyncResult.created + fileSyncResult.updated + fileSyncResult.deleted >
+        0) {
+        forumLogger.log(`FS -> Discord ${runtimeState.forumChannelId}: +${fileSyncResult.created} ~${fileSyncResult.updated} -${fileSyncResult.deleted} (skip ${fileSyncResult.skipped})`);
+    }
+    // Refresh the FS mirror for any threads that were touched
+    const discordSyncResult = await syncForumToFiles({
+        discordClient,
+        forumChannelId: runtimeState.forumChannelId,
+        outputDir: runtimeState.outputDir,
+        runtimeState,
+        forceThreadIds: runtimeState.dirtyThreadIds,
+    });
+    if (discordSyncResult instanceof Error) {
+        forumLogger.warn(`Discord -> FS refresh failed for ${runtimeState.forumChannelId}: ${discordSyncResult.message}`);
+        return;
+    }
+    runtimeState.dirtyThreadIds.clear();
+}
+function queueFileEvent({ runtimeState, filePath, eventType, discordClient, }) {
+    if (shouldIgnorePath({ runtimeState, filePath }))
+        return;
+    runtimeState.queuedFileEvents.set(filePath, eventType);
+    if (runtimeState.fileDebounceTimer) {
+        clearTimeout(runtimeState.fileDebounceTimer);
+    }
+    runtimeState.fileDebounceTimer = setTimeout(() => {
+        runtimeState.fileDebounceTimer = null;
+        void runQueuedFileEvents({ runtimeState, discordClient });
+    }, DEFAULT_DEBOUNCE_MS);
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// DISCORD EVENT HANDLING
+// ═══════════════════════════════════════════════════════════════════════════
+function scheduleDiscordSync({ runtimeState, threadId, discordClient, }) {
+    runtimeState.dirtyThreadIds.add(threadId);
+    if (runtimeState.discordDebounceTimer) {
+        clearTimeout(runtimeState.discordDebounceTimer);
+    }
+    runtimeState.discordDebounceTimer = setTimeout(() => {
+        runtimeState.discordDebounceTimer = null;
+        void (async () => {
+            const syncResult = await syncForumToFiles({
+                discordClient,
+                forumChannelId: runtimeState.forumChannelId,
+                outputDir: runtimeState.outputDir,
+                runtimeState,
+                forceThreadIds: runtimeState.dirtyThreadIds,
+            });
+            if (syncResult instanceof Error) {
+                forumLogger.warn(`Debounced Discord -> FS sync failed for ${runtimeState.forumChannelId}: ${syncResult.message}`);
+                return;
+            }
+            runtimeState.dirtyThreadIds.clear();
+        })();
+    }, DEFAULT_DEBOUNCE_MS);
+}
+function getThreadEventData({ channel, }) {
+    if (!channel)
+        return null;
+    if (channel.type !== ChannelType.PublicThread &&
+        channel.type !== ChannelType.PrivateThread &&
+        channel.type !== ChannelType.AnnouncementThread) {
+        return null;
+    }
+    if (!channel.parentId)
+        return null;
+    return { forumChannelId: channel.parentId, threadId: channel.id };
+}
+function getEventThreadFromMessage({ message, }) {
+    const channel = message.channel;
+    if (!channel || !channel.isThread())
+        return null;
+    return channel;
+}
+function tryHandleThreadEvent({ channel, discordClient, }) {
+    const data = getThreadEventData({ channel });
+    if (!data)
+        return;
+    const runtimeState = forumStateById.get(data.forumChannelId);
+    if (!runtimeState)
+        return;
+    scheduleDiscordSync({ runtimeState, threadId: data.threadId, discordClient });
+}
+/**
+ * Find the file path for a thread, checking root and one level of subdirectories.
+ */
+function findThreadFilePath({ outputDir, threadId, }) {
+    const rootPath = getCanonicalThreadFilePath({ outputDir, threadId });
+    if (fs.existsSync(rootPath))
+        return rootPath;
+    const dirEntries = (() => {
+        try {
+            return fs.readdirSync(outputDir, { withFileTypes: true });
+        }
+        catch {
+            return [];
+        }
+    })();
+    for (const entry of dirEntries) {
+        if (!entry.isDirectory())
+            continue;
+        const subPath = getCanonicalThreadFilePath({
+            outputDir,
+            threadId,
+            subfolder: entry.name,
+        });
+        if (fs.existsSync(subPath))
+            return subPath;
+    }
+    return null;
+}
+function registerDiscordSyncListeners({ discordClient, }) {
+    if (discordListenersRegistered)
+        return;
+    discordListenersRegistered = true;
+    discordClient.on(Events.MessageCreate, (message) => {
+        if (message.author?.bot)
+            return;
+        const thread = getEventThreadFromMessage({ message });
+        tryHandleThreadEvent({ channel: thread, discordClient });
+    });
+    discordClient.on(Events.MessageUpdate, (_oldMessage, newMessage) => {
+        const thread = getEventThreadFromMessage({ message: newMessage });
+        tryHandleThreadEvent({ channel: thread, discordClient });
+    });
+    discordClient.on(Events.ThreadUpdate, (_oldThread, newThread) => {
+        tryHandleThreadEvent({ channel: newThread, discordClient });
+    });
+    discordClient.on(Events.ThreadDelete, async (thread) => {
+        const data = getThreadEventData({ channel: thread });
+        if (!data)
+            return;
+        const runtimeState = forumStateById.get(data.forumChannelId);
+        if (!runtimeState)
+            return;
+        const targetPath = findThreadFilePath({
+            outputDir: runtimeState.outputDir,
+            threadId: data.threadId,
+        });
+        if (!targetPath)
+            return;
+        addIgnoredPath({ runtimeState, filePath: targetPath });
+        await fs.promises.unlink(targetPath).catch((cause) => {
+            forumLogger.warn(`Failed to delete forum file on thread delete ${targetPath}:`, cause);
+        });
+    });
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// FILE WATCHER SETUP
+// ═══════════════════════════════════════════════════════════════════════════
+async function startWatcherForRuntimeState({ runtimeState, discordClient, }) {
+    if (runtimeState.direction !== 'bidirectional')
+        return;
+    const subscription = await parcelWatcher
+        .subscribe(runtimeState.outputDir, (_error, events) => {
+        const mdEvents = events.filter((event) => event.path.endsWith('.md'));
+        mdEvents
+            .filter((event) => event.type === 'create' ||
+            event.type === 'update' ||
+            event.type === 'delete')
+            .map((event) => {
+            queueFileEvent({
+                runtimeState,
+                filePath: event.path,
+                eventType: event.type,
+                discordClient,
+            });
+        });
+    })
+        .catch((cause) => new ForumSyncOperationError({
+        forumChannelId: runtimeState.forumChannelId,
+        reason: `failed to subscribe watcher for ${runtimeState.outputDir}`,
+        cause,
+    }));
+    if (subscription instanceof Error)
+        return subscription;
+    watcherUnsubscribeByForumId.set(runtimeState.forumChannelId, () => {
+        return subscription.unsubscribe();
+    });
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// PUBLIC API
+// ═══════════════════════════════════════════════════════════════════════════
+export async function stopConfiguredForumSync() {
+    const unsubscribers = Array.from(watcherUnsubscribeByForumId.values());
+    watcherUnsubscribeByForumId.clear();
+    forumStateById.clear();
+    await Promise.all(unsubscribers.map(async (unsubscribe) => {
+        await unsubscribe().catch((cause) => {
+            forumLogger.warn('Failed to unsubscribe forum watcher:', cause);
+        });
+    }));
+}
+export async function startConfiguredForumSync({ discordClient, appId, }) {
+    const loadedConfig = await readForumSyncConfig({ appId });
+    if (loadedConfig instanceof Error)
+        return loadedConfig;
+    if (loadedConfig.length === 0)
+        return;
+    registerDiscordSyncListeners({ discordClient });
+    // Process each config independently so one stale/deleted forum channel
+    // doesn't block the watcher from starting for other valid configs.
+    for (const entry of loadedConfig) {
+        const runtimeState = buildRuntimeState({
+            forumChannelId: entry.forumChannelId,
+            outputDir: entry.outputDir,
+            direction: entry.direction,
+        });
+        forumStateById.set(entry.forumChannelId, runtimeState);
+        const ensureResult = await ensureDirectory({ directory: entry.outputDir });
+        if (ensureResult instanceof Error) {
+            forumLogger.warn(`Skipping forum ${entry.forumChannelId}: failed to create ${entry.outputDir}`);
+            continue;
+        }
+        const fileToDiscordResult = await syncFilesToForum({
+            discordClient,
+            forumChannelId: entry.forumChannelId,
+            outputDir: entry.outputDir,
+            runtimeState,
+        });
+        if (fileToDiscordResult instanceof Error) {
+            forumLogger.warn(`Skipping forum ${entry.forumChannelId}: FS->Discord sync failed: ${fileToDiscordResult.message}`);
+            continue;
+        }
+        const discordToFileResult = await syncForumToFiles({
+            discordClient,
+            forumChannelId: entry.forumChannelId,
+            outputDir: entry.outputDir,
+            forceFullRefresh: true,
+            runtimeState,
+        });
+        if (discordToFileResult instanceof Error) {
+            forumLogger.warn(`Skipping forum ${entry.forumChannelId}: Discord->FS sync failed: ${discordToFileResult.message}`);
+            continue;
+        }
+        const watcherResult = await startWatcherForRuntimeState({
+            runtimeState,
+            discordClient,
+        });
+        if (watcherResult instanceof Error) {
+            forumLogger.warn(`Skipping forum ${entry.forumChannelId}: watcher failed: ${watcherResult.message}`);
+            continue;
+        }
+        forumLogger.log(`Forum sync started for ${entry.forumChannelId} (${entry.direction}) -> ${entry.outputDir}`);
+        forumLogger.log(`Initial sync: Discord->FS synced ${discordToFileResult.synced}, skipped ${discordToFileResult.skipped}, deleted ${discordToFileResult.deleted}; FS->Discord created ${fileToDiscordResult.created}, updated ${fileToDiscordResult.updated}, deleted ${fileToDiscordResult.deleted}`);
+    }
+}