@otto-assistant/otto 0.1.2 → 0.7.15

package/src/forum-sync/config.ts

@@ -0,0 +1,92 @@
+// Forum sync configuration from SQLite database.
+// Reads forum_sync_configs table and resolves relative output dirs.
+// On first run, migrates any existing forum-sync.json into the DB.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import YAML from 'yaml'
+import { getDataDir } from '../config.js'
+import { getForumSyncConfigs, upsertForumSyncConfig } from '../database.js'
+import { createLogger } from '../logger.js'
+import type { ForumSyncDirection, LoadedForumConfig } from './types.js'
+
+const forumLogger = createLogger('FORUM')
+
+const LEGACY_CONFIG_FILE = 'forum-sync.json'
+
+function isForumSyncDirection(value: unknown): value is ForumSyncDirection {
+  return value === 'discord-to-files' || value === 'bidirectional'
+}
+
+function resolveOutputDir(outputDir: string): string {
+  if (path.isAbsolute(outputDir)) return outputDir
+  return path.resolve(getDataDir(), outputDir)
+}
+
+/**
+ * One-time migration: if the legacy forum-sync.json exists, import its entries
+ * into the DB and rename the file so it's not re-imported on next startup.
+ */
+async function migrateLegacyConfig({ appId }: { appId: string }) {
+  const configPath = path.join(getDataDir(), LEGACY_CONFIG_FILE)
+  if (!fs.existsSync(configPath)) return
+
+  forumLogger.log(`Migrating legacy ${LEGACY_CONFIG_FILE} into database...`)
+
+  const raw = fs.readFileSync(configPath, 'utf8')
+  let parsed: unknown
+  try {
+    parsed = YAML.parse(raw)
+  } catch {
+    forumLogger.warn(
+      `Failed to parse legacy ${LEGACY_CONFIG_FILE}, skipping migration`,
+    )
+    return
+  }
+
+  if (!parsed || typeof parsed !== 'object') return
+  const forums = (parsed as Record<string, unknown>).forums
+  if (!Array.isArray(forums)) return
+
+  for (const item of forums) {
+    if (!item || typeof item !== 'object') continue
+    const entry = item as Record<string, unknown>
+    const forumChannelId =
+      typeof entry.forumChannelId === 'string' ? entry.forumChannelId : ''
+    const outputDir = typeof entry.outputDir === 'string' ? entry.outputDir : ''
+    const direction = isForumSyncDirection(entry.direction)
+      ? entry.direction
+      : 'bidirectional'
+    if (!forumChannelId || !outputDir) continue
+
+    await upsertForumSyncConfig({
+      appId,
+      forumChannelId,
+      outputDir: resolveOutputDir(outputDir),
+      direction,
+    })
+  }
+
+  // Rename so we don't re-import next time
+  const backupPath = configPath + '.migrated'
+  fs.renameSync(configPath, backupPath)
+  forumLogger.log(
+    `Legacy config migrated and renamed to ${path.basename(backupPath)}`,
+  )
+}
+
+export async function readForumSyncConfig({ appId }: { appId?: string }) {
+  if (!appId) return []
+
+  // Migrate legacy JSON file on first run
+  await migrateLegacyConfig({ appId })
+
+  const rows = await getForumSyncConfigs({ appId })
+  return rows.map<LoadedForumConfig>((row) => ({
+    forumChannelId: row.forumChannelId,
+    outputDir: resolveOutputDir(row.outputDir),
+    direction: isForumSyncDirection(row.direction)
+      ? row.direction
+      : 'bidirectional',
+  }))
+}

package/src/forum-sync/discord-operations.ts

@@ -0,0 +1,241 @@
+// Discord API operations for forum sync.
+// Resolves forum channels, fetches threads (active + archived) with pagination,
+// fetches thread messages, loads existing forum files from disk, and ensures directories.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import {
+  ChannelType,
+  type Client,
+  type ForumChannel,
+  type Message,
+  type ThreadChannel,
+} from 'discord.js'
+import { createLogger } from '../logger.js'
+import { parseFrontmatter, getStringValue } from './markdown.js'
+import {
+  DEFAULT_RATE_LIMIT_DELAY_MS,
+  ForumChannelResolveError,
+  ForumSyncOperationError,
+  delay,
+  type ExistingForumFile,
+} from './types.js'
+
+const forumLogger = createLogger('FORUM')
+
+export function getCanonicalThreadFilePath({
+  outputDir,
+  threadId,
+  subfolder,
+}: {
+  outputDir: string
+  threadId: string
+  subfolder?: string
+}) {
+  if (subfolder) {
+    return path.join(outputDir, subfolder, `${threadId}.md`)
+  }
+  return path.join(outputDir, `${threadId}.md`)
+}
+
+export async function ensureDirectory({ directory }: { directory: string }) {
+  const result = await fs.promises.mkdir(directory, { recursive: true }).catch(
+    (cause) =>
+      new ForumSyncOperationError({
+        forumChannelId: 'unknown',
+        reason: directory,
+        cause,
+      }),
+  )
+  if (result instanceof Error) return result
+}
+
+export async function resolveForumChannel({
+  discordClient,
+  forumChannelId,
+}: {
+  discordClient: Client
+  forumChannelId: string
+}): Promise<ForumChannel | ForumChannelResolveError> {
+  const channel = await discordClient.channels
+    .fetch(forumChannelId)
+    .catch((cause) => new ForumChannelResolveError({ forumChannelId, cause }))
+  if (channel instanceof Error) return channel
+
+  if (!channel || channel.type !== ChannelType.GuildForum) {
+    return new ForumChannelResolveError({ forumChannelId })
+  }
+
+  return channel
+}
+
+export async function fetchForumThreads({
+  forumChannel,
+}: {
+  forumChannel: ForumChannel
+}): Promise<ThreadChannel[] | ForumSyncOperationError> {
+  const byId = new Map<string, ThreadChannel>()
+
+  const active = await forumChannel.threads.fetchActive().catch(
+    (cause) =>
+      new ForumSyncOperationError({
+        forumChannelId: forumChannel.id,
+        reason: 'fetchActive failed',
+        cause,
+      }),
+  )
+  if (active instanceof Error) return active
+
+  for (const [id, thread] of active.threads) {
+    byId.set(id, thread)
+  }
+
+  let before: Date | undefined
+  while (true) {
+    const archived = await forumChannel.threads
+      .fetchArchived({ type: 'public', limit: 100, before })
+      .catch(
+        (cause) =>
+          new ForumSyncOperationError({
+            forumChannelId: forumChannel.id,
+            reason: 'fetchArchived failed',
+            cause,
+          }),
+      )
+    if (archived instanceof Error) return archived
+
+    const threads = Array.from(archived.threads.values())
+    for (const thread of threads) {
+      byId.set(thread.id, thread)
+    }
+
+    if (!archived.hasMore || threads.length === 0) break
+
+    const timestamps = threads
+      .map((thread) => thread.archiveTimestamp ?? thread.createdTimestamp)
+      .filter((value): value is number => value !== null)
+
+    const oldestTimestamp = Math.min(...timestamps)
+    if (!Number.isFinite(oldestTimestamp)) break
+
+    before = new Date(oldestTimestamp - 1)
+    await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS })
+  }
+
+  return Array.from(byId.values())
+}
+
+export async function fetchThreadMessages({
+  thread,
+}: {
+  thread: ThreadChannel
+}): Promise<Message[] | ForumSyncOperationError> {
+  const byId = new Map<string, Message>()
+  let before: string | undefined
+
+  while (true) {
+    const fetched = await thread.messages.fetch({ limit: 100, before }).catch(
+      (cause) =>
+        new ForumSyncOperationError({
+          forumChannelId: thread.parentId || 'unknown',
+          reason: `message fetch failed for thread ${thread.id}`,
+          cause,
+        }),
+    )
+    if (fetched instanceof Error) return fetched
+
+    const messages = Array.from(fetched.values())
+    for (const message of messages) {
+      byId.set(message.id, message)
+    }
+
+    if (messages.length < 100 || messages.length === 0) break
+
+    // Find oldest message for cursor - messages are sorted by Discord, last is oldest
+    const oldest = messages[messages.length - 1]
+    if (!oldest) break
+
+    before = oldest.id
+    await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS })
+  }
+
+  return Array.from(byId.values()).sort(
+    (a, b) => a.createdTimestamp - b.createdTimestamp,
+  )
+}
+
+/**
+ * Recursively walks a directory collecting all .md files with their relative subfolder path.
+ */
+async function collectMarkdownFiles({
+  dir,
+  outputDir,
+}: {
+  dir: string
+  outputDir: string
+}): Promise<Array<{ filePath: string; subfolder?: string }>> {
+  if (!fs.existsSync(dir)) return []
+
+  const entries = await fs.promises.readdir(dir, { withFileTypes: true })
+  const relativeSub = path.relative(outputDir, dir)
+  const subfolder = relativeSub && relativeSub !== '.' ? relativeSub : undefined
+
+  const mdFiles: Array<{ filePath: string; subfolder?: string }> = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+    .map((entry) => ({ filePath: path.join(dir, entry.name), subfolder }))
+
+  const subdirs = entries.filter((entry) => entry.isDirectory())
+  const nestedResults = await Promise.all(
+    subdirs.map((subdir) =>
+      collectMarkdownFiles({
+        dir: path.join(dir, subdir.name),
+        outputDir,
+      }),
+    ),
+  )
+
+  return [...mdFiles, ...nestedResults.flat()]
+}
+
+export async function loadExistingForumFiles({
+  outputDir,
+}: {
+  outputDir: string
+}): Promise<ExistingForumFile[]> {
+  const markdownEntries = await collectMarkdownFiles({
+    dir: outputDir,
+    outputDir,
+  })
+
+  const loaded = await Promise.all(
+    markdownEntries.map(async ({ filePath, subfolder }) => {
+      const content = await fs.promises
+        .readFile(filePath, 'utf8')
+        .catch((cause) => {
+          forumLogger.warn(`Failed to read forum file ${filePath}:`, cause)
+          return null
+        })
+      if (content === null) return null
+
+      const parsed = parseFrontmatter({ markdown: content })
+      const threadIdFromFrontmatter = getStringValue({
+        value: parsed.frontmatter.threadId,
+      })
+      const threadIdFromFilename = path.basename(filePath, '.md')
+      const threadId =
+        threadIdFromFrontmatter ||
+        (/^\d+$/.test(threadIdFromFilename) ? threadIdFromFilename : '')
+      if (!threadId) return null
+
+      const result: ExistingForumFile = {
+        filePath,
+        threadId,
+        frontmatter: parsed.frontmatter,
+        subfolder,
+      }
+      return result
+    }),
+  )
+
+  return loaded.filter((item): item is ExistingForumFile => item !== null)
+}

package/src/forum-sync/index.ts

@@ -0,0 +1,9 @@
+// Forum sync module entry point.
+// Re-exports the public API for forum <-> markdown synchronization.
+
+export {
+  startConfiguredForumSync,
+  stopConfiguredForumSync,
+} from './watchers.js'
+export { syncForumToFiles } from './sync-to-files.js'
+export { syncFilesToForum } from './sync-to-discord.js'

package/src/forum-sync/markdown.ts

@@ -0,0 +1,172 @@
+// Markdown parsing, serialization, and section formatting for forum sync.
+// Handles frontmatter extraction, message section building, and
+// conversion between Discord messages and markdown format.
+
+import YAML from 'yaml'
+import * as errore from 'errore'
+import type { Message } from 'discord.js'
+import {
+  ForumFrontmatterParseError,
+  type ForumMarkdownFrontmatter,
+  type ForumMessageSection,
+  type ParsedMarkdownFile,
+} from './types.js'
+
+export function toStringArray({ value }: { value: unknown }): string[] {
+  if (!Array.isArray(value)) return []
+  return value.filter((item): item is string => typeof item === 'string')
+}
+
+export function getStringValue({ value }: { value: unknown }): string {
+  if (typeof value !== 'string') return ''
+  return value
+}
+
+export function parseFrontmatter({
+  markdown,
+}: {
+  markdown: string
+}): ParsedMarkdownFile {
+  if (!markdown.startsWith('---\n')) {
+    return { frontmatter: {}, body: markdown.trim() }
+  }
+
+  const end = markdown.indexOf('\n---\n', 4)
+  if (end === -1) {
+    return { frontmatter: {}, body: markdown.trim() }
+  }
+
+  const rawFrontmatter = markdown.slice(4, end)
+  const body = markdown.slice(end + 5).trim()
+
+  const parsed = errore.try({
+    try: () => YAML.parse(rawFrontmatter),
+    catch: (cause) =>
+      new ForumFrontmatterParseError({ reason: 'yaml parse failed', cause }),
+  })
+
+  if (parsed instanceof Error || !parsed || typeof parsed !== 'object') {
+    return { frontmatter: {}, body }
+  }
+
+  return { frontmatter: parsed as Record<string, unknown>, body }
+}
+
+export function stringifyFrontmatter({
+  frontmatter,
+  body,
+}: {
+  frontmatter: ForumMarkdownFrontmatter
+  body: string
+}) {
+  const yamlText = YAML.stringify(frontmatter, null, {
+    lineWidth: 120,
+  }).trim()
+  return `---\n${yamlText}\n---\n\n${body.trim()}\n`
+}
+
+export function splitSections({ body }: { body: string }) {
+  return body
+    .split(/\r?\n---\r?\n/g)
+    .map((part) => part.trim())
+    .filter((part) => part.length > 0)
+}
+
+export function extractStarterContent({ body }: { body: string }) {
+  const sections = splitSections({ body })
+  const firstSection = sections[0] || ''
+  const match = firstSection.match(
+    /^\*\*.+?\*\* \(\d+\) - .+?(?: \(edited .+?\))?\r?\n\r?\n([\s\S]*)$/,
+  )
+  if (!match) return body.trim()
+  return (match[1] || '').trim()
+}
+
+export function buildMessageSections({
+  messages,
+}: {
+  messages: Message[]
+}): ForumMessageSection[] {
+  return messages.map((message) => {
+    const attachmentLines = Array.from(message.attachments.values()).map(
+      (attachment) => `Attachment: ${attachment.url}`,
+    )
+
+    const contentParts: string[] = []
+    const trimmedContent = message.content.trim()
+    if (trimmedContent) {
+      contentParts.push(trimmedContent)
+    }
+    if (attachmentLines.length > 0) {
+      contentParts.push(attachmentLines.join('\n'))
+    }
+
+    const content =
+      contentParts.length > 0
+        ? contentParts.join('\n\n')
+        : '_(no text content)_'
+
+    return {
+      messageId: message.id,
+      authorName: message.author.username,
+      authorId: message.author.id,
+      createdAt: new Date(message.createdTimestamp).toISOString(),
+      editedAt: message.editedTimestamp
+        ? new Date(message.editedTimestamp).toISOString()
+        : null,
+      content,
+    } satisfies ForumMessageSection
+  })
+}
+
+export function formatMessageSection({
+  section,
+}: {
+  section: ForumMessageSection
+}) {
+  const editedSuffix = section.editedAt ? ` (edited ${section.editedAt})` : ''
+  return `**${section.authorName}** (${section.authorId}) - ${section.createdAt}${editedSuffix}\n\n${section.content}`
+}
+
+// Channel mention footer stored in the Discord starter message so
+// projectChannelId survives a full re-sync from Discord (no local files).
+// Uses <#id> so Discord renders it as a clickable channel link.
+// Matches at start-of-string or after a newline so it works even when the
+// footer is the only content in the message (e.g. empty body).
+const PROJECT_CHANNEL_FOOTER_RE = /(?:^|\n)channel: <#(\d{17,20})>\s*$/
+const MAX_STARTER_MESSAGE_LENGTH = 2_000
+
+/** Append a channel mention footer, truncating the body so the total
+ *  never exceeds Discord's 2000-char starter message limit. */
+export function appendProjectChannelFooter({
+  content,
+  projectChannelId,
+}: {
+  content: string
+  projectChannelId?: string
+}): string {
+  if (!projectChannelId) return content
+  const footer = `\nchannel: <#${projectChannelId}>`
+  const maxContentLength = MAX_STARTER_MESSAGE_LENGTH - footer.length
+  const truncated =
+    content.length > maxContentLength
+      ? content.slice(0, maxContentLength)
+      : content
+  return `${truncated}${footer}`
+}
+
+export function extractProjectChannelFromContent({
+  content,
+}: {
+  content: string
+}): {
+  cleanContent: string
+  projectChannelId?: string
+} {
+  const match = content.match(PROJECT_CHANNEL_FOOTER_RE)
+  if (!match) return { cleanContent: content }
+  return {
+    cleanContent: content.replace(PROJECT_CHANNEL_FOOTER_RE, '').trim(),
+    projectChannelId: match[1],
+  }
+}