@brianli/kimaki 0.4.72-brianli.1

package/src/format-tables.test.ts

@@ -0,0 +1,223 @@
+import { test, expect, describe } from 'vitest'
+import {
+  splitTablesFromMarkdown,
+  buildTableComponents,
+  type ContentSegment,
+} from './format-tables.js'
+import { Lexer, type Tokens } from 'marked'
+
+function parseTable(markdown: string): Tokens.Table {
+  const lexer = new Lexer()
+  const tokens = lexer.lex(markdown)
+  return tokens.find((t) => t.type === 'table') as Tokens.Table
+}
+
+/** Extract the first container's children from buildTableComponents result */
+function getContainerChildren(
+  segments: ContentSegment[],
+): { type: number; content?: string; divider?: boolean; spacing?: number }[] {
+  const seg = segments[0]!
+  if (seg.type !== 'components') {
+    throw new Error('Expected components segment')
+  }
+  const container = seg.components[0] as { type: number; components: unknown[] }
+  return container.components as {
+    type: number
+    content?: string
+    divider?: boolean
+    spacing?: number
+  }[]
+}
+
+describe('buildTableComponents', () => {
+  test('builds container with key-value TextDisplays', () => {
+    const table = parseTable(`| Name | Age |
+| --- | --- |
+| Alice | 30 |
+| Bob | 25 |`)
+    const result = buildTableComponents(table)
+    expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Name** Alice
+      **Age** 30",
+                  "type": 10,
+                },
+                {
+                  "divider": true,
+                  "spacing": 1,
+                  "type": 14,
+                },
+                {
+                  "content": "**Name** Bob
+      **Age** 25",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `)
+  })
+
+  test('adds separators between row groups', () => {
+    const table = parseTable(`| Key | Value |
+| --- | --- |
+| a | 1 |
+| b | 2 |
+| c | 3 |`)
+    const result = buildTableComponents(table)
+    const types = getContainerChildren(result).map((c) => c.type)
+    // type 10 = TextDisplay, type 14 = Separator
+    expect(types).toMatchInlineSnapshot(`
+      [
+        10,
+        14,
+        10,
+        14,
+        10,
+      ]
+    `)
+  })
+
+  test('single-row table has one TextDisplay, no separators', () => {
+    const table = parseTable(`| Method | Endpoint |
+| --- | --- |
+| GET | /api/users |`)
+    const result = buildTableComponents(table)
+    const children = getContainerChildren(result)
+    expect(children).toHaveLength(1)
+    expect(children[0]!.type).toBe(10)
+    expect(children[0]!.content).toMatchInlineSnapshot(`
+      "**Method** GET
+      **Endpoint** /api/users"
+    `)
+  })
+
+  test('splits large table into multiple container segments', () => {
+    // 25 rows: exceeds 19 rows per container, so splits into 2 containers
+    const headers = '| A | B |'
+    const sep = '| --- | --- |'
+    const rows = Array.from({ length: 25 }, (_, i) => {
+      return `| ${i}a | ${i}b |`
+    }).join('\n')
+    const table = parseTable(`${headers}\n${sep}\n${rows}`)
+    const result = buildTableComponents(table)
+    expect(result).toHaveLength(2)
+    expect(result[0]!.type).toBe('components')
+    expect(result[1]!.type).toBe('components')
+    // First container has 19 rows (19 TDs + 18 seps = 37 children)
+    const firstChildren = getContainerChildren([result[0]!])
+    expect(firstChildren).toHaveLength(19 + 18)
+    // Second container has 6 rows (6 TDs + 5 seps = 11 children)
+    const secondChildren = getContainerChildren([result[1]!])
+    expect(secondChildren).toHaveLength(6 + 5)
+  })
+
+  test('strips formatting from cells', () => {
+    const table = parseTable(`| Header | Value |
+| --- | --- |
+| **Bold text** | Normal |
+| *Italic* | \`code\` |`)
+    const result = buildTableComponents(table)
+    const children = getContainerChildren(result)
+    expect(children[0]!.content).toMatchInlineSnapshot(`
+      "**Header** Bold text
+      **Value** Normal"
+    `)
+  })
+})
+
+describe('splitTablesFromMarkdown', () => {
+  test('returns single text segment for content without tables', () => {
+    const result = splitTablesFromMarkdown('Just some text.\n\nMore text.')
+    expect(result).toHaveLength(1)
+    expect(result[0]!.type).toBe('text')
+  })
+
+  test('returns single components segment for table-only content', () => {
+    const result = splitTablesFromMarkdown(`| A | B |
+| --- | --- |
+| 1 | 2 |`)
+    expect(result).toHaveLength(1)
+    expect(result[0]!.type).toBe('components')
+  })
+
+  test('splits text before and after table into separate segments', () => {
+    const result = splitTablesFromMarkdown(`Text before.
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Text after.`)
+    expect(result).toHaveLength(3)
+    expect(result[0]!.type).toBe('text')
+    expect(result[1]!.type).toBe('components')
+    expect(result[2]!.type).toBe('text')
+  })
+
+  test('handles multiple tables with text between', () => {
+    const result = splitTablesFromMarkdown(`First table:
+
+| A | B |
+| --- | --- |
+| 1 | 2 |
+
+Middle text.
+
+| X | Y |
+| --- | --- |
+| a | b |`)
+    expect(result).toHaveLength(4)
+    expect(result.map((s) => s.type)).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+        "components",
+      ]
+    `)
+  })
+
+  test('splits oversized table into multiple component segments', () => {
+    const headers = '| A | B |'
+    const sep = '| --- | --- |'
+    const rows = Array.from({ length: 25 }, (_, i) => {
+      return `| ${i}a | ${i}b |`
+    }).join('\n')
+    const result = splitTablesFromMarkdown(`${headers}\n${sep}\n${rows}`)
+    // 25 rows splits into 2 container segments
+    expect(result).toHaveLength(2)
+    expect(result.every((s) => s.type === 'components')).toBe(true)
+  })
+
+  test('preserves code blocks alongside tables', () => {
+    const result = splitTablesFromMarkdown(`Some code:
+
+\`\`\`js
+const x = 1
+\`\`\`
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Done.`)
+    const types = result.map((s) => s.type)
+    expect(types).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+      ]
+    `)
+  })
+})

package/src/format-tables.ts

@@ -0,0 +1,145 @@
+// Markdown table formatter for Discord.
+// Converts GFM tables to Discord Components V2 (ContainerBuilder with TextDisplay
+// key-value pairs and Separators between row groups). Large tables are split
+// across multiple Container components to stay within the 40-component limit.
+
+import { Lexer, type Token, type Tokens } from 'marked'
+import {
+  SeparatorSpacingSize,
+  type APIContainerComponent,
+  type APITextDisplayComponent,
+  type APISeparatorComponent,
+  type APIMessageTopLevelComponent,
+} from 'discord.js'
+
+export type ContentSegment =
+  | { type: 'text'; text: string }
+  | { type: 'components'; components: APIMessageTopLevelComponent[] }
+
+// Max 40 components per message (nested components count toward the limit).
+// Each container uses: 1 (container) + M (TextDisplays) + M-1 (separators) = 2M children.
+// So max rows per container = floor((40 - 1) / 2) = 19.
+const MAX_COMPONENTS = 40
+const MAX_ROWS_PER_CONTAINER = Math.floor((MAX_COMPONENTS - 1) / 2)
+
+/**
+ * Split markdown into text and table component segments.
+ * Tables are rendered as CV2 Container components with bold key-value TextDisplay
+ * pairs. Large tables are split across multiple component segments.
+ */
+export function splitTablesFromMarkdown(markdown: string): ContentSegment[] {
+  const lexer = new Lexer()
+  const tokens = lexer.lex(markdown)
+  const segments: ContentSegment[] = []
+  let textBuffer = ''
+
+  for (const token of tokens) {
+    if (token.type === 'table') {
+      if (textBuffer.trim()) {
+        segments.push({ type: 'text', text: textBuffer })
+        textBuffer = ''
+      }
+      const componentSegments = buildTableComponents(token as Tokens.Table)
+      segments.push(...componentSegments)
+    } else {
+      textBuffer += token.raw
+    }
+  }
+
+  if (textBuffer.trim()) {
+    segments.push({ type: 'text', text: textBuffer })
+  }
+
+  return segments
+}
+
+/**
+ * Build CV2 components for a table. Each data row becomes a single TextDisplay
+ * with all key-value pairs joined by newlines (header bold as key). Separator
+ * dividers are placed between row groups.
+ * Large tables are split into multiple component segments, each containing a
+ * Container with up to MAX_ROWS_PER_CONTAINER rows.
+ */
+export function buildTableComponents(table: Tokens.Table): ContentSegment[] {
+  const headers = table.header.map((cell) => {
+    return extractCellText(cell.tokens)
+  })
+  const rows = table.rows.map((row) => {
+    return row.map((cell) => {
+      return extractCellText(cell.tokens)
+    })
+  })
+
+  // Split rows into chunks that fit within the component limit
+  const chunks: string[][][] = []
+  for (let i = 0; i < rows.length; i += MAX_ROWS_PER_CONTAINER) {
+    chunks.push(rows.slice(i, i + MAX_ROWS_PER_CONTAINER))
+  }
+
+  return chunks.map((chunkRows) => {
+    const children: (APITextDisplayComponent | APISeparatorComponent)[] = []
+
+    for (let i = 0; i < chunkRows.length; i++) {
+      if (i > 0) {
+        children.push({
+          type: 14,
+          divider: true,
+          spacing: SeparatorSpacingSize.Small,
+        })
+      }
+      const row = chunkRows[i]!
+      const lines = headers.map((key, j) => {
+        const value = row[j] || ''
+        return `**${key}** ${value}`
+      })
+      children.push({ type: 10, content: lines.join('\n') })
+    }
+
+    const container: APIContainerComponent = {
+      type: 17,
+      components: children,
+    }
+
+    return {
+      type: 'components' as const,
+      components: [container] as APIMessageTopLevelComponent[],
+    }
+  })
+}
+
+function extractCellText(tokens: Token[]): string {
+  const parts: string[] = []
+  for (const token of tokens) {
+    parts.push(extractTokenText(token))
+  }
+  return parts.join('').trim()
+}
+
+function extractTokenText(token: Token): string {
+  switch (token.type) {
+    case 'text':
+    case 'codespan':
+    case 'escape':
+      return token.text
+    case 'link':
+      return token.href
+    case 'image':
+      return token.href
+    case 'strong':
+    case 'em':
+    case 'del':
+      return token.tokens ? extractCellText(token.tokens) : token.text
+    case 'br':
+      return ' '
+    default: {
+      const tokenAny = token as { tokens?: Token[]; text?: string }
+      if (tokenAny.tokens && Array.isArray(tokenAny.tokens)) {
+        return extractCellText(tokenAny.tokens)
+      }
+      if (typeof tokenAny.text === 'string') {
+        return tokenAny.text
+      }
+      return ''
+    }
+  }
+}

package/src/forum-sync/config.ts

@@ -0,0 +1,92 @@
+// Forum sync configuration from SQLite database.
+// Reads forum_sync_configs table and resolves relative output dirs.
+// On first run, migrates any existing forum-sync.json into the DB.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import yaml from 'js-yaml'
+import { getDataDir } from '../config.js'
+import { getForumSyncConfigs, upsertForumSyncConfig } from '../database.js'
+import { createLogger } from '../logger.js'
+import type { ForumSyncDirection, LoadedForumConfig } from './types.js'
+
+const forumLogger = createLogger('FORUM')
+
+const LEGACY_CONFIG_FILE = 'forum-sync.json'
+
+function isForumSyncDirection(value: unknown): value is ForumSyncDirection {
+  return value === 'discord-to-files' || value === 'bidirectional'
+}
+
+function resolveOutputDir(outputDir: string): string {
+  if (path.isAbsolute(outputDir)) return outputDir
+  return path.resolve(getDataDir(), outputDir)
+}
+
+/**
+ * One-time migration: if the legacy forum-sync.json exists, import its entries
+ * into the DB and rename the file so it's not re-imported on next startup.
+ */
+async function migrateLegacyConfig({ appId }: { appId: string }) {
+  const configPath = path.join(getDataDir(), LEGACY_CONFIG_FILE)
+  if (!fs.existsSync(configPath)) return
+
+  forumLogger.log(`Migrating legacy ${LEGACY_CONFIG_FILE} into database...`)
+
+  const raw = fs.readFileSync(configPath, 'utf8')
+  let parsed: unknown
+  try {
+    parsed = yaml.load(raw)
+  } catch {
+    forumLogger.warn(
+      `Failed to parse legacy ${LEGACY_CONFIG_FILE}, skipping migration`,
+    )
+    return
+  }
+
+  if (!parsed || typeof parsed !== 'object') return
+  const forums = (parsed as Record<string, unknown>).forums
+  if (!Array.isArray(forums)) return
+
+  for (const item of forums) {
+    if (!item || typeof item !== 'object') continue
+    const entry = item as Record<string, unknown>
+    const forumChannelId =
+      typeof entry.forumChannelId === 'string' ? entry.forumChannelId : ''
+    const outputDir = typeof entry.outputDir === 'string' ? entry.outputDir : ''
+    const direction = isForumSyncDirection(entry.direction)
+      ? entry.direction
+      : 'bidirectional'
+    if (!forumChannelId || !outputDir) continue
+
+    await upsertForumSyncConfig({
+      appId,
+      forumChannelId,
+      outputDir: resolveOutputDir(outputDir),
+      direction,
+    })
+  }
+
+  // Rename so we don't re-import next time
+  const backupPath = configPath + '.migrated'
+  fs.renameSync(configPath, backupPath)
+  forumLogger.log(
+    `Legacy config migrated and renamed to ${path.basename(backupPath)}`,
+  )
+}
+
+export async function readForumSyncConfig({ appId }: { appId?: string }) {
+  if (!appId) return []
+
+  // Migrate legacy JSON file on first run
+  await migrateLegacyConfig({ appId })
+
+  const rows = await getForumSyncConfigs({ appId })
+  return rows.map<LoadedForumConfig>((row) => ({
+    forumChannelId: row.forumChannelId,
+    outputDir: resolveOutputDir(row.outputDir),
+    direction: isForumSyncDirection(row.direction)
+      ? row.direction
+      : 'bidirectional',
+  }))
+}

package/src/forum-sync/discord-operations.ts

@@ -0,0 +1,241 @@
+// Discord API operations for forum sync.
+// Resolves forum channels, fetches threads (active + archived) with pagination,
+// fetches thread messages, loads existing forum files from disk, and ensures directories.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import {
+  ChannelType,
+  type Client,
+  type ForumChannel,
+  type Message,
+  type ThreadChannel,
+} from 'discord.js'
+import { createLogger } from '../logger.js'
+import { parseFrontmatter, getStringValue } from './markdown.js'
+import {
+  DEFAULT_RATE_LIMIT_DELAY_MS,
+  ForumChannelResolveError,
+  ForumSyncOperationError,
+  delay,
+  type ExistingForumFile,
+} from './types.js'
+
+const forumLogger = createLogger('FORUM')
+
+export function getCanonicalThreadFilePath({
+  outputDir,
+  threadId,
+  subfolder,
+}: {
+  outputDir: string
+  threadId: string
+  subfolder?: string
+}) {
+  if (subfolder) {
+    return path.join(outputDir, subfolder, `${threadId}.md`)
+  }
+  return path.join(outputDir, `${threadId}.md`)
+}
+
+export async function ensureDirectory({ directory }: { directory: string }) {
+  const result = await fs.promises.mkdir(directory, { recursive: true }).catch(
+    (cause) =>
+      new ForumSyncOperationError({
+        forumChannelId: 'unknown',
+        reason: directory,
+        cause,
+      }),
+  )
+  if (result instanceof Error) return result
+}
+
+export async function resolveForumChannel({
+  discordClient,
+  forumChannelId,
+}: {
+  discordClient: Client
+  forumChannelId: string
+}): Promise<ForumChannel | ForumChannelResolveError> {
+  const channel = await discordClient.channels
+    .fetch(forumChannelId)
+    .catch((cause) => new ForumChannelResolveError({ forumChannelId, cause }))
+  if (channel instanceof Error) return channel
+
+  if (!channel || channel.type !== ChannelType.GuildForum) {
+    return new ForumChannelResolveError({ forumChannelId })
+  }
+
+  return channel
+}
+
+export async function fetchForumThreads({
+  forumChannel,
+}: {
+  forumChannel: ForumChannel
+}): Promise<ThreadChannel[] | ForumSyncOperationError> {
+  const byId = new Map<string, ThreadChannel>()
+
+  const active = await forumChannel.threads.fetchActive().catch(
+    (cause) =>
+      new ForumSyncOperationError({
+        forumChannelId: forumChannel.id,
+        reason: 'fetchActive failed',
+        cause,
+      }),
+  )
+  if (active instanceof Error) return active
+
+  for (const [id, thread] of active.threads) {
+    byId.set(id, thread)
+  }
+
+  let before: Date | undefined
+  while (true) {
+    const archived = await forumChannel.threads
+      .fetchArchived({ type: 'public', limit: 100, before })
+      .catch(
+        (cause) =>
+          new ForumSyncOperationError({
+            forumChannelId: forumChannel.id,
+            reason: 'fetchArchived failed',
+            cause,
+          }),
+      )
+    if (archived instanceof Error) return archived
+
+    const threads = Array.from(archived.threads.values())
+    for (const thread of threads) {
+      byId.set(thread.id, thread)
+    }
+
+    if (!archived.hasMore || threads.length === 0) break
+
+    const timestamps = threads
+      .map((thread) => thread.archiveTimestamp ?? thread.createdTimestamp)
+      .filter((value): value is number => value !== null)
+
+    const oldestTimestamp = Math.min(...timestamps)
+    if (!Number.isFinite(oldestTimestamp)) break
+
+    before = new Date(oldestTimestamp - 1)
+    await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS })
+  }
+
+  return Array.from(byId.values())
+}
+
+export async function fetchThreadMessages({
+  thread,
+}: {
+  thread: ThreadChannel
+}): Promise<Message[] | ForumSyncOperationError> {
+  const byId = new Map<string, Message>()
+  let before: string | undefined
+
+  while (true) {
+    const fetched = await thread.messages.fetch({ limit: 100, before }).catch(
+      (cause) =>
+        new ForumSyncOperationError({
+          forumChannelId: thread.parentId || 'unknown',
+          reason: `message fetch failed for thread ${thread.id}`,
+          cause,
+        }),
+    )
+    if (fetched instanceof Error) return fetched
+
+    const messages = Array.from(fetched.values())
+    for (const message of messages) {
+      byId.set(message.id, message)
+    }
+
+    if (messages.length < 100 || messages.length === 0) break
+
+    // Find oldest message for cursor - messages are sorted by Discord, last is oldest
+    const oldest = messages[messages.length - 1]
+    if (!oldest) break
+
+    before = oldest.id
+    await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS })
+  }
+
+  return Array.from(byId.values()).sort(
+    (a, b) => a.createdTimestamp - b.createdTimestamp,
+  )
+}
+
+/**
+ * Recursively walks a directory collecting all .md files with their relative subfolder path.
+ */
+async function collectMarkdownFiles({
+  dir,
+  outputDir,
+}: {
+  dir: string
+  outputDir: string
+}): Promise<Array<{ filePath: string; subfolder?: string }>> {
+  if (!fs.existsSync(dir)) return []
+
+  const entries = await fs.promises.readdir(dir, { withFileTypes: true })
+  const relativeSub = path.relative(outputDir, dir)
+  const subfolder = relativeSub && relativeSub !== '.' ? relativeSub : undefined
+
+  const mdFiles: Array<{ filePath: string; subfolder?: string }> = entries
+    .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+    .map((entry) => ({ filePath: path.join(dir, entry.name), subfolder }))
+
+  const subdirs = entries.filter((entry) => entry.isDirectory())
+  const nestedResults = await Promise.all(
+    subdirs.map((subdir) =>
+      collectMarkdownFiles({
+        dir: path.join(dir, subdir.name),
+        outputDir,
+      }),
+    ),
+  )
+
+  return [...mdFiles, ...nestedResults.flat()]
+}
+
+export async function loadExistingForumFiles({
+  outputDir,
+}: {
+  outputDir: string
+}): Promise<ExistingForumFile[]> {
+  const markdownEntries = await collectMarkdownFiles({
+    dir: outputDir,
+    outputDir,
+  })
+
+  const loaded = await Promise.all(
+    markdownEntries.map(async ({ filePath, subfolder }) => {
+      const content = await fs.promises
+        .readFile(filePath, 'utf8')
+        .catch((cause) => {
+          forumLogger.warn(`Failed to read forum file ${filePath}:`, cause)
+          return null
+        })
+      if (content === null) return null
+
+      const parsed = parseFrontmatter({ markdown: content })
+      const threadIdFromFrontmatter = getStringValue({
+        value: parsed.frontmatter.threadId,
+      })
+      const threadIdFromFilename = path.basename(filePath, '.md')
+      const threadId =
+        threadIdFromFrontmatter ||
+        (/^\d+$/.test(threadIdFromFilename) ? threadIdFromFilename : '')
+      if (!threadId) return null
+
+      const result: ExistingForumFile = {
+        filePath,
+        threadId,
+        frontmatter: parsed.frontmatter,
+        subfolder,
+      }
+      return result
+    }),
+  )
+
+  return loaded.filter((item): item is ExistingForumFile => item !== null)
+}

package/src/forum-sync/index.ts

@@ -0,0 +1,9 @@
+// Forum sync module entry point.
+// Re-exports the public API for forum <-> markdown synchronization.
+
+export {
+  startConfiguredForumSync,
+  stopConfiguredForumSync,
+} from './watchers.js'
+export { syncForumToFiles } from './sync-to-files.js'
+export { syncFilesToForum } from './sync-to-discord.js'