@brianli/kimaki 0.4.72-brianli.1

package/dist/format-tables.js

@@ -0,0 +1,122 @@
+// Markdown table formatter for Discord.
+// Converts GFM tables to Discord Components V2 (ContainerBuilder with TextDisplay
+// key-value pairs and Separators between row groups). Large tables are split
+// across multiple Container components to stay within the 40-component limit.
+import { Lexer } from 'marked';
+import { SeparatorSpacingSize, } from 'discord.js';
+// Max 40 components per message (nested components count toward the limit).
+// Each container uses: 1 (container) + M (TextDisplays) + M-1 (separators) = 2M children.
+// So max rows per container = floor((40 - 1) / 2) = 19.
+const MAX_COMPONENTS = 40;
+const MAX_ROWS_PER_CONTAINER = Math.floor((MAX_COMPONENTS - 1) / 2);
+/**
+ * Split markdown into text and table component segments.
+ * Tables are rendered as CV2 Container components with bold key-value TextDisplay
+ * pairs. Large tables are split across multiple component segments.
+ */
+export function splitTablesFromMarkdown(markdown) {
+    const lexer = new Lexer();
+    const tokens = lexer.lex(markdown);
+    const segments = [];
+    let textBuffer = '';
+    for (const token of tokens) {
+        if (token.type === 'table') {
+            if (textBuffer.trim()) {
+                segments.push({ type: 'text', text: textBuffer });
+                textBuffer = '';
+            }
+            const componentSegments = buildTableComponents(token);
+            segments.push(...componentSegments);
+        }
+        else {
+            textBuffer += token.raw;
+        }
+    }
+    if (textBuffer.trim()) {
+        segments.push({ type: 'text', text: textBuffer });
+    }
+    return segments;
+}
+/**
+ * Build CV2 components for a table. Each data row becomes a single TextDisplay
+ * with all key-value pairs joined by newlines (header bold as key). Separator
+ * dividers are placed between row groups.
+ * Large tables are split into multiple component segments, each containing a
+ * Container with up to MAX_ROWS_PER_CONTAINER rows.
+ */
+export function buildTableComponents(table) {
+    const headers = table.header.map((cell) => {
+        return extractCellText(cell.tokens);
+    });
+    const rows = table.rows.map((row) => {
+        return row.map((cell) => {
+            return extractCellText(cell.tokens);
+        });
+    });
+    // Split rows into chunks that fit within the component limit
+    const chunks = [];
+    for (let i = 0; i < rows.length; i += MAX_ROWS_PER_CONTAINER) {
+        chunks.push(rows.slice(i, i + MAX_ROWS_PER_CONTAINER));
+    }
+    return chunks.map((chunkRows) => {
+        const children = [];
+        for (let i = 0; i < chunkRows.length; i++) {
+            if (i > 0) {
+                children.push({
+                    type: 14,
+                    divider: true,
+                    spacing: SeparatorSpacingSize.Small,
+                });
+            }
+            const row = chunkRows[i];
+            const lines = headers.map((key, j) => {
+                const value = row[j] || '';
+                return `**${key}** ${value}`;
+            });
+            children.push({ type: 10, content: lines.join('\n') });
+        }
+        const container = {
+            type: 17,
+            components: children,
+        };
+        return {
+            type: 'components',
+            components: [container],
+        };
+    });
+}
+function extractCellText(tokens) {
+    const parts = [];
+    for (const token of tokens) {
+        parts.push(extractTokenText(token));
+    }
+    return parts.join('').trim();
+}
+function extractTokenText(token) {
+    switch (token.type) {
+        case 'text':
+        case 'codespan':
+        case 'escape':
+            return token.text;
+        case 'link':
+            return token.href;
+        case 'image':
+            return token.href;
+        case 'strong':
+        case 'em':
+        case 'del':
+            return token.tokens ? extractCellText(token.tokens) : token.text;
+        case 'br':
+            return ' ';
+        default: {
+            const tokenAny = token;
+            if (tokenAny.tokens && Array.isArray(tokenAny.tokens)) {
+                return extractCellText(tokenAny.tokens);
+            }
+            if (typeof tokenAny.text === 'string') {
+                return tokenAny.text;
+            }
+            return '';
+        }
+    }
+}

package/dist/format-tables.test.js

@@ -0,0 +1,199 @@
+import { test, expect, describe } from 'vitest';
+import { splitTablesFromMarkdown, buildTableComponents, } from './format-tables.js';
+import { Lexer } from 'marked';
+function parseTable(markdown) {
+    const lexer = new Lexer();
+    const tokens = lexer.lex(markdown);
+    return tokens.find((t) => t.type === 'table');
+}
+/** Extract the first container's children from buildTableComponents result */
+function getContainerChildren(segments) {
+    const seg = segments[0];
+    if (seg.type !== 'components') {
+        throw new Error('Expected components segment');
+    }
+    const container = seg.components[0];
+    return container.components;
+}
+describe('buildTableComponents', () => {
+    test('builds container with key-value TextDisplays', () => {
+        const table = parseTable(`| Name | Age |
+| --- | --- |
+| Alice | 30 |
+| Bob | 25 |`);
+        const result = buildTableComponents(table);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Name** Alice
+      **Age** 30",
+                  "type": 10,
+                },
+                {
+                  "divider": true,
+                  "spacing": 1,
+                  "type": 14,
+                },
+                {
+                  "content": "**Name** Bob
+      **Age** 25",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('adds separators between row groups', () => {
+        const table = parseTable(`| Key | Value |
+| --- | --- |
+| a | 1 |
+| b | 2 |
+| c | 3 |`);
+        const result = buildTableComponents(table);
+        const types = getContainerChildren(result).map((c) => c.type);
+        // type 10 = TextDisplay, type 14 = Separator
+        expect(types).toMatchInlineSnapshot(`
+      [
+        10,
+        14,
+        10,
+        14,
+        10,
+      ]
+    `);
+    });
+    test('single-row table has one TextDisplay, no separators', () => {
+        const table = parseTable(`| Method | Endpoint |
+| --- | --- |
+| GET | /api/users |`);
+        const result = buildTableComponents(table);
+        const children = getContainerChildren(result);
+        expect(children).toHaveLength(1);
+        expect(children[0].type).toBe(10);
+        expect(children[0].content).toMatchInlineSnapshot(`
+      "**Method** GET
+      **Endpoint** /api/users"
+    `);
+    });
+    test('splits large table into multiple container segments', () => {
+        // 25 rows: exceeds 19 rows per container, so splits into 2 containers
+        const headers = '| A | B |';
+        const sep = '| --- | --- |';
+        const rows = Array.from({ length: 25 }, (_, i) => {
+            return `| ${i}a | ${i}b |`;
+        }).join('\n');
+        const table = parseTable(`${headers}\n${sep}\n${rows}`);
+        const result = buildTableComponents(table);
+        expect(result).toHaveLength(2);
+        expect(result[0].type).toBe('components');
+        expect(result[1].type).toBe('components');
+        // First container has 19 rows (19 TDs + 18 seps = 37 children)
+        const firstChildren = getContainerChildren([result[0]]);
+        expect(firstChildren).toHaveLength(19 + 18);
+        // Second container has 6 rows (6 TDs + 5 seps = 11 children)
+        const secondChildren = getContainerChildren([result[1]]);
+        expect(secondChildren).toHaveLength(6 + 5);
+    });
+    test('strips formatting from cells', () => {
+        const table = parseTable(`| Header | Value |
+| --- | --- |
+| **Bold text** | Normal |
+| *Italic* | \`code\` |`);
+        const result = buildTableComponents(table);
+        const children = getContainerChildren(result);
+        expect(children[0].content).toMatchInlineSnapshot(`
+      "**Header** Bold text
+      **Value** Normal"
+    `);
+    });
+});
+describe('splitTablesFromMarkdown', () => {
+    test('returns single text segment for content without tables', () => {
+        const result = splitTablesFromMarkdown('Just some text.\n\nMore text.');
+        expect(result).toHaveLength(1);
+        expect(result[0].type).toBe('text');
+    });
+    test('returns single components segment for table-only content', () => {
+        const result = splitTablesFromMarkdown(`| A | B |
+| --- | --- |
+| 1 | 2 |`);
+        expect(result).toHaveLength(1);
+        expect(result[0].type).toBe('components');
+    });
+    test('splits text before and after table into separate segments', () => {
+        const result = splitTablesFromMarkdown(`Text before.
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Text after.`);
+        expect(result).toHaveLength(3);
+        expect(result[0].type).toBe('text');
+        expect(result[1].type).toBe('components');
+        expect(result[2].type).toBe('text');
+    });
+    test('handles multiple tables with text between', () => {
+        const result = splitTablesFromMarkdown(`First table:
+
+| A | B |
+| --- | --- |
+| 1 | 2 |
+
+Middle text.
+
+| X | Y |
+| --- | --- |
+| a | b |`);
+        expect(result).toHaveLength(4);
+        expect(result.map((s) => s.type)).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+        "components",
+      ]
+    `);
+    });
+    test('splits oversized table into multiple component segments', () => {
+        const headers = '| A | B |';
+        const sep = '| --- | --- |';
+        const rows = Array.from({ length: 25 }, (_, i) => {
+            return `| ${i}a | ${i}b |`;
+        }).join('\n');
+        const result = splitTablesFromMarkdown(`${headers}\n${sep}\n${rows}`);
+        // 25 rows splits into 2 container segments
+        expect(result).toHaveLength(2);
+        expect(result.every((s) => s.type === 'components')).toBe(true);
+    });
+    test('preserves code blocks alongside tables', () => {
+        const result = splitTablesFromMarkdown(`Some code:
+
+\`\`\`js
+const x = 1
+\`\`\`
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Done.`);
+        const types = result.map((s) => s.type);
+        expect(types).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+      ]
+    `);
+    });
+});

package/dist/forum-sync/config.js

@@ -0,0 +1,79 @@
+// Forum sync configuration from SQLite database.
+// Reads forum_sync_configs table and resolves relative output dirs.
+// On first run, migrates any existing forum-sync.json into the DB.
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'js-yaml';
+import { getDataDir } from '../config.js';
+import { getForumSyncConfigs, upsertForumSyncConfig } from '../database.js';
+import { createLogger } from '../logger.js';
+const forumLogger = createLogger('FORUM');
+const LEGACY_CONFIG_FILE = 'forum-sync.json';
+function isForumSyncDirection(value) {
+    return value === 'discord-to-files' || value === 'bidirectional';
+}
+function resolveOutputDir(outputDir) {
+    if (path.isAbsolute(outputDir))
+        return outputDir;
+    return path.resolve(getDataDir(), outputDir);
+}
+/**
+ * One-time migration: if the legacy forum-sync.json exists, import its entries
+ * into the DB and rename the file so it's not re-imported on next startup.
+ */
+async function migrateLegacyConfig({ appId }) {
+    const configPath = path.join(getDataDir(), LEGACY_CONFIG_FILE);
+    if (!fs.existsSync(configPath))
+        return;
+    forumLogger.log(`Migrating legacy ${LEGACY_CONFIG_FILE} into database...`);
+    const raw = fs.readFileSync(configPath, 'utf8');
+    let parsed;
+    try {
+        parsed = yaml.load(raw);
+    }
+    catch {
+        forumLogger.warn(`Failed to parse legacy ${LEGACY_CONFIG_FILE}, skipping migration`);
+        return;
+    }
+    if (!parsed || typeof parsed !== 'object')
+        return;
+    const forums = parsed.forums;
+    if (!Array.isArray(forums))
+        return;
+    for (const item of forums) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const entry = item;
+        const forumChannelId = typeof entry.forumChannelId === 'string' ? entry.forumChannelId : '';
+        const outputDir = typeof entry.outputDir === 'string' ? entry.outputDir : '';
+        const direction = isForumSyncDirection(entry.direction)
+            ? entry.direction
+            : 'bidirectional';
+        if (!forumChannelId || !outputDir)
+            continue;
+        await upsertForumSyncConfig({
+            appId,
+            forumChannelId,
+            outputDir: resolveOutputDir(outputDir),
+            direction,
+        });
+    }
+    // Rename so we don't re-import next time
+    const backupPath = configPath + '.migrated';
+    fs.renameSync(configPath, backupPath);
+    forumLogger.log(`Legacy config migrated and renamed to ${path.basename(backupPath)}`);
+}
+export async function readForumSyncConfig({ appId }) {
+    if (!appId)
+        return [];
+    // Migrate legacy JSON file on first run
+    await migrateLegacyConfig({ appId });
+    const rows = await getForumSyncConfigs({ appId });
+    return rows.map((row) => ({
+        forumChannelId: row.forumChannelId,
+        outputDir: resolveOutputDir(row.outputDir),
+        direction: isForumSyncDirection(row.direction)
+            ? row.direction
+            : 'bidirectional',
+    }));
+}

package/dist/forum-sync/discord-operations.js

@@ -0,0 +1,154 @@
+// Discord API operations for forum sync.
+// Resolves forum channels, fetches threads (active + archived) with pagination,
+// fetches thread messages, loads existing forum files from disk, and ensures directories.
+import fs from 'node:fs';
+import path from 'node:path';
+import { ChannelType, } from 'discord.js';
+import { createLogger } from '../logger.js';
+import { parseFrontmatter, getStringValue } from './markdown.js';
+import { DEFAULT_RATE_LIMIT_DELAY_MS, ForumChannelResolveError, ForumSyncOperationError, delay, } from './types.js';
+const forumLogger = createLogger('FORUM');
+export function getCanonicalThreadFilePath({ outputDir, threadId, subfolder, }) {
+    if (subfolder) {
+        return path.join(outputDir, subfolder, `${threadId}.md`);
+    }
+    return path.join(outputDir, `${threadId}.md`);
+}
+export async function ensureDirectory({ directory }) {
+    const result = await fs.promises.mkdir(directory, { recursive: true }).catch((cause) => new ForumSyncOperationError({
+        forumChannelId: 'unknown',
+        reason: directory,
+        cause,
+    }));
+    if (result instanceof Error)
+        return result;
+}
+export async function resolveForumChannel({ discordClient, forumChannelId, }) {
+    const channel = await discordClient.channels
+        .fetch(forumChannelId)
+        .catch((cause) => new ForumChannelResolveError({ forumChannelId, cause }));
+    if (channel instanceof Error)
+        return channel;
+    if (!channel || channel.type !== ChannelType.GuildForum) {
+        return new ForumChannelResolveError({ forumChannelId });
+    }
+    return channel;
+}
+export async function fetchForumThreads({ forumChannel, }) {
+    const byId = new Map();
+    const active = await forumChannel.threads.fetchActive().catch((cause) => new ForumSyncOperationError({
+        forumChannelId: forumChannel.id,
+        reason: 'fetchActive failed',
+        cause,
+    }));
+    if (active instanceof Error)
+        return active;
+    for (const [id, thread] of active.threads) {
+        byId.set(id, thread);
+    }
+    let before;
+    while (true) {
+        const archived = await forumChannel.threads
+            .fetchArchived({ type: 'public', limit: 100, before })
+            .catch((cause) => new ForumSyncOperationError({
+            forumChannelId: forumChannel.id,
+            reason: 'fetchArchived failed',
+            cause,
+        }));
+        if (archived instanceof Error)
+            return archived;
+        const threads = Array.from(archived.threads.values());
+        for (const thread of threads) {
+            byId.set(thread.id, thread);
+        }
+        if (!archived.hasMore || threads.length === 0)
+            break;
+        const timestamps = threads
+            .map((thread) => thread.archiveTimestamp ?? thread.createdTimestamp)
+            .filter((value) => value !== null);
+        const oldestTimestamp = Math.min(...timestamps);
+        if (!Number.isFinite(oldestTimestamp))
+            break;
+        before = new Date(oldestTimestamp - 1);
+        await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS });
+    }
+    return Array.from(byId.values());
+}
+export async function fetchThreadMessages({ thread, }) {
+    const byId = new Map();
+    let before;
+    while (true) {
+        const fetched = await thread.messages.fetch({ limit: 100, before }).catch((cause) => new ForumSyncOperationError({
+            forumChannelId: thread.parentId || 'unknown',
+            reason: `message fetch failed for thread ${thread.id}`,
+            cause,
+        }));
+        if (fetched instanceof Error)
+            return fetched;
+        const messages = Array.from(fetched.values());
+        for (const message of messages) {
+            byId.set(message.id, message);
+        }
+        if (messages.length < 100 || messages.length === 0)
+            break;
+        // Find oldest message for cursor - messages are sorted by Discord, last is oldest
+        const oldest = messages[messages.length - 1];
+        if (!oldest)
+            break;
+        before = oldest.id;
+        await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS });
+    }
+    return Array.from(byId.values()).sort((a, b) => a.createdTimestamp - b.createdTimestamp);
+}
+/**
+ * Recursively walks a directory collecting all .md files with their relative subfolder path.
+ */
+async function collectMarkdownFiles({ dir, outputDir, }) {
+    if (!fs.existsSync(dir))
+        return [];
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    const relativeSub = path.relative(outputDir, dir);
+    const subfolder = relativeSub && relativeSub !== '.' ? relativeSub : undefined;
+    const mdFiles = entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+        .map((entry) => ({ filePath: path.join(dir, entry.name), subfolder }));
+    const subdirs = entries.filter((entry) => entry.isDirectory());
+    const nestedResults = await Promise.all(subdirs.map((subdir) => collectMarkdownFiles({
+        dir: path.join(dir, subdir.name),
+        outputDir,
+    })));
+    return [...mdFiles, ...nestedResults.flat()];
+}
+export async function loadExistingForumFiles({ outputDir, }) {
+    const markdownEntries = await collectMarkdownFiles({
+        dir: outputDir,
+        outputDir,
+    });
+    const loaded = await Promise.all(markdownEntries.map(async ({ filePath, subfolder }) => {
+        const content = await fs.promises
+            .readFile(filePath, 'utf8')
+            .catch((cause) => {
+            forumLogger.warn(`Failed to read forum file ${filePath}:`, cause);
+            return null;
+        });
+        if (content === null)
+            return null;
+        const parsed = parseFrontmatter({ markdown: content });
+        const threadIdFromFrontmatter = getStringValue({
+            value: parsed.frontmatter.threadId,
+        });
+        const threadIdFromFilename = path.basename(filePath, '.md');
+        const threadId = threadIdFromFrontmatter ||
+            (/^\d+$/.test(threadIdFromFilename) ? threadIdFromFilename : '');
+        if (!threadId)
+            return null;
+        const result = {
+            filePath,
+            threadId,
+            frontmatter: parsed.frontmatter,
+            subfolder,
+        };
+        return result;
+    }));
+    return loaded.filter((item) => item !== null);
+}

package/dist/forum-sync/index.js

@@ -0,0 +1,5 @@
+// Forum sync module entry point.
+// Re-exports the public API for forum <-> markdown synchronization.
+export { startConfiguredForumSync, stopConfiguredForumSync, } from './watchers.js';
+export { syncForumToFiles } from './sync-to-files.js';
+export { syncFilesToForum } from './sync-to-discord.js';

package/dist/forum-sync/markdown.js

@@ -0,0 +1,117 @@
+// Markdown parsing, serialization, and section formatting for forum sync.
+// Handles frontmatter extraction, message section building, and
+// conversion between Discord messages and markdown format.
+import yaml from 'js-yaml';
+import * as errore from 'errore';
+import { ForumFrontmatterParseError, } from './types.js';
+export function toStringArray({ value }) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((item) => typeof item === 'string');
+}
+export function getStringValue({ value }) {
+    if (typeof value !== 'string')
+        return '';
+    return value;
+}
+export function parseFrontmatter({ markdown, }) {
+    if (!markdown.startsWith('---\n')) {
+        return { frontmatter: {}, body: markdown.trim() };
+    }
+    const end = markdown.indexOf('\n---\n', 4);
+    if (end === -1) {
+        return { frontmatter: {}, body: markdown.trim() };
+    }
+    const rawFrontmatter = markdown.slice(4, end);
+    const body = markdown.slice(end + 5).trim();
+    const parsed = errore.try({
+        try: () => yaml.load(rawFrontmatter),
+        catch: (cause) => new ForumFrontmatterParseError({ reason: 'yaml parse failed', cause }),
+    });
+    if (parsed instanceof Error || !parsed || typeof parsed !== 'object') {
+        return { frontmatter: {}, body };
+    }
+    return { frontmatter: parsed, body };
+}
+export function stringifyFrontmatter({ frontmatter, body, }) {
+    const yamlText = yaml
+        .dump(frontmatter, {
+        lineWidth: 120,
+        noRefs: true,
+        sortKeys: false,
+    })
+        .trim();
+    return `---\n${yamlText}\n---\n\n${body.trim()}\n`;
+}
+export function splitSections({ body }) {
+    return body
+        .split(/\r?\n---\r?\n/g)
+        .map((part) => part.trim())
+        .filter((part) => part.length > 0);
+}
+export function extractStarterContent({ body }) {
+    const sections = splitSections({ body });
+    const firstSection = sections[0] || '';
+    const match = firstSection.match(/^\*\*.+?\*\* \(\d+\) - .+?(?: \(edited .+?\))?\r?\n\r?\n([\s\S]*)$/);
+    if (!match)
+        return body.trim();
+    return (match[1] || '').trim();
+}
+export function buildMessageSections({ messages, }) {
+    return messages.map((message) => {
+        const attachmentLines = Array.from(message.attachments.values()).map((attachment) => `Attachment: ${attachment.url}`);
+        const contentParts = [];
+        const trimmedContent = message.content.trim();
+        if (trimmedContent) {
+            contentParts.push(trimmedContent);
+        }
+        if (attachmentLines.length > 0) {
+            contentParts.push(attachmentLines.join('\n'));
+        }
+        const content = contentParts.length > 0
+            ? contentParts.join('\n\n')
+            : '_(no text content)_';
+        return {
+            messageId: message.id,
+            authorName: message.author.username,
+            authorId: message.author.id,
+            createdAt: new Date(message.createdTimestamp).toISOString(),
+            editedAt: message.editedTimestamp
+                ? new Date(message.editedTimestamp).toISOString()
+                : null,
+            content,
+        };
+    });
+}
+export function formatMessageSection({ section, }) {
+    const editedSuffix = section.editedAt ? ` (edited ${section.editedAt})` : '';
+    return `**${section.authorName}** (${section.authorId}) - ${section.createdAt}${editedSuffix}\n\n${section.content}`;
+}
+// Channel mention footer stored in the Discord starter message so
+// projectChannelId survives a full re-sync from Discord (no local files).
+// Uses <#id> so Discord renders it as a clickable channel link.
+// Matches at start-of-string or after a newline so it works even when the
+// footer is the only content in the message (e.g. empty body).
+const PROJECT_CHANNEL_FOOTER_RE = /(?:^|\n)channel: <#(\d{17,20})>\s*$/;
+const MAX_STARTER_MESSAGE_LENGTH = 2_000;
+/** Append a channel mention footer, truncating the body so the total
+ *  never exceeds Discord's 2000-char starter message limit. */
+export function appendProjectChannelFooter({ content, projectChannelId, }) {
+    if (!projectChannelId)
+        return content;
+    const footer = `\nchannel: <#${projectChannelId}>`;
+    const maxContentLength = MAX_STARTER_MESSAGE_LENGTH - footer.length;
+    const truncated = content.length > maxContentLength
+        ? content.slice(0, maxContentLength)
+        : content;
+    return `${truncated}${footer}`;
+}
+export function extractProjectChannelFromContent({ content, }) {
+    const match = content.match(PROJECT_CHANNEL_FOOTER_RE);
+    if (!match)
+        return { cleanContent: content };
+    return {
+        cleanContent: content.replace(PROJECT_CHANNEL_FOOTER_RE, '').trim(),
+        projectChannelId: match[1],
+    };
+}