@otto-assistant/otto 0.1.1 → 0.7.15

package/dist/format-tables.test.js

@@ -0,0 +1,478 @@
+import { test, expect, describe } from 'vitest';
+import { splitTablesFromMarkdown, buildTableComponents, } from './format-tables.js';
+import { Lexer } from 'marked';
+import { ComponentType } from 'discord.js';
+function isTableToken(token) {
+    return (token.type === 'table' &&
+        Object.hasOwn(token, 'header') &&
+        Object.hasOwn(token, 'rows'));
+}
+function parseTable(markdown) {
+    const lexer = new Lexer();
+    const tokens = lexer.lex(markdown);
+    const table = tokens.find((token) => {
+        return isTableToken(token);
+    });
+    if (!table || !isTableToken(table)) {
+        throw new Error('Expected markdown to contain a table token');
+    }
+    return table;
+}
+/** Extract the first container's children from buildTableComponents result */
+function getContainerChildren(segments) {
+    const seg = segments[0];
+    if (seg.type !== 'components') {
+        throw new Error('Expected components segment');
+    }
+    const container = seg.components[0];
+    if (!container || container.type !== ComponentType.Container) {
+        throw new Error('Expected first top-level component to be a container');
+    }
+    return container.components.map((component) => {
+        const content = component.type === ComponentType.TextDisplay ? component.content : undefined;
+        const divider = component.type === ComponentType.Separator ? component.divider : undefined;
+        const spacing = component.type === ComponentType.Separator ? component.spacing : undefined;
+        return {
+            type: component.type,
+            content,
+            divider,
+            spacing,
+        };
+    });
+}
+describe('buildTableComponents', () => {
+    test('builds container with key-value TextDisplays', () => {
+        const table = parseTable(`| Name | Age |
+| --- | --- |
+| Alice | 30 |
+| Bob | 25 |`);
+        const result = buildTableComponents(table);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Name** Alice
+      **Age** 30",
+                  "type": 10,
+                },
+                {
+                  "divider": true,
+                  "spacing": 1,
+                  "type": 14,
+                },
+                {
+                  "content": "**Name** Bob
+      **Age** 25",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('adds separators between row groups', () => {
+        const table = parseTable(`| Key | Value |
+| --- | --- |
+| a | 1 |
+| b | 2 |
+| c | 3 |`);
+        const result = buildTableComponents(table);
+        const types = getContainerChildren(result).map((c) => c.type);
+        // type 10 = TextDisplay, type 14 = Separator
+        expect(types).toMatchInlineSnapshot(`
+      [
+        10,
+        14,
+        10,
+        14,
+        10,
+      ]
+    `);
+    });
+    test('single-row table has one TextDisplay, no separators', () => {
+        const table = parseTable(`| Method | Endpoint |
+| --- | --- |
+| GET | /api/users |`);
+        const result = buildTableComponents(table);
+        const children = getContainerChildren(result);
+        expect(children).toHaveLength(1);
+        expect(children[0].type).toBe(10);
+        expect(children[0].content).toMatchInlineSnapshot(`
+      "**Method** GET
+      **Endpoint** /api/users"
+    `);
+    });
+    test('splits large table into multiple container segments', () => {
+        // 25 rows: exceeds 19 rows per container, so splits into 2 containers
+        const headers = '| A | B |';
+        const sep = '| --- | --- |';
+        const rows = Array.from({ length: 25 }, (_, i) => {
+            return `| ${i}a | ${i}b |`;
+        }).join('\n');
+        const table = parseTable(`${headers}\n${sep}\n${rows}`);
+        const result = buildTableComponents(table);
+        expect(result).toHaveLength(2);
+        expect(result[0].type).toBe('components');
+        expect(result[1].type).toBe('components');
+        // First container has 20 rows (20 TDs + 19 seps = 39 children)
+        const firstChildren = getContainerChildren([result[0]]);
+        expect(firstChildren).toHaveLength(20 + 19);
+        // Second container has 5 rows (5 TDs + 4 seps = 9 children)
+        const secondChildren = getContainerChildren([result[1]]);
+        expect(secondChildren).toHaveLength(5 + 4);
+    });
+    test('strips formatting from cells', () => {
+        const table = parseTable(`| Header | Value |
+| --- | --- |
+| **Bold text** | Normal |
+| *Italic* | \`code\` |`);
+        const result = buildTableComponents(table);
+        const children = getContainerChildren(result);
+        expect(children[0].content).toMatchInlineSnapshot(`
+      "**Header** Bold text
+      **Value** Normal"
+    `);
+    });
+    test('renders button cells as action rows inside the container', () => {
+        const table = parseTable(`| Name | Action |
+| --- | --- |
+| feature-a | <button id="delete-a" variant="secondary">Delete</button> |`);
+        const result = buildTableComponents(table, {
+            resolveButtonCustomId: ({ button }) => {
+                return `html_action:${button.id}`;
+            },
+        });
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Name** feature-a",
+                  "type": 10,
+                },
+                {
+                  "components": [
+                    {
+                      "custom_id": "html_action:delete-a",
+                      "disabled": false,
+                      "label": "Delete",
+                      "style": 2,
+                      "type": 2,
+                    },
+                  ],
+                  "type": 1,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('falls back to button text when no resolver is provided', () => {
+        const table = parseTable(`| Name | Action |
+| --- | --- |
+| feature-a | <button id="delete-a" variant="secondary">Delete</button> |`);
+        const result = buildTableComponents(table);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Name** feature-a
+      **Action** Delete",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('renders wide rows with buttons without using sections', () => {
+        const table = parseTable(`| Thread | Name | Status | Created | Folder | Action |
+| --- | --- | --- | --- | --- | --- |
+| thread | feature-a | merged | 1m ago | /tmp/feature-a | <button id="delete-a" variant="secondary">Delete</button> |`);
+        const result = buildTableComponents(table, {
+            resolveButtonCustomId: ({ button }) => {
+                return `html_action:${button.id}`;
+            },
+        });
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "components": [
+                {
+                  "content": "**Thread** thread
+      **Name** feature-a
+      **Status** merged
+      **Created** 1m ago
+      **Folder** /tmp/feature-a",
+                  "type": 10,
+                },
+                {
+                  "components": [
+                    {
+                      "custom_id": "html_action:delete-a",
+                      "disabled": false,
+                      "label": "Delete",
+                      "style": 2,
+                      "type": 2,
+                    },
+                  ],
+                  "type": 1,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+});
+describe('splitTablesFromMarkdown', () => {
+    test('returns single text segment for content without tables', () => {
+        const result = splitTablesFromMarkdown('Just some text.\n\nMore text.');
+        expect(result).toHaveLength(1);
+        expect(result[0].type).toBe('text');
+    });
+    test('returns single components segment for table-only content', () => {
+        const result = splitTablesFromMarkdown(`| A | B |
+| --- | --- |
+| 1 | 2 |`);
+        expect(result).toHaveLength(1);
+        expect(result[0].type).toBe('components');
+    });
+    test('splits text before and after table into separate segments', () => {
+        const result = splitTablesFromMarkdown(`Text before.
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Text after.`);
+        expect(result).toHaveLength(3);
+        expect(result[0].type).toBe('text');
+        expect(result[1].type).toBe('components');
+        expect(result[2].type).toBe('text');
+    });
+    test('handles multiple tables with text between', () => {
+        const result = splitTablesFromMarkdown(`First table:
+
+| A | B |
+| --- | --- |
+| 1 | 2 |
+
+Middle text.
+
+| X | Y |
+| --- | --- |
+| a | b |`);
+        expect(result).toHaveLength(4);
+        expect(result.map((s) => s.type)).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+        "components",
+      ]
+    `);
+    });
+    test('splits oversized table into multiple component segments', () => {
+        const headers = '| A | B |';
+        const sep = '| --- | --- |';
+        const rows = Array.from({ length: 25 }, (_, i) => {
+            return `| ${i}a | ${i}b |`;
+        }).join('\n');
+        const result = splitTablesFromMarkdown(`${headers}\n${sep}\n${rows}`);
+        // 25 rows splits into 2 container segments
+        expect(result).toHaveLength(2);
+        expect(result.every((s) => s.type === 'components')).toBe(true);
+    });
+    test('preserves code blocks alongside tables', () => {
+        const result = splitTablesFromMarkdown(`Some code:
+
+\`\`\`js
+const x = 1
+\`\`\`
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+
+Done.`);
+        const types = result.map((s) => s.type);
+        expect(types).toMatchInlineSnapshot(`
+      [
+        "text",
+        "components",
+        "text",
+      ]
+    `);
+    });
+    test('renders callout text inside an accented container', () => {
+        const result = splitTablesFromMarkdown(`<callout accent="#2b7fff">
+## Important
+
+Read this first.
+</callout>`);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "accent_color": 2850815,
+              "components": [
+                {
+                  "content": "## Important
+
+      Read this first.",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('renders tables inside callouts recursively', () => {
+        const result = splitTablesFromMarkdown(`<callout accent="#2b7fff">
+## Important
+
+| Key | Value |
+| --- | --- |
+| a | 1 |
+</callout>`);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "accent_color": 2850815,
+              "components": [
+                {
+                  "content": "## Important",
+                  "type": 10,
+                },
+                {
+                  "content": "**Key** a
+      **Value** 1",
+                  "type": 10,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('renders button rows inside callouts recursively', () => {
+        const result = splitTablesFromMarkdown(`<callout accent="#2b7fff">
+## Actions
+
+| Name | Action |
+| --- | --- |
+| feature-a | <button id="delete-a" variant="secondary">Delete</button> |
+</callout>`, {
+            resolveButtonCustomId: ({ button }) => {
+                return `html_action:${button.id}`;
+            },
+        });
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "components": [
+            {
+              "accent_color": 2850815,
+              "components": [
+                {
+                  "content": "## Actions",
+                  "type": 10,
+                },
+                {
+                  "content": "**Name** feature-a",
+                  "type": 10,
+                },
+                {
+                  "components": [
+                    {
+                      "custom_id": "html_action:delete-a",
+                      "disabled": false,
+                      "label": "Delete",
+                      "style": 2,
+                      "type": 2,
+                    },
+                  ],
+                  "type": 1,
+                },
+              ],
+              "type": 17,
+            },
+          ],
+          "type": "components",
+        },
+      ]
+    `);
+    });
+    test('renders callout that was prefixed with ⬥ as plain text (regression)', () => {
+        // Before the fix, formatPart would add ⬥ prefix to callout lines,
+        // breaking the callout parser. Now formatPart skips the prefix for callouts.
+        const result = splitTablesFromMarkdown(`⬥ <callout accent="#ef4444">
+## Top priority
+- **Stripe dispute** deadline
+</callout>`);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "text": "⬥ <callout accent="#ef4444">
+      ## Top priority
+      - **Stripe dispute** deadline
+      </callout>",
+          "type": "text",
+        },
+      ]
+    `);
+    });
+    test('falls back to plain text when a callout is not closed', () => {
+        const result = splitTablesFromMarkdown(`<callout accent="#2b7fff">
+## Important
+
+Still open`);
+        expect(result).toMatchInlineSnapshot(`
+      [
+        {
+          "text": "<callout accent="#2b7fff">
+      ## Important
+
+      Still open",
+          "type": "text",
+        },
+      ]
+    `);
+    });
+});

package/dist/forum-sync/config.js

@@ -0,0 +1,79 @@
+// Forum sync configuration from SQLite database.
+// Reads forum_sync_configs table and resolves relative output dirs.
+// On first run, migrates any existing forum-sync.json into the DB.
+import fs from 'node:fs';
+import path from 'node:path';
+import YAML from 'yaml';
+import { getDataDir } from '../config.js';
+import { getForumSyncConfigs, upsertForumSyncConfig } from '../database.js';
+import { createLogger } from '../logger.js';
+const forumLogger = createLogger('FORUM');
+const LEGACY_CONFIG_FILE = 'forum-sync.json';
+function isForumSyncDirection(value) {
+    return value === 'discord-to-files' || value === 'bidirectional';
+}
+function resolveOutputDir(outputDir) {
+    if (path.isAbsolute(outputDir))
+        return outputDir;
+    return path.resolve(getDataDir(), outputDir);
+}
+/**
+ * One-time migration: if the legacy forum-sync.json exists, import its entries
+ * into the DB and rename the file so it's not re-imported on next startup.
+ */
+async function migrateLegacyConfig({ appId }) {
+    const configPath = path.join(getDataDir(), LEGACY_CONFIG_FILE);
+    if (!fs.existsSync(configPath))
+        return;
+    forumLogger.log(`Migrating legacy ${LEGACY_CONFIG_FILE} into database...`);
+    const raw = fs.readFileSync(configPath, 'utf8');
+    let parsed;
+    try {
+        parsed = YAML.parse(raw);
+    }
+    catch {
+        forumLogger.warn(`Failed to parse legacy ${LEGACY_CONFIG_FILE}, skipping migration`);
+        return;
+    }
+    if (!parsed || typeof parsed !== 'object')
+        return;
+    const forums = parsed.forums;
+    if (!Array.isArray(forums))
+        return;
+    for (const item of forums) {
+        if (!item || typeof item !== 'object')
+            continue;
+        const entry = item;
+        const forumChannelId = typeof entry.forumChannelId === 'string' ? entry.forumChannelId : '';
+        const outputDir = typeof entry.outputDir === 'string' ? entry.outputDir : '';
+        const direction = isForumSyncDirection(entry.direction)
+            ? entry.direction
+            : 'bidirectional';
+        if (!forumChannelId || !outputDir)
+            continue;
+        await upsertForumSyncConfig({
+            appId,
+            forumChannelId,
+            outputDir: resolveOutputDir(outputDir),
+            direction,
+        });
+    }
+    // Rename so we don't re-import next time
+    const backupPath = configPath + '.migrated';
+    fs.renameSync(configPath, backupPath);
+    forumLogger.log(`Legacy config migrated and renamed to ${path.basename(backupPath)}`);
+}
+export async function readForumSyncConfig({ appId }) {
+    if (!appId)
+        return [];
+    // Migrate legacy JSON file on first run
+    await migrateLegacyConfig({ appId });
+    const rows = await getForumSyncConfigs({ appId });
+    return rows.map((row) => ({
+        forumChannelId: row.forumChannelId,
+        outputDir: resolveOutputDir(row.outputDir),
+        direction: isForumSyncDirection(row.direction)
+            ? row.direction
+            : 'bidirectional',
+    }));
+}

package/dist/forum-sync/discord-operations.js

@@ -0,0 +1,154 @@
+// Discord API operations for forum sync.
+// Resolves forum channels, fetches threads (active + archived) with pagination,
+// fetches thread messages, loads existing forum files from disk, and ensures directories.
+import fs from 'node:fs';
+import path from 'node:path';
+import { ChannelType, } from 'discord.js';
+import { createLogger } from '../logger.js';
+import { parseFrontmatter, getStringValue } from './markdown.js';
+import { DEFAULT_RATE_LIMIT_DELAY_MS, ForumChannelResolveError, ForumSyncOperationError, delay, } from './types.js';
+const forumLogger = createLogger('FORUM');
+export function getCanonicalThreadFilePath({ outputDir, threadId, subfolder, }) {
+    if (subfolder) {
+        return path.join(outputDir, subfolder, `${threadId}.md`);
+    }
+    return path.join(outputDir, `${threadId}.md`);
+}
+export async function ensureDirectory({ directory }) {
+    const result = await fs.promises.mkdir(directory, { recursive: true }).catch((cause) => new ForumSyncOperationError({
+        forumChannelId: 'unknown',
+        reason: directory,
+        cause,
+    }));
+    if (result instanceof Error)
+        return result;
+}
+export async function resolveForumChannel({ discordClient, forumChannelId, }) {
+    const channel = await discordClient.channels
+        .fetch(forumChannelId)
+        .catch((cause) => new ForumChannelResolveError({ forumChannelId, cause }));
+    if (channel instanceof Error)
+        return channel;
+    if (!channel || channel.type !== ChannelType.GuildForum) {
+        return new ForumChannelResolveError({ forumChannelId });
+    }
+    return channel;
+}
+export async function fetchForumThreads({ forumChannel, }) {
+    const byId = new Map();
+    const active = await forumChannel.threads.fetchActive().catch((cause) => new ForumSyncOperationError({
+        forumChannelId: forumChannel.id,
+        reason: 'fetchActive failed',
+        cause,
+    }));
+    if (active instanceof Error)
+        return active;
+    for (const [id, thread] of active.threads) {
+        byId.set(id, thread);
+    }
+    let before;
+    while (true) {
+        const archived = await forumChannel.threads
+            .fetchArchived({ type: 'public', limit: 100, before })
+            .catch((cause) => new ForumSyncOperationError({
+            forumChannelId: forumChannel.id,
+            reason: 'fetchArchived failed',
+            cause,
+        }));
+        if (archived instanceof Error)
+            return archived;
+        const threads = Array.from(archived.threads.values());
+        for (const thread of threads) {
+            byId.set(thread.id, thread);
+        }
+        if (!archived.hasMore || threads.length === 0)
+            break;
+        const timestamps = threads
+            .map((thread) => thread.archiveTimestamp ?? thread.createdTimestamp)
+            .filter((value) => value !== null);
+        const oldestTimestamp = Math.min(...timestamps);
+        if (!Number.isFinite(oldestTimestamp))
+            break;
+        before = new Date(oldestTimestamp - 1);
+        await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS });
+    }
+    return Array.from(byId.values());
+}
+export async function fetchThreadMessages({ thread, }) {
+    const byId = new Map();
+    let before;
+    while (true) {
+        const fetched = await thread.messages.fetch({ limit: 100, before }).catch((cause) => new ForumSyncOperationError({
+            forumChannelId: thread.parentId || 'unknown',
+            reason: `message fetch failed for thread ${thread.id}`,
+            cause,
+        }));
+        if (fetched instanceof Error)
+            return fetched;
+        const messages = Array.from(fetched.values());
+        for (const message of messages) {
+            byId.set(message.id, message);
+        }
+        if (messages.length < 100 || messages.length === 0)
+            break;
+        // Find oldest message for cursor - messages are sorted by Discord, last is oldest
+        const oldest = messages[messages.length - 1];
+        if (!oldest)
+            break;
+        before = oldest.id;
+        await delay({ ms: DEFAULT_RATE_LIMIT_DELAY_MS });
+    }
+    return Array.from(byId.values()).sort((a, b) => a.createdTimestamp - b.createdTimestamp);
+}
+/**
+ * Recursively walks a directory collecting all .md files with their relative subfolder path.
+ */
+async function collectMarkdownFiles({ dir, outputDir, }) {
+    if (!fs.existsSync(dir))
+        return [];
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    const relativeSub = path.relative(outputDir, dir);
+    const subfolder = relativeSub && relativeSub !== '.' ? relativeSub : undefined;
+    const mdFiles = entries
+        .filter((entry) => entry.isFile() && entry.name.endsWith('.md'))
+        .map((entry) => ({ filePath: path.join(dir, entry.name), subfolder }));
+    const subdirs = entries.filter((entry) => entry.isDirectory());
+    const nestedResults = await Promise.all(subdirs.map((subdir) => collectMarkdownFiles({
+        dir: path.join(dir, subdir.name),
+        outputDir,
+    })));
+    return [...mdFiles, ...nestedResults.flat()];
+}
+export async function loadExistingForumFiles({ outputDir, }) {
+    const markdownEntries = await collectMarkdownFiles({
+        dir: outputDir,
+        outputDir,
+    });
+    const loaded = await Promise.all(markdownEntries.map(async ({ filePath, subfolder }) => {
+        const content = await fs.promises
+            .readFile(filePath, 'utf8')
+            .catch((cause) => {
+            forumLogger.warn(`Failed to read forum file ${filePath}:`, cause);
+            return null;
+        });
+        if (content === null)
+            return null;
+        const parsed = parseFrontmatter({ markdown: content });
+        const threadIdFromFrontmatter = getStringValue({
+            value: parsed.frontmatter.threadId,
+        });
+        const threadIdFromFilename = path.basename(filePath, '.md');
+        const threadId = threadIdFromFrontmatter ||
+            (/^\d+$/.test(threadIdFromFilename) ? threadIdFromFilename : '');
+        if (!threadId)
+            return null;
+        const result = {
+            filePath,
+            threadId,
+            frontmatter: parsed.frontmatter,
+            subfolder,
+        };
+        return result;
+    }));
+    return loaded.filter((item) => item !== null);
+}