@abraca/cli 1.9.1 → 2.4.0

package/src/commands/wiki/index.ts

@@ -0,0 +1,471 @@
+/**
+ * `abracadabra wiki <title>` — fetch Wikipedia articles and seed them into
+ * the active space as a graph of docs.
+ *
+ * Streaming flow (no buffering): every newly-discovered title becomes a
+ * shell doc immediately (visible in the dashboard right away), then bodies
+ * are filled in one fetch at a time. The user sees the tree skeleton appear
+ * before the first body is written.
+ *
+ * The command authenticates separately from the parent CLI (it's listed in
+ * NO_CONNECT_COMMANDS in src/index.ts) so the parent harness doesn't open a
+ * second connection. We use the modern DocumentManager API from @abraca/dabra.
+ */
+import type { DocumentManager } from '@abraca/dabra'
+import { registerCommand } from '../../command.ts'
+import type { CLIConnection } from '../../connection.ts'
+import type { ParsedArgs } from '../../parser.ts'
+import { WikipediaClient } from './wikipedia.ts'
+import { snapshotArticle, canonicalTitle, prettyCategoryLabel } from './snapshot.ts'
+import {
+  ICONS,
+  pickSectionType,
+  renderArticleLead,
+  renderArticleSingleDoc,
+  renderInfoboxBody,
+  renderCategoryBody,
+  rewriteLinks,
+} from './render.ts'
+import { openSession } from './connect.ts'
+import type { WikiOptions, ExtractMode, ExtractedArticle, ExtractedSection } from './types.ts'
+
+registerCommand({
+  name: 'wiki',
+  aliases: ['wikipedia'],
+  description: 'Fetch Wikipedia articles into a graph of docs (streaming).',
+  usage: [
+    'wiki "<Article Title>"',
+    '  mode=single|split   single doc per article OR split into sections+infobox  [split]',
+    '  depth=<n>           follow internal links to depth N                       [1]',
+    '  category-depth=<n>  recurse into sub-categories                            [1]',
+    '  lang=<code>         wiki language                                          [en]',
+    '  domain=<host>       3rd-party MediaWiki host (overrides lang)',
+    '  parent=<docId>      parent doc for the new graph                           [active space root]',
+    '  user-agent=<str>    Api-User-Agent header (REQUIRED by Wikimedia etiquette)',
+    '  rate=<rps>          max wikipedia requests per second                      [3]',
+    '  --include-categories   expand each article\'s categories into nested graphs',
+    '  --dry-run              fetch only the entry article, print outline, no writes',
+  ].join('\n'),
+  async run(_conn: CLIConnection | null, args: ParsedArgs): Promise<string> {
+    const opts = parseOptions(args)
+    if (typeof opts === 'string') return opts
+
+    const log = (msg: string) => {
+      if (!args.flags.has('quiet') && !args.flags.has('q')) {
+        console.error(`[wiki] ${msg}`)
+      }
+    }
+
+    const wp = new WikipediaClient({
+      lang: opts.lang,
+      domain: opts.domain,
+      userAgent: opts.userAgent,
+      rate: opts.rate,
+    })
+
+    if (opts.dryRun) {
+      // Dry-run: fetch only the entry, print its outline, no server.
+      log(`fetch ${opts.title}`)
+      const doc = await wp.fetchArticle(opts.title)
+      if (!doc) return `Article not found: "${opts.title}"`
+      const snap = snapshotArticle(doc, canonicalTitle(doc.title?.() ?? opts.title))
+      return [
+        `Entry: ${snap.title}`,
+        `URL: ${snap.url ?? '(none)'}`,
+        `Internal links: ${snap.linkTitles.length}`,
+        `Categories: ${snap.categories.length}`,
+        `Sections: ${snap.sections.length}`,
+        `Has infobox: ${snap.infobox && snap.infobox.length > 0 ? 'yes' : 'no'}`,
+        '',
+        '── Sections ──',
+        printSections(snap.sections, ''),
+      ].join('\n')
+    }
+
+    // ── Connect ──────────────────────────────────────────────────────────
+    // process.env access uses bracket notation to satisfy noUncheckedIndexedAccess.
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const env = (globalThis as any).process?.env ?? {}
+    const url = env['ABRA_URL']
+    if (!url) {
+      return 'ABRA_URL is required to write to the server. Set it or pass --dry-run.'
+    }
+    const { dm } = await openSession({
+      url,
+      name: env['ABRA_NAME'],
+      color: env['ABRA_COLOR'],
+      inviteCode: env['ABRA_INVITE_CODE'],
+      keyFile: env['ABRA_KEY_FILE'],
+      quiet: args.flags.has('quiet') || args.flags.has('q'),
+    })
+
+    try {
+      const result = await runStreaming(dm, wp, opts, log)
+      return [
+        `Done. Created ${result.articleCount} articles${
+          result.categoryCount > 0 ? ` + ${result.categoryCount} categories` : ''
+        }.`,
+        `Root: ${result.rootDocId}`,
+      ].join('\n')
+    } finally {
+      await dm.destroy().catch(() => {})
+    }
+  },
+})
+
+// ─────────────────────────────────────────────────────────────────────────
+// Streaming orchestrator
+// ─────────────────────────────────────────────────────────────────────────
+
+interface StreamResult {
+  rootDocId: string
+  articleCount: number
+  categoryCount: number
+}
+
+async function runStreaming(
+  dm: DocumentManager,
+  wp: WikipediaClient,
+  opts: WikiOptions,
+  log: (msg: string) => void,
+): Promise<StreamResult> {
+  // Title → docId map. Drives [[Title]] → [[docId|label]] rewriting at write time.
+  const titleToDocId = new Map<string, string>()
+  // Snapshots of articles we've already fetched (avoid re-fetching).
+  const fetched = new Map<string, ExtractedArticle>()
+  // Articles whose section/infobox children have been created (split mode).
+  const childrenCreated = new Set<string>()
+  // Categories whose shells have been created.
+  const categoryToDocId = new Map<string, string>()
+  let categoriesContainerId: string | null = null
+
+  // ── Fetch entry first; we need its title to label the root graph ─────
+  log(`fetch ${opts.title}`)
+  const entryDoc = await wp.fetchArticle(opts.title)
+  if (!entryDoc) {
+    throw new Error(`Article not found: "${opts.title}"`)
+  }
+  const entryTitle = canonicalTitle(entryDoc.title?.() ?? opts.title)
+  const entrySnap = snapshotArticle(entryDoc, entryTitle)
+  fetched.set(entryTitle, entrySnap)
+
+  // ── Step 1: create root graph doc (visible immediately) ──────────────
+  const rootEntry = dm.tree.create({
+    parentId: opts.parentDocId ?? null,
+    label: entryTitle,
+    type: 'graph',
+    meta: { icon: ICONS.graph },
+  })
+  log(`+ ${rootEntry.id.slice(0, 8)}…  ${entryTitle} (graph)`)
+
+  // ── Step 2: create the entry article shell ───────────────────────────
+  const entryArticleId = createArticleShell(dm, entrySnap, rootEntry.id, log)
+  titleToDocId.set(entryTitle, entryArticleId)
+
+  // Queue of (title, depth) to process. Each entry is guaranteed to have
+  // a shell doc already in titleToDocId.
+  const queue: Array<{ title: string; depth: number }> = [{ title: entryTitle, depth: 0 }]
+  let articleCount = 0
+
+  // ── Step 3: streaming process ───────────────────────────────────────
+  while (queue.length > 0) {
+    const { title, depth } = queue.shift()!
+    const articleDocId = titleToDocId.get(title)!
+
+    // Ensure we've fetched this article.
+    let snap = fetched.get(title)
+    if (!snap) {
+      log(`fetch [d${depth}] ${title}`)
+      try {
+        const doc = await wp.fetchArticle(title)
+        if (!doc) {
+          log(`  not found — leaving stub`)
+          continue
+        }
+        snap = snapshotArticle(doc, canonicalTitle(doc.title?.() ?? title))
+        fetched.set(title, snap)
+      } catch (err: any) {
+        log(`! fetch failed: ${err?.message ?? err}`)
+        continue
+      }
+    }
+
+    // Create this article's section/infobox children (split mode only,
+    // and only once per article).
+    if (opts.mode === 'split' && !childrenCreated.has(title)) {
+      createArticleChildren(dm, snap, articleDocId, log)
+      childrenCreated.add(title)
+    }
+
+    // Discover new linked titles and pre-allocate shells immediately.
+    if (depth < opts.depth) {
+      for (const linkTitle of snap.linkTitles) {
+        if (titleToDocId.has(linkTitle)) continue
+        const shell = dm.tree.create({
+          parentId: rootEntry.id,
+          label: linkTitle,
+          type: 'doc',
+          meta: { icon: ICONS.article },
+        })
+        titleToDocId.set(linkTitle, shell.id)
+        queue.push({ title: linkTitle, depth: depth + 1 })
+        log(`+ ${shell.id.slice(0, 8)}…  ${linkTitle} (doc, shell)`)
+      }
+    }
+
+    // Pre-allocate category shells when first discovered.
+    if (opts.includeCategories && snap.categories.length > 0) {
+      if (!categoriesContainerId) {
+        const c = dm.tree.create({
+          parentId: rootEntry.id,
+          label: 'Categories',
+          type: 'graph',
+          meta: { icon: ICONS.categories },
+        })
+        categoriesContainerId = c.id
+        log(`+ ${c.id.slice(0, 8)}…  Categories (graph)`)
+      }
+      for (const catTitle of snap.categories) {
+        if (categoryToDocId.has(catTitle)) continue
+        const cat = dm.tree.create({
+          parentId: categoriesContainerId,
+          label: prettyCategoryLabel(catTitle),
+          type: 'graph',
+          meta: { icon: ICONS.category },
+        })
+        categoryToDocId.set(catTitle, cat.id)
+        log(`+ ${cat.id.slice(0, 8)}…  ${prettyCategoryLabel(catTitle)} (graph, cat)`)
+      }
+    }
+
+    // Write this article's body NOW (links resolve to whatever shells we
+    // have allocated so far — that's all of this article's links since we
+    // just allocated them above).
+    const body =
+      opts.mode === 'split' ? renderArticleLead(snap) : renderArticleSingleDoc(snap)
+    if (body.trim().length > 0) {
+      const rewritten = rewriteLinks(body, titleToDocId)
+      try {
+        await dm.content.write(articleDocId, rewritten)
+        log(`✓ body  ${title}`)
+      } catch (err: any) {
+        log(`! body write failed for ${title}: ${err?.message ?? err}`)
+      }
+    }
+
+    // In split mode, also write each section/infobox doc body.
+    if (opts.mode === 'split') {
+      await writeChildrenBodies(dm, snap, articleDocId, titleToDocId, log)
+    }
+
+    articleCount++
+  }
+
+  // ── Step 4: fill in category bodies ─────────────────────────────────
+  let categoryCount = 0
+  if (opts.includeCategories && categoryToDocId.size > 0) {
+    for (const [catTitle, catDocId] of categoryToDocId) {
+      log(`category ${catTitle}`)
+      try {
+        const members = await wp.fetchCategoryPages(
+          catTitle,
+          opts.categoryDepth > 0,
+          Math.max(0, opts.categoryDepth),
+        )
+        const memberArticles: string[] = []
+        const subcats: string[] = []
+        for (const m of members) {
+          if (m.type === 'subcat') subcats.push(prettyCategoryLabel(m.title))
+          else memberArticles.push(m.title)
+        }
+        const body = renderCategoryBody(memberArticles, subcats)
+        const rewritten = rewriteLinks(body, titleToDocId)
+        if (rewritten.trim().length > 0) {
+          await dm.content.write(catDocId, rewritten)
+          log(`✓ body  category ${catTitle}`)
+        }
+        categoryCount++
+      } catch (err: any) {
+        log(`! category ${catTitle}: ${err?.message ?? err}`)
+      }
+    }
+  }
+
+  return { rootDocId: rootEntry.id, articleCount, categoryCount }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Shell + body helpers
+// ─────────────────────────────────────────────────────────────────────────
+
+function createArticleShell(
+  dm: DocumentManager,
+  article: ExtractedArticle,
+  parentId: string,
+  log: (msg: string) => void,
+): string {
+  const meta: Record<string, unknown> = { icon: ICONS.article }
+  if (article.url) meta.url = article.url
+  const entry = dm.tree.create({
+    parentId,
+    label: article.title,
+    type: 'doc',
+    meta,
+  })
+  log(`+ ${entry.id.slice(0, 8)}…  ${article.title} (doc)`)
+  return entry.id
+}
+
+/**
+ * Create section + infobox child docs for a split-mode article. Returns nothing
+ * — children get bodies written later in writeChildrenBodies.
+ */
+function createArticleChildren(
+  dm: DocumentManager,
+  article: ExtractedArticle,
+  articleDocId: string,
+  log: (msg: string) => void,
+): void {
+  if (article.infobox && article.infobox.length > 0) {
+    const ib = dm.tree.create({
+      parentId: articleDocId,
+      label: 'Infobox',
+      type: 'outline',
+      meta: { icon: ICONS.infobox },
+    })
+    log(`  + ${ib.id.slice(0, 8)}…  Infobox (outline)`)
+    // We attach the docId to the article object so writeChildrenBodies
+    // can find it without a second tree query.
+    ;(article as any)._infoboxDocId = ib.id
+  }
+  for (const section of article.sections) {
+    createSectionShell(dm, section, articleDocId, log)
+  }
+}
+
+function createSectionShell(
+  dm: DocumentManager,
+  section: ExtractedSection,
+  parentDocId: string,
+  log: (msg: string) => void,
+): void {
+  const hasChildren = section.children.length > 0
+  if (!section.body.trim() && !hasChildren) return
+  const { type, icon } = pickSectionType(section)
+  const entry = dm.tree.create({
+    parentId: parentDocId,
+    label: section.title || 'Untitled section',
+    type,
+    meta: { icon },
+  })
+  log(`  + ${entry.id.slice(0, 8)}…  ${entry.label} (${type})`)
+  ;(section as any)._docId = entry.id
+  for (const child of section.children) {
+    createSectionShell(dm, child, entry.id, log)
+  }
+}
+
+async function writeChildrenBodies(
+  dm: DocumentManager,
+  article: ExtractedArticle,
+  _articleDocId: string,
+  titleToDocId: Map<string, string>,
+  log: (msg: string) => void,
+): Promise<void> {
+  const infoboxDocId = (article as any)._infoboxDocId as string | undefined
+  if (infoboxDocId && article.infobox && article.infobox.length > 0) {
+    try {
+      await dm.content.write(infoboxDocId, renderInfoboxBody(article.infobox))
+    } catch (err: any) {
+      log(`! infobox body write failed: ${err?.message ?? err}`)
+    }
+  }
+  for (const section of article.sections) {
+    await writeSectionBody(dm, section, titleToDocId, log)
+  }
+}
+
+async function writeSectionBody(
+  dm: DocumentManager,
+  section: ExtractedSection,
+  titleToDocId: Map<string, string>,
+  log: (msg: string) => void,
+): Promise<void> {
+  const docId = (section as any)._docId as string | undefined
+  if (docId && section.body.trim().length > 0) {
+    try {
+      await dm.content.write(docId, rewriteLinks(section.body, titleToDocId))
+    } catch (err: any) {
+      log(`! section body write failed for ${section.title}: ${err?.message ?? err}`)
+    }
+  }
+  for (const child of section.children) {
+    await writeSectionBody(dm, child, titleToDocId, log)
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Argument parsing + dry-run printing
+// ─────────────────────────────────────────────────────────────────────────
+
+function parseOptions(args: ParsedArgs): WikiOptions | string {
+  const title = args.positional[0]?.trim() || args.params['title']
+  if (!title) return 'Missing required positional argument: <title>. Example: abracadabra wiki "Toronto Raptors"'
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const env = (globalThis as any).process?.env ?? {}
+  const userAgent = args.params['user-agent'] || args.params['userAgent'] || env['ABRA_WIKI_USER_AGENT']
+  if (!userAgent) {
+    return [
+      'Missing required parameter: user-agent="your-name (you@example.com)"',
+      '(Wikimedia etiquette requires an Api-User-Agent header. Pass user-agent=... or set ABRA_WIKI_USER_AGENT.)',
+    ].join('\n')
+  }
+
+  const mode = (args.params['mode'] ?? 'split') as ExtractMode
+  if (mode !== 'single' && mode !== 'split') {
+    return `Invalid mode "${mode}". Use mode=single or mode=split.`
+  }
+
+  const depth = parseIntOr(args.params['depth'], 1)
+  const categoryDepth = parseIntOr(args.params['category-depth'] ?? args.params['categoryDepth'], 1)
+  const rate = parseFloatOr(args.params['rate'], 3)
+
+  return {
+    title,
+    mode,
+    depth,
+    categoryDepth,
+    includeCategories: args.flags.has('include-categories') || args.flags.has('includeCategories'),
+    lang: args.params['lang'] ?? 'en',
+    domain: args.params['domain'],
+    parentDocId: args.params['parent'],
+    userAgent,
+    rate,
+    dryRun: args.flags.has('dry-run') || args.flags.has('dryRun'),
+  }
+}
+
+function parseIntOr(s: string | undefined, fallback: number): number {
+  if (!s) return fallback
+  const n = Number.parseInt(s, 10)
+  return Number.isFinite(n) && n >= 0 ? n : fallback
+}
+
+function parseFloatOr(s: string | undefined, fallback: number): number {
+  if (!s) return fallback
+  const n = Number.parseFloat(s)
+  return Number.isFinite(n) && n > 0 ? n : fallback
+}
+
+function printSections(sections: ExtractedSection[], indent: string): string {
+  const lines: string[] = []
+  for (const s of sections) {
+    const hint = s.body ? ` (${s.body.length}b)` : ''
+    lines.push(`${indent}- ${s.title}${hint}${s.children.length > 0 ? ` [${s.children.length} sub]` : ''}`)
+    if (s.children.length > 0) {
+      lines.push(printSections(s.children, indent + '  '))
+    }
+  }
+  return lines.join('\n')
+}

package/src/commands/wiki/render.ts

@@ -0,0 +1,91 @@
+/**
+ * Body rendering + page-type decisions for the streaming orchestrator.
+ *
+ * All rendering is title-driven: bodies are rendered with `[[Title]]`
+ * placeholders, and `rewriteLinks` rewrites them to `[[docId|label]]`
+ * using the live title→docId map at write time.
+ */
+import type { ExtractedArticle, ExtractedSection } from './types.ts'
+
+export const ICONS = {
+  graph: 'git-fork',
+  article: 'book-open',
+  category: 'tag',
+  infobox: 'info',
+  outline: 'list',
+  gallery: 'images',
+  section: 'pilcrow',
+  categories: 'tags',
+} as const
+
+/** Decide a page type for a section based on its shape. */
+export function pickSectionType(section: ExtractedSection): { type: string; icon: string } {
+  if (section.children.length > 0) return { type: 'outline', icon: ICONS.outline }
+  if (section.isList && section.listLength >= 5) return { type: 'outline', icon: ICONS.outline }
+  return { type: 'doc', icon: ICONS.section }
+}
+
+/** Render the lead paragraph as the article-doc body. */
+export function renderArticleLead(article: ExtractedArticle): string {
+  return article.lead ?? ''
+}
+
+/** Render the article as a single doc, sections + infobox inlined. */
+export function renderArticleSingleDoc(article: ExtractedArticle): string {
+  const parts: string[] = []
+  if (article.lead) parts.push(article.lead)
+  if (article.infobox && article.infobox.length > 0) {
+    parts.push('## Infobox', renderInfoboxBody(article.infobox))
+  }
+  for (const section of article.sections) {
+    parts.push(...renderSectionInline(section, 2))
+  }
+  return parts.join('\n\n')
+}
+
+function renderSectionInline(section: ExtractedSection, level: number): string[] {
+  const out: string[] = []
+  const prefix = '#'.repeat(Math.min(6, level))
+  if (section.title) out.push(`${prefix} ${section.title}`)
+  if (section.body.trim()) out.push(section.body)
+  for (const child of section.children) {
+    out.push(...renderSectionInline(child, level + 1))
+  }
+  return out
+}
+
+export function renderInfoboxBody(rows: Array<{ key: string; value: string }>): string {
+  return rows.map((r) => `- **${r.key}:** ${r.value}`).join('\n')
+}
+
+export function renderCategoryBody(members: string[], subcategories: string[]): string {
+  const parts: string[] = []
+  if (members.length > 0) {
+    parts.push('## Pages')
+    parts.push(members.map((m) => `- [[${m}]]`).join('\n'))
+  }
+  if (subcategories.length > 0) {
+    parts.push('## Sub-categories')
+    parts.push(subcategories.map((s) => `- ${s}`).join('\n'))
+  }
+  return parts.join('\n\n')
+}
+
+/**
+ * Replace `[[Title]]` / `[[Title|Alias]]` in markdown with
+ * `[[docId|label]]` using the title→docId map. Unresolved titles fall
+ * back to plain text (their alias or original title).
+ */
+export function rewriteLinks(
+  markdown: string,
+  titleToDocId: Map<string, string>,
+): string {
+  const re = /\[\[([^\]|]+?)(?:\|([^\]]+?))?\]\]/g
+  return markdown.replace(re, (_match, target: string, alias?: string) => {
+    const title = target.trim()
+    const docId = titleToDocId.get(title)
+    const display = (alias && alias.trim().length > 0 ? alias : title).trim()
+    if (!docId) return display
+    return `[[${docId}|${display}]]`
+  })
+}