@abraca/cli 2.25.0 → 2.27.0

package/src/commands/wiki/snapshot.ts

@@ -1,210 +0,0 @@
-/**
- * Snapshot a wtf_wikipedia Document into a plain-data shape that's easy to
- * work with downstream. No BFS / no plan-building here — just a pure read
- * of one parsed page.
- */
-import type { ExtractedArticle, ExtractedSection } from './types.ts'
-import { canonicalTitle, isCategoryTitle, stripCategoryPrefix } from './wikipedia.ts'
-
-export { canonicalTitle, isCategoryTitle, stripCategoryPrefix }
-
-export function snapshotArticle(doc: any, title: string): ExtractedArticle {
-  return {
-    title,
-    linkTitles: collectLinkTitles(doc),
-    categories: collectCategories(doc),
-    sections: snapshotSections(doc.sections?.() ?? []),
-    infobox: snapshotInfobox(doc.infobox?.()),
-    lead: leadParagraph(doc),
-    url: typeof doc.url === 'function' ? doc.url() : null,
-  }
-}
-
-export function prettyCategoryLabel(catTitle: string): string {
-  return stripCategoryPrefix(catTitle)
-}
-
-// ─────────────────────────────────────────────────────────────────────────
-// Link / category extraction
-// ─────────────────────────────────────────────────────────────────────────
-
-function collectLinkTitles(doc: any): string[] {
-  const links = doc.links?.() ?? []
-  const out = new Set<string>()
-  for (const l of links) {
-    if (!l) continue
-    const page = typeof l.page === 'function' ? l.page() : null
-    if (typeof page !== 'string' || page.length === 0) continue
-    if (isCategoryTitle(page)) continue
-    out.add(canonicalTitle(page))
-  }
-  return [...out]
-}
-
-function collectCategories(doc: any): string[] {
-  const out: string[] = []
-  for (const c of (doc.categories?.() as string[] | undefined) ?? []) {
-    const norm = canonicalTitle(c)
-    if (norm) out.push(norm)
-  }
-  return out
-}
-
-// ─────────────────────────────────────────────────────────────────────────
-// Sections — flatten wtf's parent-child references into a real tree
-// ─────────────────────────────────────────────────────────────────────────
-
-function snapshotSections(rawSections: any[]): ExtractedSection[] {
-  const all = rawSections.map((s) => ({
-    raw: s,
-    title: s.title?.() || '',
-    parentRef: typeof s.parent === 'function' ? s.parent() : null,
-    children: [] as ExtractedSection[],
-  }))
-
-  const byRaw = new Map<any, (typeof all)[number]>()
-  for (const s of all) byRaw.set(s.raw, s)
-
-  const roots: (typeof all)[number][] = []
-  for (const s of all) {
-    if (s.parentRef && byRaw.has(s.parentRef)) {
-      byRaw.get(s.parentRef)!.children.push(materialize(s))
-    } else {
-      roots.push(s)
-    }
-  }
-  return roots.map(materialize)
-}
-
-function materialize(node: {
-  raw: any
-  title: string
-  children: ExtractedSection[]
-}): ExtractedSection {
-  const lists = node.raw.lists?.() ?? []
-  const paragraphs = node.raw.paragraphs?.() ?? []
-
-  let listLength = 0
-  for (const l of lists) {
-    const lines = l.lines?.() ?? []
-    listLength += lines.length
-  }
-  const isList =
-    lists.length > 0 && (paragraphs.length === 0 || listLength >= paragraphs.length * 2)
-
-  const bodyParts: string[] = []
-  for (const p of paragraphs) {
-    const md = paragraphMarkdown(p)
-    if (md) bodyParts.push(md)
-  }
-  for (const l of lists) {
-    const lines = (l.lines?.() ?? []) as any[]
-    for (const line of lines) {
-      const text = lineText(line)
-      if (text) bodyParts.push(`- ${text}`)
-    }
-  }
-
-  return {
-    title: node.title,
-    body: bodyParts.join('\n\n'),
-    isList,
-    listLength,
-    children: node.children,
-  }
-}
-
-// ─────────────────────────────────────────────────────────────────────────
-// Infobox
-// ─────────────────────────────────────────────────────────────────────────
-
-function snapshotInfobox(box: any | null | undefined): Array<{ key: string; value: string }> | undefined {
-  if (!box) return undefined
-  const data = typeof box.json === 'function' ? box.json() : null
-  if (!data || typeof data !== 'object') return undefined
-  const rows: Array<{ key: string; value: string }> = []
-  for (const [key, val] of Object.entries(data)) {
-    const value = stringifyInfoboxValue(val)
-    if (!value) continue
-    rows.push({ key: humanKey(key), value })
-  }
-  return rows.length > 0 ? rows : undefined
-}
-
-function stringifyInfoboxValue(val: unknown): string {
-  if (val == null) return ''
-  if (typeof val === 'string') return val
-  if (typeof val === 'number' || typeof val === 'boolean') return String(val)
-  if (Array.isArray(val)) {
-    return val.map(stringifyInfoboxValue).filter(Boolean).join(', ')
-  }
-  if (typeof val === 'object') {
-    const o = val as Record<string, unknown>
-    if (typeof o.text === 'string') return o.text
-    if (typeof o.number === 'number') return String(o.number)
-  }
-  return ''
-}
-
-function humanKey(k: string): string {
-  return k.replace(/_/g, ' ').replace(/^./, (m) => m.toUpperCase())
-}
-
-// ─────────────────────────────────────────────────────────────────────────
-// Markdown rendering
-// ─────────────────────────────────────────────────────────────────────────
-
-function leadParagraph(doc: any): string {
-  const paras = doc.paragraphs?.() ?? []
-  const first = paras[0]
-  if (!first) return ''
-  return paragraphMarkdown(first)
-}
-
-/**
- * Render a paragraph as markdown, replacing internal links with `[[Title]]`.
- * The streaming orchestrator's link rewriter later swaps `[[Title]]` →
- * `[[docId|label]]` once IDs are known.
- */
-function paragraphMarkdown(paragraph: any): string {
-  const sentences = paragraph.sentences?.() ?? []
-  const out: string[] = []
-  for (const s of sentences) {
-    out.push(sentenceWithWikilinks(s))
-  }
-  return out.join(' ').trim()
-}
-
-function sentenceWithWikilinks(sentence: any): string {
-  const text: string = (sentence.text?.() ?? '').toString()
-  const links = sentence.links?.() ?? []
-  if (links.length === 0) return text
-
-  let result = text
-  const replacements = links
-    .map((l: any) => {
-      const page = typeof l.page === 'function' ? l.page() : null
-      const display = typeof l.text === 'function' ? l.text() : null
-      if (typeof page !== 'string' || page.length === 0) return null
-      if (isCategoryTitle(page)) return null
-      const shown = (display && display.length > 0 ? display : page) as string
-      return { page: canonicalTitle(page), shown }
-    })
-    .filter((x: any): x is { page: string; shown: string } => x !== null)
-    .sort((a: any, b: any) => b.shown.length - a.shown.length)
-
-  for (const { page, shown } of replacements) {
-    if (!result.includes(shown)) continue
-    const replacement = shown === page ? `[[${page}]]` : `[[${page}|${shown}]]`
-    result = result.replace(shown, replacement)
-  }
-  return result
-}
-
-function lineText(line: any): string {
-  if (!line) return ''
-  if (typeof line === 'string') return line
-  if (typeof line.text === 'string') return line.text
-  if (typeof line.text === 'function') return line.text()
-  return ''
-}

package/src/commands/wiki/types.ts

@@ -1,45 +0,0 @@
-/**
- * Internal types for the Wikipedia extractor command.
- */
-
-export type ExtractMode = 'single' | 'split'
-
-export interface WikiOptions {
-  title: string
-  mode: ExtractMode
-  depth: number
-  includeCategories: boolean
-  categoryDepth: number
-  lang: string
-  domain?: string
-  parentDocId?: string
-  userAgent: string
-  rate: number
-  /** Print plan-tree to stderr without writing (no server connection). */
-  dryRun: boolean
-}
-
-/** A simplified article snapshot extracted from a wtf_wikipedia Document. */
-export interface ExtractedArticle {
-  title: string
-  /** Wikipedia internal links (page titles) referenced by this article. */
-  linkTitles: string[]
-  /** Categories this article belongs to. */
-  categories: string[]
-  /** Top-level sections; each section has nested subsections. */
-  sections: ExtractedSection[]
-  /** Infobox key-value rows (first infobox only). */
-  infobox?: Array<{ key: string; value: string }>
-  /** Plaintext lead paragraph(s) for the single-doc body. */
-  lead: string
-  /** Source URL on Wikipedia. */
-  url: string | null
-}
-
-export interface ExtractedSection {
-  title: string
-  body: string
-  isList: boolean
-  listLength: number
-  children: ExtractedSection[]
-}

package/src/commands/wiki/wikipedia.ts

@@ -1,154 +0,0 @@
-/**
- * Rate-limited wrapper around wtf_wikipedia + wtf-plugin-api.
- *
- * Responsibilities:
- *   - Throttle requests to respect Wikimedia API etiquette
- *   - Cache parsed Documents by canonical title
- *   - Resolve redirects so callers always see the redirect target
- *   - Expose getCategoryPages via wtf-plugin-api
- */
-// @ts-ignore — wtf_wikipedia ships its own types but they are imprecise; we
-// cast Link/Section APIs as needed below.
-import wtf from 'wtf_wikipedia'
-// @ts-ignore — wtf-plugin-api is JS-only with no types; treat as opaque.
-import wtfApiPlugin from 'wtf-plugin-api'
-
-// Augment wtf at module load so getCategoryPages becomes available.
-let pluginExtended = false
-function ensurePlugin(): void {
-  if (pluginExtended) return
-  // @ts-ignore — extend is dynamically attached to the wtf default export.
-  wtf.extend(wtfApiPlugin)
-  pluginExtended = true
-}
-
-export interface WikipediaClientConfig {
-  lang: string
-  domain?: string
-  userAgent: string
-  /** Max requests per second. */
-  rate: number
-}
-
-interface FetchOpts {
-  lang?: string
-  domain?: string
-  'Api-User-Agent'?: string
-  follow_redirects?: boolean
-}
-
-/** A token-bucket-ish throttle: at most `rate` calls per second, FIFO. */
-class RateLimiter {
-  private lastTickMs = 0
-  constructor(private intervalMs: number) {}
-
-  async wait(): Promise<void> {
-    const now = Date.now()
-    const earliest = this.lastTickMs + this.intervalMs
-    if (now < earliest) {
-      await new Promise((r) => setTimeout(r, earliest - now))
-    }
-    this.lastTickMs = Math.max(now, earliest)
-  }
-}
-
-export class WikipediaClient {
-  private cache = new Map<string, any>()
-  private redirects = new Map<string, string>()
-  private limiter: RateLimiter
-  private fetchOpts: FetchOpts
-
-  constructor(private config: WikipediaClientConfig) {
-    ensurePlugin()
-    this.limiter = new RateLimiter(Math.max(50, Math.floor(1000 / Math.max(0.1, config.rate))))
-    this.fetchOpts = {
-      lang: config.lang,
-      'Api-User-Agent': config.userAgent,
-      follow_redirects: true,
-    }
-    if (config.domain) this.fetchOpts.domain = config.domain
-  }
-
-  /**
-   * Fetch and parse a Wikipedia article.
-   * - Returns the cached Document if we've seen this title before.
-   * - Follows redirects and caches under both source and target titles.
-   * - Returns null when the page does not exist.
-   */
-  async fetchArticle(rawTitle: string): Promise<any | null> {
-    const title = canonicalTitle(rawTitle)
-    if (this.cache.has(title)) return this.cache.get(title)
-    if (this.redirects.has(title)) {
-      const target = this.redirects.get(title)!
-      return this.cache.get(target) ?? null
-    }
-
-    await this.limiter.wait()
-    let doc: any
-    try {
-      doc = await (wtf as any).fetch(title, this.fetchOpts)
-    } catch (err: any) {
-      throw new Error(`Wikipedia fetch failed for "${title}": ${err?.message ?? err}`)
-    }
-    if (!doc) return null
-
-    // wtf usually follows redirects automatically when follow_redirects=true,
-    // but defensively handle the case where it surfaces a redirect doc.
-    if (typeof doc.isRedirect === 'function' && doc.isRedirect()) {
-      const target = doc.redirectTo?.()?.page
-      if (typeof target === 'string') {
-        this.redirects.set(title, canonicalTitle(target))
-        const inner = await this.fetchArticle(target)
-        return inner
-      }
-    }
-
-    const resolvedTitle = canonicalTitle(doc.title?.() ?? title)
-    this.cache.set(resolvedTitle, doc)
-    if (resolvedTitle !== title) this.redirects.set(title, resolvedTitle)
-    return doc
-  }
-
-  /**
-   * Fetch the member pages of a category (and optionally sub-categories).
-   * @param category Category title (with or without "Category:" prefix).
-   * @param recursive Whether to traverse sub-categories.
-   * @param maxDepth Recursion depth when recursive=true.
-   */
-  async fetchCategoryPages(
-    category: string,
-    recursive: boolean,
-    maxDepth: number,
-  ): Promise<Array<{ title: string; type: 'page' | 'subcat' }>> {
-    await this.limiter.wait()
-    const opts: Record<string, unknown> = {
-      lang: this.config.lang,
-      'Api-User-Agent': this.config.userAgent,
-      recursive,
-      maxDepth,
-    }
-    if (this.config.domain) opts.domain = this.config.domain
-    // @ts-ignore — getCategoryPages is attached at runtime via wtf.extend.
-    const list: any[] = await wtf.getCategoryPages(category, opts)
-    return (list ?? []).map((m) => ({
-      title: canonicalTitle(m.title),
-      type: m.type === 'subcat' ? 'subcat' : 'page',
-    }))
-  }
-}
-
-/** Normalize a Wikipedia title — trim, collapse spaces, strip leading/trailing colons. */
-export function canonicalTitle(s: string): string {
-  return (s ?? '').toString().replace(/_/g, ' ').replace(/\s+/g, ' ').trim()
-}
-
-/** Detect a category-namespaced title. */
-const CATEGORY_PREFIX = /^(Category|Catégorie|Kategorie|Categoría|Categoria|Categorie|Kategoria):/i
-export function isCategoryTitle(title: string): boolean {
-  return CATEGORY_PREFIX.test(title)
-}
-
-/** Strip the "Category:" prefix for display. */
-export function stripCategoryPrefix(title: string): string {
-  return title.replace(CATEGORY_PREFIX, '').trim()
-}