@abraca/cli 1.9.1 → 2.3.0

package/src/commands/wiki/snapshot.ts

@@ -0,0 +1,210 @@
+/**
+ * Snapshot a wtf_wikipedia Document into a plain-data shape that's easy to
+ * work with downstream. No BFS / no plan-building here — just a pure read
+ * of one parsed page.
+ */
+import type { ExtractedArticle, ExtractedSection } from './types.ts'
+import { canonicalTitle, isCategoryTitle, stripCategoryPrefix } from './wikipedia.ts'
+
+export { canonicalTitle, isCategoryTitle, stripCategoryPrefix }
+
+export function snapshotArticle(doc: any, title: string): ExtractedArticle {
+  return {
+    title,
+    linkTitles: collectLinkTitles(doc),
+    categories: collectCategories(doc),
+    sections: snapshotSections(doc.sections?.() ?? []),
+    infobox: snapshotInfobox(doc.infobox?.()),
+    lead: leadParagraph(doc),
+    url: typeof doc.url === 'function' ? doc.url() : null,
+  }
+}
+
+export function prettyCategoryLabel(catTitle: string): string {
+  return stripCategoryPrefix(catTitle)
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Link / category extraction
+// ─────────────────────────────────────────────────────────────────────────
+
+function collectLinkTitles(doc: any): string[] {
+  const links = doc.links?.() ?? []
+  const out = new Set<string>()
+  for (const l of links) {
+    if (!l) continue
+    const page = typeof l.page === 'function' ? l.page() : null
+    if (typeof page !== 'string' || page.length === 0) continue
+    if (isCategoryTitle(page)) continue
+    out.add(canonicalTitle(page))
+  }
+  return [...out]
+}
+
+function collectCategories(doc: any): string[] {
+  const out: string[] = []
+  for (const c of (doc.categories?.() as string[] | undefined) ?? []) {
+    const norm = canonicalTitle(c)
+    if (norm) out.push(norm)
+  }
+  return out
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Sections — flatten wtf's parent-child references into a real tree
+// ─────────────────────────────────────────────────────────────────────────
+
+function snapshotSections(rawSections: any[]): ExtractedSection[] {
+  const all = rawSections.map((s) => ({
+    raw: s,
+    title: s.title?.() || '',
+    parentRef: typeof s.parent === 'function' ? s.parent() : null,
+    children: [] as ExtractedSection[],
+  }))
+
+  const byRaw = new Map<any, (typeof all)[number]>()
+  for (const s of all) byRaw.set(s.raw, s)
+
+  const roots: (typeof all)[number][] = []
+  for (const s of all) {
+    if (s.parentRef && byRaw.has(s.parentRef)) {
+      byRaw.get(s.parentRef)!.children.push(materialize(s))
+    } else {
+      roots.push(s)
+    }
+  }
+  return roots.map(materialize)
+}
+
+function materialize(node: {
+  raw: any
+  title: string
+  children: ExtractedSection[]
+}): ExtractedSection {
+  const lists = node.raw.lists?.() ?? []
+  const paragraphs = node.raw.paragraphs?.() ?? []
+
+  let listLength = 0
+  for (const l of lists) {
+    const lines = l.lines?.() ?? []
+    listLength += lines.length
+  }
+  const isList =
+    lists.length > 0 && (paragraphs.length === 0 || listLength >= paragraphs.length * 2)
+
+  const bodyParts: string[] = []
+  for (const p of paragraphs) {
+    const md = paragraphMarkdown(p)
+    if (md) bodyParts.push(md)
+  }
+  for (const l of lists) {
+    const lines = (l.lines?.() ?? []) as any[]
+    for (const line of lines) {
+      const text = lineText(line)
+      if (text) bodyParts.push(`- ${text}`)
+    }
+  }
+
+  return {
+    title: node.title,
+    body: bodyParts.join('\n\n'),
+    isList,
+    listLength,
+    children: node.children,
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Infobox
+// ─────────────────────────────────────────────────────────────────────────
+
+function snapshotInfobox(box: any | null | undefined): Array<{ key: string; value: string }> | undefined {
+  if (!box) return undefined
+  const data = typeof box.json === 'function' ? box.json() : null
+  if (!data || typeof data !== 'object') return undefined
+  const rows: Array<{ key: string; value: string }> = []
+  for (const [key, val] of Object.entries(data)) {
+    const value = stringifyInfoboxValue(val)
+    if (!value) continue
+    rows.push({ key: humanKey(key), value })
+  }
+  return rows.length > 0 ? rows : undefined
+}
+
+function stringifyInfoboxValue(val: unknown): string {
+  if (val == null) return ''
+  if (typeof val === 'string') return val
+  if (typeof val === 'number' || typeof val === 'boolean') return String(val)
+  if (Array.isArray(val)) {
+    return val.map(stringifyInfoboxValue).filter(Boolean).join(', ')
+  }
+  if (typeof val === 'object') {
+    const o = val as Record<string, unknown>
+    if (typeof o.text === 'string') return o.text
+    if (typeof o.number === 'number') return String(o.number)
+  }
+  return ''
+}
+
+function humanKey(k: string): string {
+  return k.replace(/_/g, ' ').replace(/^./, (m) => m.toUpperCase())
+}
+
+// ─────────────────────────────────────────────────────────────────────────
+// Markdown rendering
+// ─────────────────────────────────────────────────────────────────────────
+
+function leadParagraph(doc: any): string {
+  const paras = doc.paragraphs?.() ?? []
+  const first = paras[0]
+  if (!first) return ''
+  return paragraphMarkdown(first)
+}
+
+/**
+ * Render a paragraph as markdown, replacing internal links with `[[Title]]`.
+ * The streaming orchestrator's link rewriter later swaps `[[Title]]` →
+ * `[[docId|label]]` once IDs are known.
+ */
+function paragraphMarkdown(paragraph: any): string {
+  const sentences = paragraph.sentences?.() ?? []
+  const out: string[] = []
+  for (const s of sentences) {
+    out.push(sentenceWithWikilinks(s))
+  }
+  return out.join(' ').trim()
+}
+
+function sentenceWithWikilinks(sentence: any): string {
+  const text: string = (sentence.text?.() ?? '').toString()
+  const links = sentence.links?.() ?? []
+  if (links.length === 0) return text
+
+  let result = text
+  const replacements = links
+    .map((l: any) => {
+      const page = typeof l.page === 'function' ? l.page() : null
+      const display = typeof l.text === 'function' ? l.text() : null
+      if (typeof page !== 'string' || page.length === 0) return null
+      if (isCategoryTitle(page)) return null
+      const shown = (display && display.length > 0 ? display : page) as string
+      return { page: canonicalTitle(page), shown }
+    })
+    .filter((x: any): x is { page: string; shown: string } => x !== null)
+    .sort((a: any, b: any) => b.shown.length - a.shown.length)
+
+  for (const { page, shown } of replacements) {
+    if (!result.includes(shown)) continue
+    const replacement = shown === page ? `[[${page}]]` : `[[${page}|${shown}]]`
+    result = result.replace(shown, replacement)
+  }
+  return result
+}
+
+function lineText(line: any): string {
+  if (!line) return ''
+  if (typeof line === 'string') return line
+  if (typeof line.text === 'string') return line.text
+  if (typeof line.text === 'function') return line.text()
+  return ''
+}

package/src/commands/wiki/types.ts

@@ -0,0 +1,45 @@
+/**
+ * Internal types for the Wikipedia extractor command.
+ */
+
+export type ExtractMode = 'single' | 'split'
+
+export interface WikiOptions {
+  title: string
+  mode: ExtractMode
+  depth: number
+  includeCategories: boolean
+  categoryDepth: number
+  lang: string
+  domain?: string
+  parentDocId?: string
+  userAgent: string
+  rate: number
+  /** Print plan-tree to stderr without writing (no server connection). */
+  dryRun: boolean
+}
+
+/** A simplified article snapshot extracted from a wtf_wikipedia Document. */
+export interface ExtractedArticle {
+  title: string
+  /** Wikipedia internal links (page titles) referenced by this article. */
+  linkTitles: string[]
+  /** Categories this article belongs to. */
+  categories: string[]
+  /** Top-level sections; each section has nested subsections. */
+  sections: ExtractedSection[]
+  /** Infobox key-value rows (first infobox only). */
+  infobox?: Array<{ key: string; value: string }>
+  /** Plaintext lead paragraph(s) for the single-doc body. */
+  lead: string
+  /** Source URL on Wikipedia. */
+  url: string | null
+}
+
+export interface ExtractedSection {
+  title: string
+  body: string
+  isList: boolean
+  listLength: number
+  children: ExtractedSection[]
+}

package/src/commands/wiki/wikipedia.ts

@@ -0,0 +1,154 @@
+/**
+ * Rate-limited wrapper around wtf_wikipedia + wtf-plugin-api.
+ *
+ * Responsibilities:
+ *   - Throttle requests to respect Wikimedia API etiquette
+ *   - Cache parsed Documents by canonical title
+ *   - Resolve redirects so callers always see the redirect target
+ *   - Expose getCategoryPages via wtf-plugin-api
+ */
+// @ts-ignore — wtf_wikipedia ships its own types but they are imprecise; we
+// cast Link/Section APIs as needed below.
+import wtf from 'wtf_wikipedia'
+// @ts-ignore — wtf-plugin-api is JS-only with no types; treat as opaque.
+import wtfApiPlugin from 'wtf-plugin-api'
+
+// Augment wtf at module load so getCategoryPages becomes available.
+let pluginExtended = false
+function ensurePlugin(): void {
+  if (pluginExtended) return
+  // @ts-ignore — extend is dynamically attached to the wtf default export.
+  wtf.extend(wtfApiPlugin)
+  pluginExtended = true
+}
+
+export interface WikipediaClientConfig {
+  lang: string
+  domain?: string
+  userAgent: string
+  /** Max requests per second. */
+  rate: number
+}
+
+interface FetchOpts {
+  lang?: string
+  domain?: string
+  'Api-User-Agent'?: string
+  follow_redirects?: boolean
+}
+
+/** A token-bucket-ish throttle: at most `rate` calls per second, FIFO. */
+class RateLimiter {
+  private lastTickMs = 0
+  constructor(private intervalMs: number) {}
+
+  async wait(): Promise<void> {
+    const now = Date.now()
+    const earliest = this.lastTickMs + this.intervalMs
+    if (now < earliest) {
+      await new Promise((r) => setTimeout(r, earliest - now))
+    }
+    this.lastTickMs = Math.max(now, earliest)
+  }
+}
+
+export class WikipediaClient {
+  private cache = new Map<string, any>()
+  private redirects = new Map<string, string>()
+  private limiter: RateLimiter
+  private fetchOpts: FetchOpts
+
+  constructor(private config: WikipediaClientConfig) {
+    ensurePlugin()
+    this.limiter = new RateLimiter(Math.max(50, Math.floor(1000 / Math.max(0.1, config.rate))))
+    this.fetchOpts = {
+      lang: config.lang,
+      'Api-User-Agent': config.userAgent,
+      follow_redirects: true,
+    }
+    if (config.domain) this.fetchOpts.domain = config.domain
+  }
+
+  /**
+   * Fetch and parse a Wikipedia article.
+   * - Returns the cached Document if we've seen this title before.
+   * - Follows redirects and caches under both source and target titles.
+   * - Returns null when the page does not exist.
+   */
+  async fetchArticle(rawTitle: string): Promise<any | null> {
+    const title = canonicalTitle(rawTitle)
+    if (this.cache.has(title)) return this.cache.get(title)
+    if (this.redirects.has(title)) {
+      const target = this.redirects.get(title)!
+      return this.cache.get(target) ?? null
+    }
+
+    await this.limiter.wait()
+    let doc: any
+    try {
+      doc = await (wtf as any).fetch(title, this.fetchOpts)
+    } catch (err: any) {
+      throw new Error(`Wikipedia fetch failed for "${title}": ${err?.message ?? err}`)
+    }
+    if (!doc) return null
+
+    // wtf usually follows redirects automatically when follow_redirects=true,
+    // but defensively handle the case where it surfaces a redirect doc.
+    if (typeof doc.isRedirect === 'function' && doc.isRedirect()) {
+      const target = doc.redirectTo?.()?.page
+      if (typeof target === 'string') {
+        this.redirects.set(title, canonicalTitle(target))
+        const inner = await this.fetchArticle(target)
+        return inner
+      }
+    }
+
+    const resolvedTitle = canonicalTitle(doc.title?.() ?? title)
+    this.cache.set(resolvedTitle, doc)
+    if (resolvedTitle !== title) this.redirects.set(title, resolvedTitle)
+    return doc
+  }
+
+  /**
+   * Fetch the member pages of a category (and optionally sub-categories).
+   * @param category Category title (with or without "Category:" prefix).
+   * @param recursive Whether to traverse sub-categories.
+   * @param maxDepth Recursion depth when recursive=true.
+   */
+  async fetchCategoryPages(
+    category: string,
+    recursive: boolean,
+    maxDepth: number,
+  ): Promise<Array<{ title: string; type: 'page' | 'subcat' }>> {
+    await this.limiter.wait()
+    const opts: Record<string, unknown> = {
+      lang: this.config.lang,
+      'Api-User-Agent': this.config.userAgent,
+      recursive,
+      maxDepth,
+    }
+    if (this.config.domain) opts.domain = this.config.domain
+    // @ts-ignore — getCategoryPages is attached at runtime via wtf.extend.
+    const list: any[] = await wtf.getCategoryPages(category, opts)
+    return (list ?? []).map((m) => ({
+      title: canonicalTitle(m.title),
+      type: m.type === 'subcat' ? 'subcat' : 'page',
+    }))
+  }
+}
+
+/** Normalize a Wikipedia title — trim, collapse spaces, strip leading/trailing colons. */
+export function canonicalTitle(s: string): string {
+  return (s ?? '').toString().replace(/_/g, ' ').replace(/\s+/g, ' ').trim()
+}
+
+/** Detect a category-namespaced title. */
+const CATEGORY_PREFIX = /^(Category|Catégorie|Kategorie|Categoría|Categoria|Categorie|Kategoria):/i
+export function isCategoryTitle(title: string): boolean {
+  return CATEGORY_PREFIX.test(title)
+}
+
+/** Strip the "Category:" prefix for display. */
+export function stripCategoryPrefix(title: string): string {
+  return title.replace(CATEGORY_PREFIX, '').trim()
+}

package/src/connection.ts

@@ -5,8 +5,8 @@
  * Reuses the same patterns as AbracadabraMCPServer but without MCP SDK dependency.
  */
 import * as Y from 'yjs'
-import { AbracadabraProvider, AbracadabraClient } from '@abraca/dabra'
-import type { ServerInfo, DocumentMeta, SpaceMeta } from '@abraca/dabra'
+import { AbracadabraProvider, AbracadabraClient, Kind } from '@abraca/dabra'
+import type { ServerInfo, DocumentMeta } from '@abraca/dabra'
 import { loadOrCreateKeypair, signChallenge } from './crypto.ts'
 
 export interface CLIConnectionConfig {
@@ -43,33 +43,13 @@ function waitForSync(
   })
 }
 
-/** Map a DocumentMeta to SpaceMeta shape for display compatibility. */
-function docToSpaceMeta(doc: DocumentMeta): SpaceMeta {
-  const publicAccess = doc.public_access
-  let visibility: SpaceMeta['visibility'] = 'private'
-  if (publicAccess && publicAccess !== 'none') visibility = 'public'
-
-  return {
-    id: doc.id,
-    doc_id: doc.id,
-    name: doc.label ?? doc.id,
-    description: doc.description ?? null,
-    visibility,
-    is_hub: doc.is_hub ?? false,
-    owner_id: doc.owner_id ?? null,
-    created_at: 0,
-    updated_at: doc.updated_at ?? 0,
-    public_access: publicAccess ?? null,
-  }
-}
-
 export class CLIConnection {
   readonly config: CLIConnectionConfig
   readonly client: AbracadabraClient
 
   private _serverInfo: ServerInfo | null = null
   private _rootDocId: string | null = null
-  private _spaces: SpaceMeta[] = []
+  private _spaces: DocumentMeta[] = []
   private _rootDoc: Y.Doc | null = null
   private _rootProvider: AbracadabraProvider | null = null
   private _userId: string | null = null
@@ -100,7 +80,7 @@ export class CLIConnection {
     return this._rootDocId
   }
 
-  get spaces(): SpaceMeta[] {
+  get spaces(): DocumentMeta[] {
     return this._spaces
   }
 
@@ -154,35 +134,20 @@ export class CLIConnection {
     // Step 3: Discover server info
     this._serverInfo = await this.client.serverInfo()
 
-    // Step 4: Discover root documents
-    let initialDocId: string | null = this._serverInfo.index_doc_id ?? null
-    try {
-      const roots = await this.client.listRootDocuments()
-      this._spaces = roots.map(docToSpaceMeta)
-      const hub = roots.find((d: any) => d.is_hub)
-      if (hub) {
-        initialDocId = hub.id
-        this.log(`Hub document: ${hub.label ?? hub.id} (${hub.id})`)
-      } else if (roots.length > 0) {
-        initialDocId = roots[0].id
-        this.log(`No hub, using first root doc: ${roots[0].label ?? roots[0].id}`)
-      }
-    } catch {
-      try {
-        this._spaces = await this.client.listSpaces()
-        const hub = this._spaces.find(s => s.is_hub)
-        if (hub) {
-          initialDocId = hub.doc_id
-        } else if (this._spaces.length > 0) {
-          initialDocId = this._spaces[0].doc_id
-        }
-      } catch {
-        this.log('Neither /docs?root=true nor /spaces available, using index_doc_id')
-      }
+    // Step 4: Pick an entry-point doc — first Space under the server root,
+    // falling back to the first top-level doc of any kind.
+    const roots = await this.client.listChildren()
+    this._spaces = roots.filter(d => d.kind === Kind.Space)
+    const first = this._spaces[0] ?? roots[0]
+    const initialDocId = first?.id ?? null
+    if (first) {
+      this.log(`Entry document: ${first.label ?? first.id} (${first.id})`)
     }
 
     if (!initialDocId) {
-      throw new Error('No entry point found: server has neither spaces nor index_doc_id configured.')
+      throw new Error(
+        'No entry point found: server has no top-level documents. Create a Space first.',
+      )
     }
 
     this._rootDocId = initialDocId

package/src/crypto.ts

@@ -3,11 +3,12 @@
  * Mirrors @abraca/mcp/src/crypto.ts — standalone to avoid MCP SDK dependency.
  */
 import * as ed from '@noble/ed25519'
-import { sha512 } from '@noble/hashes/sha2'
+import { sha512 } from '@noble/hashes/sha2.js'
 import { readFile, writeFile, mkdir } from 'node:fs/promises'
 
-// @noble/ed25519 v2+ requires explicit hash configuration
-ed.etc.sha512Sync = (...msgs: Uint8Array[]) => sha512(ed.etc.concatBytes(...msgs))
+// @noble/ed25519 v3 hash hook
+ed.hashes.sha512 = sha512
+ed.hashes.sha512Async = (m: Uint8Array) => Promise.resolve(sha512(m))
 import { existsSync } from 'node:fs'
 import { homedir } from 'node:os'
 import { join, dirname } from 'node:path'
@@ -45,7 +46,7 @@ export async function loadOrCreateKeypair(keyPath?: string): Promise<CLIKeypair>
   }
 
   // Generate new keypair
-  const privateKey = ed.utils.randomPrivateKey()
+  const privateKey = ed.utils.randomSecretKey()
   const publicKey = ed.getPublicKey(privateKey)
 
   // Ensure directory exists and write seed with restricted permissions

package/src/index.ts

@@ -28,9 +28,18 @@ import './commands/awareness.ts'
 import './commands/files.ts'
 import './commands/permissions.ts'
 import './commands/page-types.ts'
+import './commands/wiki/index.ts'
 
 // ── Commands that don't require a connection ─────────────────────────────────
-const NO_CONNECT_COMMANDS = new Set(['help', 'h', '?', 'version', 'v', 'page-types', 'types', 'doctypes'])
+// "wiki" opens its own DocumentManager session via wiki/connect.ts, so the
+// parent harness should NOT pre-open a CLIConnection (which would authenticate
+// twice and hold an unused root provider).
+const NO_CONNECT_COMMANDS = new Set([
+  'help', 'h', '?',
+  'version', 'v',
+  'page-types', 'types', 'doctypes',
+  'wiki', 'wikipedia',
+])
 
 async function main() {
   const args = parseArgs(process.argv)