@open330/kiwimu 0.8.0 → 1.1.0

package/src/ingest/pdf.ts

@@ -1,11 +1,13 @@
 export async function extractTextFromPdf(pdfPath: string): Promise<{ title: string; text: string }> {
-  let pdfParse: (buffer: Buffer) => Promise<{ info?: { Title?: string }; text: string }>;
+  let pdfParseModule: Record<string, unknown>;
   try {
-    pdfParse = require("pdf-parse");
+    pdfParseModule = await import("pdf-parse");
   } catch {
     throw new Error("PDF support requires pdf-parse. Run: bun add pdf-parse");
   }
 
+  const pdfParse = (pdfParseModule.default ?? pdfParseModule) as (buffer: Buffer) => Promise<{ info?: { Title?: string }; text: string }>;
+
   const buffer = await Bun.file(pdfPath).arrayBuffer();
   const data = await pdfParse(Buffer.from(buffer));
 

package/src/llm-client.ts

@@ -46,6 +46,21 @@ async function geminiComplete(config: LLMConfig, system: string, userMessage: st
 
 type ProviderResult = { text: string; usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number } };
 
+function isRetryableError(error: unknown): boolean {
+  if (error instanceof Error) {
+    // Gemini: raw fetch, status in error message
+    if (/\b(429|503)\b/.test(error.message)) return true;
+  }
+  // OpenAI/Azure/Anthropic SDKs: error objects with status property
+  const status = (error as any)?.status;
+  if (status === 429 || status === 503) return true;
+  return false;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
 export class LLMClient {
   private config: LLMConfig;
   private usage: UsageStats = { totalCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
@@ -53,6 +68,8 @@ export class LLMClient {
   private _anthropicClient: InstanceType<typeof import("@anthropic-ai/sdk").default> | null = null;
   private _azureClient: InstanceType<typeof import("openai").AzureOpenAI> | null = null;
 
+  onRetry?: (attempt: number, maxRetries: number, delayMs: number) => void;
+
   constructor(config: LLMConfig) {
     this.config = config;
   }
@@ -65,7 +82,8 @@ export class LLMClient {
     if (!apiKey) {
       try {
         const keyFile = `${process.env.HOME}/keys/openai.azure.com/${this.config.model}.json`;
-        const raw = require("fs").readFileSync(keyFile, "utf-8");
+        const { readFileSync } = await import("fs");
+        const raw = readFileSync(keyFile, "utf-8");
         const keyConfig = JSON.parse(raw)[0] as { key: string; endpoint: string; deployment: string };
         apiKey = keyConfig.key;
         endpoint = keyConfig.endpoint.split("/openai/")[0];
@@ -105,7 +123,7 @@ export class LLMClient {
       this._openaiClient = new OpenAI({ apiKey: this.config.api_key });
     }
     const resp = await this._openaiClient.chat.completions.create({
-      model: this.config.model || "gpt-4o",
+      model: this.config.model || "gpt-5.4",
       messages: [
         { role: "system", content: system },
         { role: "user", content: userMessage },
@@ -128,7 +146,7 @@ export class LLMClient {
       this._anthropicClient = new Anthropic({ apiKey: this.config.api_key });
     }
     const resp = await this._anthropicClient.messages.create({
-      model: this.config.model || "claude-sonnet-4-20250514",
+      model: this.config.model || "claude-sonnet-4-6",
       max_tokens: maxTokens,
       system: system,
       messages: [{ role: "user", content: userMessage }],
@@ -145,34 +163,51 @@ export class LLMClient {
   }
 
   async chatComplete(system: string, userMessage: string, maxTokens = 8192): Promise<string> {
-    let result: ProviderResult;
-
-    switch (this.config.provider) {
-      case "gemini":
-        result = await geminiComplete(this.config, system, userMessage, maxTokens);
-        break;
-      case "azure-openai":
-        result = await this.azureComplete(system, userMessage, maxTokens);
-        break;
-      case "openai":
-        result = await this.openaiComplete(system, userMessage, maxTokens);
-        break;
-      case "anthropic":
-        result = await this.anthropicComplete(system, userMessage, maxTokens);
-        break;
-      default:
-        throw new Error(`Unknown LLM provider: ${this.config.provider}`);
-    }
+    const MAX_RETRIES = 5;
+    const BASE_DELAY_MS = 2000;
 
-    // Track usage
-    if (result.usage) {
-      this.usage.totalCalls++;
-      this.usage.promptTokens += result.usage.prompt_tokens || 0;
-      this.usage.completionTokens += result.usage.completion_tokens || 0;
-      this.usage.totalTokens += result.usage.total_tokens || 0;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        let result: ProviderResult;
+
+        switch (this.config.provider) {
+          case "gemini":
+            result = await geminiComplete(this.config, system, userMessage, maxTokens);
+            break;
+          case "azure-openai":
+            result = await this.azureComplete(system, userMessage, maxTokens);
+            break;
+          case "openai":
+            result = await this.openaiComplete(system, userMessage, maxTokens);
+            break;
+          case "anthropic":
+            result = await this.anthropicComplete(system, userMessage, maxTokens);
+            break;
+          default:
+            throw new Error(`Unknown LLM provider: ${this.config.provider}`);
+        }
+
+        // Track usage
+        if (result.usage) {
+          this.usage.totalCalls++;
+          this.usage.promptTokens += result.usage.prompt_tokens || 0;
+          this.usage.completionTokens += result.usage.completion_tokens || 0;
+          this.usage.totalTokens += result.usage.total_tokens || 0;
+        }
+
+        return result.text;
+      } catch (error) {
+        if (isRetryableError(error) && attempt < MAX_RETRIES) {
+          const delay = BASE_DELAY_MS * Math.pow(2, attempt) + Math.random() * 1000;
+          this.onRetry?.(attempt + 1, MAX_RETRIES, delay);
+          await sleep(delay);
+          continue;
+        }
+        throw error;
+      }
     }
 
-    return result.text;
+    throw new Error("Unreachable: retry loop exited without return or throw");
   }
 
   getUsageStats(): UsageStats {
@@ -211,44 +246,3 @@ export class LLMClient {
   }
 }
 
-// ── Deprecated global state wrappers (for backward compatibility) ──
-
-/** @deprecated Use LLMClient class instead */
-let _globalClient: LLMClient | null = null;
-
-/** @deprecated Use `new LLMClient(config)` instead */
-export function setLLMConfig(config: LLMConfig): void {
-  _globalClient = new LLMClient(config);
-}
-
-/** @deprecated Use LLMClient instance methods instead */
-export function getUsageStats(): UsageStats {
-  if (!_globalClient) return { totalCalls: 0, promptTokens: 0, completionTokens: 0, totalTokens: 0 };
-  return _globalClient.getUsageStats();
-}
-
-/** @deprecated Use LLMClient instance methods instead */
-export function resetUsageStats(): void {
-  if (_globalClient) _globalClient.resetUsageStats();
-}
-
-/** @deprecated Use LLMClient instance methods instead */
-export function getEstimatedCost(): number {
-  if (!_globalClient) return 0;
-  return _globalClient.getEstimatedCost();
-}
-
-/** @deprecated Use LLMClient instance methods instead */
-export function printUsageSummary(): void {
-  if (_globalClient) _globalClient.printUsageSummary();
-}
-
-/** @deprecated Use LLMClient instance methods instead */
-export async function chatComplete(
-  system: string,
-  userMessage: string,
-  maxTokens = 8192
-): Promise<string> {
-  if (!_globalClient) throw new Error("LLM config not set. Call setLLMConfig() first.");
-  return _globalClient.chatComplete(system, userMessage, maxTokens);
-}

package/src/pipeline/citations.ts

@@ -0,0 +1,107 @@
+import type { Store, Citation } from "../store";
+import { escapeHtml } from "../utils";
+
+/**
+ * Parse [^src:SLUG] citation markers in page body.
+ * Creates citation DB records and replaces markers with numbered footnote references.
+ */
+export function parseCitations(body: string, pageId: number, store: Store): string {
+  const page = store.getPageById(pageId);
+  if (!page) return body;
+
+  // Find all [^src:SLUG] markers
+  const markerRegex = /\[\^src:([a-z0-9가-힣][-a-z0-9가-힣]*)\]/gi;
+  const markers: Array<{ fullMatch: string; slug: string; index: number }> = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = markerRegex.exec(body)) !== null) {
+    markers.push({ fullMatch: match[0], slug: match[1], index: match.index });
+  }
+
+  if (markers.length === 0) return body;
+
+  // Delete existing citations for this page to avoid duplicates on re-parse
+  store.deleteCitationsForPage(pageId);
+
+  // Build footnote references
+  let result = body;
+  let footnoteNum = 0;
+  const citationMap = new Map<string, number>(); // slug -> footnote number
+
+  // Process in reverse order to preserve indices during replacement
+  for (let i = markers.length - 1; i >= 0; i--) {
+    const marker = markers[i];
+    const sourcePage = store.getPage(marker.slug);
+
+    if (!sourcePage || !sourcePage.source_id) {
+      // Remove invalid markers silently
+      result = result.slice(0, marker.index) + result.slice(marker.index + marker.fullMatch.length);
+      continue;
+    }
+
+    // Assign footnote number (reuse if same slug cited multiple times)
+    if (!citationMap.has(marker.slug)) {
+      footnoteNum = citationMap.size + 1;
+      citationMap.set(marker.slug, footnoteNum);
+    }
+    const num = citationMap.get(marker.slug)!;
+
+    // Extract context: ~80 chars surrounding the marker
+    const contextStart = Math.max(0, marker.index - 80);
+    const contextEnd = Math.min(body.length, marker.index + marker.fullMatch.length + 80);
+    const context = body.slice(contextStart, contextEnd).replace(/\[\^src:[^\]]+\]/g, '').trim();
+
+    // Create citation record
+    store.addCitation(pageId, sourcePage.source_id, sourcePage.id, null, context);
+
+    // Replace marker with footnote superscript
+    const footnoteRef = `<sup class="citation-ref"><a href="#cite-${num}" title="${escapeCitationTitle(sourcePage.title)}">[${num}]</a></sup>`;
+    result = result.slice(0, marker.index) + footnoteRef + result.slice(marker.index + marker.fullMatch.length);
+  }
+
+  return result;
+}
+
+function escapeCitationTitle(title: string): string {
+  return title.replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+}
+
+/**
+ * Render a "Sources" footer section from citation records.
+ */
+export function renderCitationFootnotes(citations: Citation[]): string {
+  if (!citations.length) return "";
+
+  // Deduplicate by source_page_id (or source_id if no page)
+  const seen = new Map<string, { num: number; citation: Citation }>();
+  let num = 0;
+
+  for (const c of citations) {
+    const key = c.source_page_id ? `page:${c.source_page_id}` : `source:${c.source_id}`;
+    if (!seen.has(key)) {
+      num++;
+      seen.set(key, { num, citation: c });
+    }
+  }
+
+  const items = Array.from(seen.values())
+    .map(({ num, citation }) => {
+      const title = citation.source_page_title || citation.source_title || `Source #${citation.source_id}`;
+      const slug = citation.source_page_slug;
+      const link = slug ? `<a href="/wiki/${encodeURIComponent(slug)}.html">${escapeHtml(title)}</a>` : escapeHtml(title);
+      const excerpt = citation.excerpt
+        ? `<span class="citation-excerpt">"${escapeHtml(citation.excerpt.slice(0, 200))}"</span>`
+        : "";
+      return `<li id="cite-${num}" class="citation-item"><span class="citation-num">[${num}]</span> ${link}${excerpt ? " — " + excerpt : ""}</li>`;
+    })
+    .join("\n");
+
+  return `<aside class="citations-section">
+<h3>Sources</h3>
+<ol class="citation-list">
+${items}
+</ol>
+</aside>`;
+}
+
+// escapeHtml imported from ../utils