@open330/kiwimu 0.7.1 → 1.1.0

package/src/pipeline/llm-chunker.ts

@@ -1,7 +1,10 @@
 import { LLMClient } from "../llm-client";
 import type { Store } from "../store";
 import { slugify } from "./chunker";
-import type { Persona } from "../config";
+import type { Persona, WikiSchema } from "../config";
+import { compileTerms, standardizeTerms } from "./standardizer";
+import { parseCitations } from "./citations";
+import { stripJsonFences } from "../utils";
 
 // ── Phase 1: Extract original document structure ──
 
@@ -16,15 +19,16 @@ TEXT:
 
 Return a JSON array of sections in order. Each element:
 - "title": string — Original section/chapter title from the document
-- "content": string — The full content of this section, converted to clean markdown. Preserve all information. Use LaTeX ($..$ inline, $$...$$ display) for equations. Clean up OCR artifacts.
+- "content": string — The full content of this section, converted to clean markdown. Preserve all information. Use LaTeX ($..$ inline, $$...$$ display) for equations. Clean up OCR artifacts. When the content describes processes, workflows, hierarchies, state transitions, or relationships, add a Mermaid diagram using fenced code blocks (\`\`\`mermaid). Supported types: flowchart, sequenceDiagram, classDiagram, stateDiagram-v2, erDiagram, gantt, pie, mindmap, timeline.
 - "level": number — 1 for chapter, 2 for section, 3 for subsection
 
 Keep the content faithful to the original. Do not add or remove information. Just clean up formatting.
+When appropriate, enhance understanding by including Mermaid diagrams that visualize key concepts, flows, or relationships described in the text.
 Return at most 8 sections per response to keep output manageable.`;
 
 // ── Phase 2: Extract concepts for separate pages ──
 
-function getConceptSystem(persona: Persona | null): string {
+function getConceptSystem(persona: Persona | null, schema?: WikiSchema): string {
   const base = `You are a study wiki editor. Given source material pages, identify important concepts, terms, and definitions that deserve their own dedicated wiki pages.
 
 Rules:
@@ -32,21 +36,53 @@ Rules:
 - Each concept page should have substantial educational content (2+ paragraphs)
 - Explain the concept clearly with definitions, formulas, examples, and context
 - Use [[wiki links]] to reference other concepts and source pages. Example: "[[Synchrotron Radiation]] is observed at [[radio frequencies]]"
+- Use LaTeX ($..$ inline, $$...$$ display) for equations
+- When a concept involves processes, relationships, hierarchies, or state transitions, include a Mermaid diagram using fenced code blocks (\`\`\`mermaid). Supported: flowchart, sequenceDiagram, classDiagram, stateDiagram-v2, erDiagram, mindmap, pie
 - Suggest Wikipedia links for further reading
 
 Return valid JSON only. No markdown fences.`;
 
+  let schemaRules = "";
+  if (schema) {
+    const rules: string[] = [];
+    if (schema.categories?.length) {
+      rules.push(`- Assign each concept to one of these categories: ${schema.categories.join(", ")}. Include a "category" field in your JSON output.`);
+    }
+    if (schema.page_template?.sections?.length) {
+      rules.push(`- Structure each concept page with these sections (use ## headings): ${schema.page_template.sections.join(", ")}`);
+    }
+    if (schema.naming_convention) {
+      const conventions: Record<string, string> = {
+        noun_phrase: "Use noun phrases for titles (e.g., 'Neural Network', 'Gradient Descent')",
+        question: "Use question form for titles (e.g., 'What is a Neural Network?', 'How does Gradient Descent work?')",
+        topic: "Use simple topic words for titles (e.g., 'Backpropagation', 'Optimization')",
+      };
+      rules.push(`- Title format: ${conventions[schema.naming_convention] || schema.naming_convention}`);
+    }
+    if (schema.terms && Object.keys(schema.terms).length > 0) {
+      const termList = Object.entries(schema.terms).map(([k, v]) => `${k} -> ${v}`).join(", ");
+      rules.push(`- Use these standard terms (replace abbreviations with full forms): ${termList}`);
+    }
+    if (rules.length > 0) {
+      schemaRules = `\n\nSchema rules:\n${rules.join("\n")}`;
+    }
+  }
+
   if (persona) {
-    return `${persona.system_prompt}\n\n${base}\n\nIMPORTANT: ${persona.content_style}`;
+    return `${persona.system_prompt}\n\n${base}${schemaRules}\n\nIMPORTANT: ${persona.content_style}`;
   }
-  return base;
+  return `${base}${schemaRules}`;
 }
 
-function getConceptPrompt(persona: Persona | null): string {
+function getConceptPrompt(persona: Persona | null, schema?: WikiSchema): string {
   const styleNote = persona
     ? `\n\nWrite content in the following style:\n${persona.content_style}`
     : "";
 
+  const categoryField = schema?.categories?.length
+    ? `\n- "category": string — One of: ${schema.categories.join(", ")}`
+    : "";
+
   return `Based on these source pages, create concept/glossary wiki pages for important terms.
 
 Source pages already created:
@@ -56,9 +92,15 @@ Create 3-6 concept pages for the most important terms, definitions, laws, and eq
 Do NOT duplicate the source pages — instead, create focused concept pages that the source pages can link to.
 Keep each page concise (2-3 paragraphs).${styleNote}
 
+IMPORTANT — Provenance citations:
+When a claim or fact comes from a specific source page, add an inline citation marker at the end of that sentence using the format [^src:SOURCE_PAGE_SLUG].
+The SOURCE_PAGE_SLUG must match one of the source page slugs listed above (the hyphenated identifier shown after the title).
+Example: "Quantum entanglement allows particles to share states instantly [^src:chapter-3-quantum-states]"
+Only cite when a fact clearly originates from a specific source page. Not every sentence needs a citation.
+
 Return a JSON array where each element has:
 - "title": string — Short concept name, 1-3 words (e.g., "Synchrotron Radiation", "Flux Density", "Angular Resolution"). Keep titles short so they match naturally in text.
-- "content": string — Educational markdown content with [[wiki links]] to other concepts and source pages
+- "content": string — Educational markdown content with [[wiki links]] to other concepts and source pages, and [^src:slug] citations where appropriate${categoryField}
 - "suggested_links": Array<{text: string, url: string}> — Wikipedia/external reference links`;
 }
 
@@ -82,6 +124,7 @@ interface StructurePage {
 interface ConceptPage {
   title: string;
   content: string;
+  category?: string;
   suggested_links?: Array<{ text: string; url: string }>;
 }
 
@@ -146,7 +189,7 @@ function splitBySize(text: string, maxSize: number): Array<{ chapterHint: string
 }
 
 function parseJSON<T>(raw: string): T {
-  let cleaned = raw.replace(/^```json?\n?/m, "").replace(/\n?```\s*$/m, "").trim();
+  let cleaned = stripJsonFences(raw);
   try {
     return JSON.parse(cleaned);
   } catch (e1) {
@@ -215,15 +258,17 @@ export async function llmChunkDocument(
   store: Store,
   maxChunks: number = 0, // 0 = unlimited
   persona: Persona | null = null,
-  llmClient?: LLMClient
+  llmClient: LLMClient,
+  onProgress?: (status: string) => void,
+  schema?: WikiSchema
 ): Promise<{ sourceCount: number; conceptCount: number }> {
-  // Use provided client or fall back to deprecated global chatComplete
-  const chat = llmClient
-    ? (system: string, user: string, maxTokens?: number) => llmClient.chatComplete(system, user, maxTokens)
-    : async (system: string, user: string, maxTokens?: number) => {
-        const { chatComplete } = await import("../llm-client");
-        return chatComplete(system, user, maxTokens);
-      };
+  const chat = (system: string, user: string, maxTokens?: number) =>
+    llmClient.chatComplete(system, user, maxTokens);
+
+  // Pre-compile term standardization regexes if schema.terms is defined
+  const compiledTerms = schema?.terms && Object.keys(schema.terms).length > 0
+    ? compileTerms(schema.terms)
+    : null;
 
   let chunks = splitByChapters(rawText);
   if (maxChunks > 0 && chunks.length > maxChunks) {
@@ -233,128 +278,230 @@ export async function llmChunkDocument(
   if (persona) {
     console.log(`\x1b[35m🎭 페르소나: ${persona.name}\x1b[0m`);
   }
-  console.log(`\x1b[34m🧠 Phase 1: 원본 구조 추출 (${chunks.length}개 청크)...\x1b[0m`);
-
   // ── Phase 1: Extract source pages (parallel LLM calls) ──
-  let completedCount = 0;
-  const structureSystem = getStructureSystem(persona);
-
-  const chunkResults = await parallelMap(chunks, 3, async (chunk, i) => {
-    console.log(`  Phase 1: 처리 중 [${i + 1}/${chunks.length}] ${chunk.chapterHint}...`);
+  let sourceCount: number;
+  let sourcePageSummaries: string[];
+
+  if (store.hasPhaseCheckpoint(sourceId, 'phase1')) {
+    // Resume: Phase 1 already done, rebuild summaries from DB
+    const existingPages = store.getSourcePages(sourceId);
+    sourceCount = existingPages.length;
+    sourcePageSummaries = existingPages.map(p =>
+      `- ${p.title} [slug: ${p.slug}]: ${p.content.slice(0, 150).replace(/\n/g, " ")}`
+    );
+    console.log(`\x1b[32m⏭ Phase 1 건너뜀 (이미 완료) — 📖 ${sourceCount}개 원본 페이지\x1b[0m`);
+    onProgress?.(`Phase 1 건너뜀 (${sourceCount}개 페이지 이미 존재)`);
+  } else {
+    console.log(`\x1b[34m⏳ Phase 1: 원본 구조 추출 중... (${chunks.length}개 청크)\x1b[0m`);
+    onProgress?.(`Phase 1: 원본 구조 추출 중... (${chunks.length}개 청크)`);
+
+    const phase1Start = performance.now();
+    let completedCount = 0;
+    const structureSystem = getStructureSystem(persona);
+
+    const chunkResults = await parallelMap(chunks, 3, async (chunk, i) => {
+      console.log(`  Phase 1: 처리 중 [${i + 1}/${chunks.length}] ${chunk.chapterHint}...`);
+
+      const prompt = STRUCTURE_PROMPT
+        .replace("{sourceTitle}", sourceTitle)
+        .replace("{text}", chunk.text.slice(0, 80000));
 
-    const prompt = STRUCTURE_PROMPT
-      .replace("{sourceTitle}", sourceTitle)
-      .replace("{text}", chunk.text.slice(0, 80000));
-
-    try {
-      let raw = await chat(structureSystem, prompt, 16384);
-      if (!raw || raw.trim().length < 10) {
-        console.log(`    \x1b[33m⚠ 빈 응답, 재시도...\x1b[0m`);
-        raw = await chat(structureSystem, prompt, 16384);
+      try {
+        let raw = await chat(structureSystem, prompt, 16384);
         if (!raw || raw.trim().length < 10) {
-          console.log(`    \x1b[31m✗ 재시도도 빈 응답\x1b[0m`);
-          completedCount++;
-          return [] as StructurePage[];
+          console.log(`    \x1b[33m⚠ 빈 응답, 재시도...\x1b[0m`);
+          raw = await chat(structureSystem, prompt, 16384);
+          if (!raw || raw.trim().length < 10) {
+            console.log(`    \x1b[31m✗ 재시도도 빈 응답\x1b[0m`);
+            completedCount++;
+            return [] as StructurePage[];
+          }
         }
+        const sections = parseJSON<StructurePage[]>(raw).filter(s => s.title && s.content && s.content.length > 30);
+        completedCount++;
+        console.log(`    → ${sections.length}개 섹션 (완료 ${completedCount}/${chunks.length})`);
+        onProgress?.(`Phase 1: ${completedCount}/${chunks.length} 청크 완료`);
+        return sections;
+      } catch (e: unknown) {
+        const message = e instanceof Error ? e.message : String(e);
+        console.log(`    \x1b[31m✗ 실패: ${message}\x1b[0m`);
+        completedCount++;
+        return [] as StructurePage[];
       }
-      const sections = parseJSON<StructurePage[]>(raw).filter(s => s.title && s.content && s.content.length > 30);
-      completedCount++;
-      console.log(`    → ${sections.length}개 섹션 (완료 ${completedCount}/${chunks.length})`);
-      return sections;
-    } catch (e: unknown) {
-      const message = e instanceof Error ? e.message : String(e);
-      console.log(`    \x1b[31m✗ 실패: ${message}\x1b[0m`);
-      completedCount++;
-      return [] as StructurePage[];
-    }
-  });
+    });
 
-  // Store results sequentially (SQLite writes must be sequential)
-  let orderCounter = 0;
-  const sourcePageSummaries: string[] = [];
+    // Store results sequentially (SQLite writes must be sequential)
+    let orderCounter = 0;
+    sourcePageSummaries = [];
 
-  for (const sections of chunkResults) {
-    for (const section of sections) {
-      const slug = slugify(section.title);
-      if (!slug) continue;
+    for (const sections of chunkResults) {
+      for (const section of sections) {
+        const slug = slugify(section.title);
+        if (!slug) continue;
 
-      const existing = store.getPage(slug);
-      if (existing) {
-        store.updatePageContent(existing.id, existing.content + "\n\n" + section.content);
-      } else {
-        store.addPage(slug, section.title, section.content, sourceId, slug, "source", orderCounter++);
-        sourcePageSummaries.push(`- ${section.title}: ${section.content.slice(0, 150).replace(/\n/g, " ")}`);
+        const existing = store.getPage(slug);
+        if (existing) {
+          store.updatePageContent(existing.id, existing.content + "\n\n" + section.content);
+        } else {
+          const page = store.addPage(slug, section.title, section.content, sourceId, slug, "source", orderCounter++);
+          store.addActivityLog('page_created', `Created page: ${section.title}`, 'page', page.id);
+          sourcePageSummaries.push(`- ${section.title} [slug: ${slug}]: ${section.content.slice(0, 150).replace(/\n/g, " ")}`);
+
+        }
       }
     }
-  }
 
-  const sourceCount = orderCounter;
-  console.log(`\x1b[32m  📖 ${sourceCount}개 원본 페이지 생성 완료\x1b[0m`);
+    sourceCount = orderCounter;
+    store.setCheckpoint(sourceId, 'phase1');
+    const phase1Sec = ((performance.now() - phase1Start) / 1000).toFixed(1);
+    console.log(`\x1b[32m✅ Phase 1 완료 (${phase1Sec}초) — 📖 ${sourceCount}개 원본 페이지 생성\x1b[0m`);
+  }
 
   // ── Phase 2: Extract concept pages ──
-  console.log(`\x1b[34m🧠 Phase 2: 개념 페이지 추출...\x1b[0m`);
-
-  // Process source pages in small batches for concept extraction
   const batchSize = 5;
   let conceptCount = 0;
+  // Cache concept pages list for reuse in Phase 2 and Phase 2.5
+  let cachedConceptPages: ReturnType<typeof store.listConceptPages> | null = null;
+
+  if (sourcePageSummaries.length === 0) {
+    console.log(`\x1b[33m⏭ Phase 2 건너뜀 (원본 페이지 없음)\x1b[0m`);
+    onProgress?.(`Phase 2 건너뜀 (원본 페이지 없음)`);
+  } else {
+    const totalBatches = Math.ceil(sourcePageSummaries.length / batchSize);
+    const lastCompletedBatch = store.getLastCompletedBatch(sourceId, 'phase2');
+
+    if (lastCompletedBatch >= totalBatches - 1 && store.hasPhaseCheckpoint(sourceId, 'phase2')) {
+      cachedConceptPages = store.listConceptPages();
+      conceptCount = cachedConceptPages.length;
+      console.log(`\x1b[32m⏭ Phase 2 건너뜀 (이미 완료) — 📝 ${conceptCount}개 개념 페이지\x1b[0m`);
+      onProgress?.(`Phase 2 건너뜀 (${conceptCount}개 개념 이미 존재)`);
+    } else {
+      const phase2Start = performance.now();
+      const resumeFrom = lastCompletedBatch + 1;
+      if (resumeFrom > 0) {
+        console.log(`\x1b[34m⏳ Phase 2: 개념 추출 재개 (배치 ${resumeFrom + 1}/${totalBatches}부터)...\x1b[0m`);
+        onProgress?.(`Phase 2: 배치 ${resumeFrom + 1}/${totalBatches}부터 재개`);
+      } else {
+        console.log(`\x1b[34m⏳ Phase 2: 개념 추출 중...\x1b[0m`);
+        onProgress?.(`Phase 2: 개념 추출 중...`);
+      }
 
-  for (let i = 0; i < sourcePageSummaries.length; i += batchSize) {
-    const batch = sourcePageSummaries.slice(i, i + batchSize);
-    const batchLabel = `  [${Math.floor(i / batchSize) + 1}/${Math.ceil(sourcePageSummaries.length / batchSize)}]`;
-    console.log(`${batchLabel} 개념 추출 중...`);
+      // Cache existing concept titles in memory to avoid repeated DB queries
+      const existingConceptTitles = new Set(store.listConceptPages().map(p => p.title));
 
-    const existingConcepts = conceptCount > 0
-      ? `\n\nAlready created concept pages (do not duplicate): ${store.listConceptPages().map(p => p.title).join(", ")}`
-      : "";
-    const conceptPrompt = getConceptPrompt(persona);
-    const prompt = conceptPrompt.replace("{sourcePages}", batch.join("\n")) + existingConcepts;
-    const conceptSystem = getConceptSystem(persona);
+      for (let i = 0; i < sourcePageSummaries.length; i += batchSize) {
+        const batchIdx = Math.floor(i / batchSize);
+        const batchLabel = `  [${batchIdx + 1}/${totalBatches}]`;
 
-    try {
-      const raw = await chat(conceptSystem, prompt, 16384);
-      const concepts = parseJSON<ConceptPage[]>(raw).filter(c => c.title && c.content && c.content.length > 50);
+        if (batchIdx <= lastCompletedBatch) {
+          console.log(`${batchLabel} 이미 완료 — 건너뜀`);
+          continue;
+        }
 
-      for (const concept of concepts) {
-        const slug = slugify(concept.title);
-        if (!slug) continue;
+        console.log(`${batchLabel} 개념 추출 중...`);
 
-        // Don't create concept page if source page with same slug exists
-        const existing = store.getPage(slug);
-        if (existing) continue;
+        const batch = sourcePageSummaries.slice(i, i + batchSize);
+        const existingConceptsNote = existingConceptTitles.size > 0
+          ? `\n\nAlready created concept pages (do not duplicate): ${[...existingConceptTitles].join(", ")}`
+          : "";
+        const conceptPrompt = getConceptPrompt(persona, schema);
+        const prompt = conceptPrompt.replace("{sourcePages}", batch.join("\n")) + existingConceptsNote;
+        const conceptSystem = getConceptSystem(persona, schema);
+
+        try {
+          const raw = await chat(conceptSystem, prompt, 16384);
+          const concepts = parseJSON<ConceptPage[]>(raw).filter(c => c.title && c.content && c.content.length > 50);
+
+          for (const concept of concepts) {
+            const slug = slugify(concept.title);
+            if (!slug) continue;
+
+            const existing = store.getPage(slug);
+            if (existing) continue;
+
+            let content = concept.content;
+            // Apply term standardization if schema.terms is defined
+            if (compiledTerms) {
+              content = standardizeTerms(content, compiledTerms);
+            }
+            if (concept.suggested_links?.length) {
+              content += "\n\n## External References\n\n";
+              for (const link of concept.suggested_links) {
+                content += `- [${link.text}](${link.url})\n`;
+              }
+            }
 
-        let content = concept.content;
-        if (concept.suggested_links?.length) {
-          content += "\n\n## External References\n\n";
-          for (const link of concept.suggested_links) {
-            content += `- [${link.text}](${link.url})\n`;
+            const conceptPage = store.addPage(slug, concept.title, content, sourceId, slug, "concept", 0);
+            store.addActivityLog('page_created', `Created page: ${concept.title}`, 'page', conceptPage.id);
+            // Store category if provided by LLM and schema supports it
+            if (concept.category && schema?.categories?.length) {
+              store.updatePageCategory(conceptPage.id, concept.category);
+            }
+            existingConceptTitles.add(concept.title);
+            conceptCount++;
           }
+          store.setCheckpoint(sourceId, 'phase2', batchIdx);
+          console.log(`    → ${concepts.length}개 개념`);
+          onProgress?.(`Phase 2: ${batchIdx + 1}/${totalBatches} 배치 완료`);
+        } catch (e: unknown) {
+          const message = e instanceof Error ? e.message : String(e);
+          console.log(`    \x1b[31m✗ 배치 ${batchIdx + 1} 실패 (건너뜀): ${message}\x1b[0m`);
+          // Non-retryable errors (parse failures, etc.) — skip batch, continue pipeline
+          // Rate-limit errors are already retried in LLMClient; if we reach here, retries exhausted
+          store.setCheckpoint(sourceId, 'phase2', batchIdx);
         }
-
-        store.addPage(slug, concept.title, content, sourceId, slug, "concept", 0);
-        conceptCount++;
       }
-      console.log(`    → ${concepts.length}개 개념`);
-    } catch (e: unknown) {
-      const message = e instanceof Error ? e.message : String(e);
-      console.log(`    \x1b[31m✗ 실패: ${message}\x1b[0m`);
+
+      const phase2Sec = ((performance.now() - phase2Start) / 1000).toFixed(1);
+      console.log(`\x1b[32m✅ Phase 2 완료 (${phase2Sec}초) — 📝 ${conceptCount}개 개념 페이지 생성\x1b[0m`);
+      // Invalidate cache since new concepts were added
+      cachedConceptPages = null;
     }
   }
 
-  console.log(`\x1b[32m  📝 ${conceptCount}개 개념 페이지 생성 완료\x1b[0m`);
+  // ── Phase 2 post-processing: Parse citation markers ──
+  {
+    const conceptPagesForCitations = store.listConceptPages();
+    let citationCount = 0;
+    for (const page of conceptPagesForCitations) {
+      if (page.content.includes("[^src:")) {
+        const parsed = parseCitations(page.content, page.id, store);
+        if (parsed !== page.content) {
+          store.updatePageContent(page.id, parsed);
+          citationCount++;
+        }
+      }
+    }
+    if (citationCount > 0) {
+      console.log(`\x1b[32m  📚 ${citationCount}개 페이지에서 인용 정보 생성 완료\x1b[0m`);
+    }
+  }
 
   // ── Phase 2.5: Generate quizzes from concept pages ──
   let quizCount = 0;
-  try {
-    const conceptPagesForQuiz = store.listConceptPages();
-    if (conceptPagesForQuiz.length > 0) {
-      console.log(`\x1b[34m🧠 Phase 2.5: 퀴즈 생성 (${conceptPagesForQuiz.length}개 개념 페이지)...\x1b[0m`);
-
-      const quizSystem = `You are a quiz generator for a study wiki. Generate quiz questions based on wiki content.
+  if (store.hasPhaseCheckpoint(sourceId, 'phase2_5')) {
+    console.log(`\x1b[32m⏭ Phase 2.5 건너뜀 (퀴즈 이미 생성됨)\x1b[0m`);
+    onProgress?.(`Phase 2.5 건너뜀 (퀴즈 이미 존재)`);
+  } else {
+    try {
+      const conceptPagesForQuiz = cachedConceptPages ?? store.listConceptPages();
+      if (conceptPagesForQuiz.length > 0) {
+        console.log(`\x1b[34m⏳ Phase 2.5: 퀴즈 생성 중... (${conceptPagesForQuiz.length}개 개념 페이지)\x1b[0m`);
+        onProgress?.(`Phase 2.5: 퀴즈 생성 중...`);
+
+        let quizSystemExtra = "";
+        if (schema?.terms && Object.keys(schema.terms).length > 0) {
+          const termList = Object.entries(schema.terms).map(([k, v]) => `${k} -> ${v}`).join(", ");
+          quizSystemExtra = `\nUse these standard terms in questions and answers (replace abbreviations with full forms): ${termList}`;
+        }
+        const quizSystem = `You are a quiz generator for a study wiki. Generate quiz questions that test UNDERSTANDING, not just memorization.
+Focus on higher-order thinking: "왜?", "어떻게?", "비교하라", "설명하라" style questions.${quizSystemExtra}
 Return valid JSON only. No markdown fences.`;
 
-      await parallelMap(conceptPagesForQuiz, 3, async (page, i) => {
-        try {
-          const quizPrompt = `Based on this wiki content, generate 2-3 quiz questions in JSON format.
+        await parallelMap(conceptPagesForQuiz, 3, async (page, i) => {
+          try {
+            const quizPrompt = `Based on this wiki content, generate 2-3 quiz questions that test UNDERSTANDING, not just memorization.
+Include questions that ask "왜?", "어떻게?", "비교하라" etc.
 Types: "fill_blank" (빈칸 채우기), "ox" (OX 퀴즈 - true/false), "short_answer" (단답형)
 
 Content title: ${page.title}
@@ -362,37 +509,43 @@ Content:
 ${page.content.slice(0, 3000)}
 
 Respond with a JSON array only:
-[{"question": "___은 양자역학에서 위치와 운동량을 동시에 측정할 수 없다는 원리이다.", "answer": "불확정성 원리", "type": "fill_blank"}]
+[{"question": "___은 양자역학에서 위치와 운동량을 동시에 측정할 수 없다는 원리이다.", "answer": "불확정성 원리", "explanation": "이 원리는 양자역학의 근본적 한계를 보여주며, 측정 행위 자체가 시스템에 영향을 주기 때문입니다.", "type": "fill_blank"}]
 
 Rules:
 - For fill_blank: use ___ to mark the blank in the question
 - For ox: question should be a statement, answer should be "O" or "X"
 - For short_answer: question should be answerable in 1-3 words
-- Questions should test understanding, not just recall
+- Include "explanation" field: a brief 1-2 sentence explanation of WHY the answer is correct
+- Questions should test understanding, application, or analysis — not just recall
 - Write questions in Korean when the content is in Korean`;
 
-          const raw = await chat(quizSystem, quizPrompt, 2048);
-          const quizzes = parseJSON<Array<{ question: string; answer: string; type: string }>>(raw);
-
-          for (const q of quizzes) {
-            if (q.question && q.answer && q.type) {
-              store.addQuiz(page.id, q.question, q.answer, q.type);
-              quizCount++;
+            const raw = await chat(quizSystem, quizPrompt, 2048);
+            const quizzes = parseJSON<Array<{ question: string; answer: string; explanation?: string; type: string }>>(raw);
+
+            for (const q of quizzes) {
+              if (q.question && q.answer && q.type) {
+                const question = compiledTerms ? standardizeTerms(q.question, compiledTerms) : q.question;
+                const answer = compiledTerms ? standardizeTerms(q.answer, compiledTerms) : q.answer;
+                const explanation = compiledTerms && q.explanation ? standardizeTerms(q.explanation, compiledTerms) : (q.explanation || "");
+                store.addQuiz(page.id, question, answer, q.type, explanation);
+                quizCount++;
+              }
             }
+          } catch (e: unknown) {
+            // Quiz generation is non-critical; silently skip failures
+            const message = e instanceof Error ? e.message : String(e);
+            console.log(`    \x1b[33m⚠ 퀴즈 생성 실패 (${page.title}): ${message}\x1b[0m`);
           }
-        } catch (e: unknown) {
-          // Quiz generation is non-critical; silently skip failures
-          const message = e instanceof Error ? e.message : String(e);
-          console.log(`    \x1b[33m⚠ 퀴즈 생성 실패 (${page.title}): ${message}\x1b[0m`);
-        }
-      });
+        });
 
-      console.log(`\x1b[32m  🧩 ${quizCount}개 퀴즈 생성 완료\x1b[0m`);
+        store.setCheckpoint(sourceId, 'phase2_5');
+        console.log(`\x1b[32m  🧩 ${quizCount}개 퀴즈 생성 완료\x1b[0m`);
+      }
+    } catch (e: unknown) {
+      // Phase 2.5 is optional — don't block the pipeline
+      const message = e instanceof Error ? e.message : String(e);
+      console.log(`\x1b[33m  ⚠ 퀴즈 생성 단계 건너뜀: ${message}\x1b[0m`);
     }
-  } catch (e: unknown) {
-    // Phase 2.5 is optional — don't block the pipeline
-    const message = e instanceof Error ? e.message : String(e);
-    console.log(`\x1b[33m  ⚠ 퀴즈 생성 단계 건너뜀: ${message}\x1b[0m`);
   }
 
   // ── Phase 3: Resolve wiki links + inject concept links into source pages ──
@@ -470,8 +623,8 @@ Rules:
   return { sourceCount, conceptCount };
 }
 
-export function htmlToRawText(html: string): string {
-  const { load } = require("cheerio");
+export async function htmlToRawText(html: string): Promise<string> {
+  const { load } = await import("cheerio");
   const $ = load(html);
   $("script, style, nav, header, footer, noscript").remove();
   return $("body").text() || $.text();

package/src/pipeline/standardizer.ts

@@ -0,0 +1,41 @@
+/**
+ * Term standardization post-processor.
+ * Replaces abbreviations/variants with their standard forms,
+ * using case-insensitive word-boundary matching.
+ */
+
+interface CompiledTerm {
+  regex: RegExp;
+  replacement: string;
+}
+
+/**
+ * Compile term mappings into reusable RegExp objects.
+ * Call once, reuse the result for multiple standardizeTerms calls.
+ */
+export function compileTerms(terms: Record<string, string>): CompiledTerm[] {
+  const compiled: CompiledTerm[] = [];
+  for (const [abbrev, standard] of Object.entries(terms)) {
+    // Escape special regex chars in the abbreviation
+    const escaped = abbrev.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    // Word-boundary aware, case-insensitive
+    // Negative lookbehind/lookahead to avoid matching inside markdown links
+    compiled.push({
+      regex: new RegExp(`(?<!\\[)\\b(${escaped})\\b(?!\\])(?![^[]*\\])`, "gi"),
+      replacement: standard,
+    });
+  }
+  return compiled;
+}
+
+/**
+ * Apply term standardization to content.
+ * Replaces abbreviations with standard terms using pre-compiled regexes.
+ */
+export function standardizeTerms(content: string, compiledTerms: CompiledTerm[]): string {
+  let result = content;
+  for (const { regex, replacement } of compiledTerms) {
+    result = result.replace(regex, replacement);
+  }
+  return result;
+}