@open330/kiwimu 0.8.0 → 1.1.0

package/src/services/dynamic-qa.ts

@@ -0,0 +1,190 @@
+import { Store } from "../store";
+import { LLMClient } from "../llm-client";
+import { loadConfig, getActivePersona, type Persona, type LLMConfig } from "../config";
+import { slugify } from "../pipeline/chunker";
+import { stripJsonFences } from "../utils";
+
+export interface DynamicQAResult {
+  pageId: number;
+  slug: string;
+  title: string;
+  content: string;
+  isPromotable: boolean;
+  suggestedTitle: string;
+  keyConcepts: string[];
+}
+
+export async function generateDynamicPage(
+  store: Store,
+  llmClient: LLMClient,
+  persona: Persona | null,
+  parentPage: { id: number; slug: string; title: string; content: string },
+  selectedText: string,
+  userQuestion: string
+): Promise<DynamicQAResult> {
+  // 1. Build context hierarchy
+  // L1: selectedText (truncate to 2000 chars)
+  // L2: parentPage.content
+  // L3: related pages (backlinks + forward links, title + first 300 chars)
+  // L4: all concept titles
+
+  const backlinks = store.getBacklinks(parentPage.id);
+  const allLinks = store.getForwardLinks(parentPage.id);
+
+  const relatedSummaries = [...backlinks, ...allLinks]
+    .map(p => `- ${p.title}: ${(p as any).content?.slice(0, 300) || ''}`)
+    .slice(0, 10)
+    .join('\n');
+
+  const conceptTitles = store.listConceptPages()
+    .slice(0, 50)
+    .map(p => p.title)
+    .join(', ');
+
+  // 2. Build system prompt
+  const personaStyle = persona ? `\n\nStyle: ${persona.content_style || persona.system_prompt || ''}` : '';
+
+  const systemPrompt = `You are a study wiki editor. A student is reading a wiki page and has selected some text. They have a follow-up question about it. Your job is to create a new, self-contained concept page that answers their question thoroughly.
+
+Rules:
+- Create a focused wiki page (2-4 paragraphs) that answers the question
+- Use [[wiki links]] to reference existing concepts where relevant
+- Include examples, formulas (LaTeX $..$ / $$...$$), and definitions as appropriate
+- The page should be educational and self-contained
+- Return valid JSON only${personaStyle}`;
+
+  const userPrompt = `## Student's Question
+${userQuestion}
+
+## Selected Text
+${selectedText.slice(0, 2000)}
+
+## Current Page: ${parentPage.title}
+${parentPage.content.slice(0, 5000)}
+
+## Related Pages
+${relatedSummaries || '(none)'}
+
+## All Wiki Concepts
+${conceptTitles || '(none)'}
+
+Return a JSON object:
+{"title": "Short concept title", "content": "Full markdown content with [[wiki links]]", "isPromotable": true, "keyConcepts": ["concept1", "concept2"]}
+
+- "isPromotable": true if the answer has enough educational substance (2+ paragraphs, definitions, examples) to be a standalone wiki page, false if it's just a brief clarification
+- "keyConcepts": array of 1-5 key concept terms mentioned in the answer`;
+
+  // 3. Call LLM
+  const raw = await llmClient.chatComplete(systemPrompt, userPrompt, 4096);
+
+  // 4. Parse response — robust JSON extraction with multiple fallbacks
+  let parsed: { title: string; content: string; isPromotable?: boolean; keyConcepts?: string[] };
+  try {
+    // Remove markdown code fences if present
+    let cleaned = stripJsonFences(raw);
+
+    // Try to extract JSON object from response
+    const jsonMatch = cleaned.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) throw new Error("No JSON found");
+
+    let jsonStr = jsonMatch[0];
+    try {
+      parsed = JSON.parse(jsonStr);
+    } catch {
+      // Try repairs for truncated JSON
+      // Close unclosed strings
+      const lastQuote = jsonStr.lastIndexOf('"');
+      const afterQuote = jsonStr.slice(lastQuote + 1);
+      if (afterQuote.indexOf('"') === -1 && afterQuote.length > 0) {
+        jsonStr = jsonStr.slice(0, jsonStr.lastIndexOf('",') + 2) + '}';
+      }
+      // Balance braces
+      const openBraces = (jsonStr.match(/\{/g) || []).length;
+      const closeBraces = (jsonStr.match(/\}/g) || []).length;
+      for (let i = 0; i < openBraces - closeBraces; i++) jsonStr += "}";
+      parsed = JSON.parse(jsonStr);
+    }
+
+    // Ensure content is not JSON-encoded (sometimes LLM double-encodes)
+    if (parsed.content && parsed.content.startsWith('{')) {
+      try {
+        const inner = JSON.parse(parsed.content);
+        if (inner.content) parsed.content = inner.content;
+      } catch { /* not double-encoded, use as-is */ }
+    }
+  } catch {
+    // Fallback: treat the entire raw response as markdown content
+    // Strip any JSON artifacts from the beginning
+    let fallbackContent = stripJsonFences(raw)
+      .replace(/^\s*\{\s*"title"\s*:\s*"[^"]*"\s*,\s*"content"\s*:\s*"?/m, "")
+      .replace(/"\s*\}\s*$/m, "")
+      .trim();
+
+    // If it still looks like JSON, try one more parse
+    if (fallbackContent.startsWith('{')) {
+      try {
+        const lastTry = JSON.parse(fallbackContent);
+        if (lastTry.content) fallbackContent = lastTry.content;
+      } catch { /* use as-is */ }
+    }
+
+    parsed = {
+      title: userQuestion.slice(0, 50),
+      content: fallbackContent || raw
+    };
+  }
+
+  // Unescape JSON string escapes that might remain in content
+  if (parsed.content) {
+    parsed.content = parsed.content
+      .replace(/\\n/g, '\n')
+      .replace(/\\t/g, '\t')
+      .replace(/\\"/g, '"')
+      .replace(/\\\\/g, '\\');
+  }
+
+  if (!parsed.title || !parsed.content || parsed.content.length < 20) {
+    throw new Error("LLM 응답이 불충분합니다. 다시 시도해주세요.");
+  }
+
+  // 5. Generate slug, handle collision
+  let slug = slugify(parsed.title);
+  if (!slug) slug = slugify(userQuestion);
+  if (!slug) slug = `dynamic-${Date.now()}`;
+
+  let finalSlug = slug;
+  let counter = 2;
+  while (store.getPage(finalSlug)) {
+    finalSlug = `${slug}-${counter++}`;
+  }
+
+  // 6. Store the page
+  const pageId = store.addDynamicPage(finalSlug, parsed.title, parsed.content, parentPage.id, userQuestion);
+
+  // 7. Add link from parent to new page
+  store.addLink(parentPage.id, pageId, parsed.title);
+
+  // 8. Log usage
+  const usage = llmClient.getUsageStats();
+  const estimatedCostUsd = llmClient.getEstimatedCost();
+  store.addUsageLog(null, usage.totalCalls, usage.promptTokens, usage.completionTokens, usage.totalTokens, estimatedCostUsd);
+
+  store.addActivityLog('query', `Asked: ${userQuestion.slice(0, 80)}`, 'page', pageId, { parentSlug: parentPage.slug, selectedText: selectedText.slice(0, 200) });
+
+  // Determine promotability: content should be substantial (2+ paragraphs, 200+ chars)
+  const isPromotable = parsed.isPromotable !== undefined
+    ? parsed.isPromotable
+    : parsed.content.length >= 200 && (parsed.content.match(/\n\n/g) || []).length >= 1;
+
+  const keyConcepts = Array.isArray(parsed.keyConcepts) ? parsed.keyConcepts.filter(c => typeof c === 'string') : [];
+
+  return {
+    pageId,
+    slug: finalSlug,
+    title: parsed.title,
+    content: parsed.content,
+    isPromotable,
+    suggestedTitle: parsed.title,
+    keyConcepts,
+  };
+}

package/src/services/embedding.ts

@@ -0,0 +1,122 @@
+import type { Store } from "../store";
+import type { LLMConfig } from "../config";
+
+// Cosine similarity between two vectors
+function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+// Get embedding — auto-detect provider
+async function getEmbedding(text: string, config: LLMConfig): Promise<Float32Array> {
+  const input = text.slice(0, 8000);
+
+  if (config.provider === "gemini") {
+    return await geminiEmbedding(input, config);
+  } else if (config.provider === "azure-openai") {
+    return await azureEmbedding(input, config);
+  } else if (config.provider === "openai") {
+    return await openaiEmbedding(input, config);
+  }
+  throw new Error(`Embedding not supported for provider: ${config.provider}`);
+}
+
+// Gemini Embedding API (free)
+async function geminiEmbedding(text: string, config: LLMConfig): Promise<Float32Array> {
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-001:embedContent`;
+  const resp = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", "x-goog-api-key": config.api_key },
+    body: JSON.stringify({
+      model: "models/gemini-embedding-001",
+      content: { parts: [{ text }] }
+    })
+  });
+  if (!resp.ok) throw new Error(`Gemini embedding error (${resp.status})`);
+  const data = await resp.json() as { embedding: { values: number[] } };
+  return new Float32Array(data.embedding.values);
+}
+
+// Azure OpenAI Embedding
+async function azureEmbedding(text: string, config: LLMConfig): Promise<Float32Array> {
+  const url = `${config.endpoint}/openai/deployments/text-embedding-3-small/embeddings?api-version=2024-06-01`;
+  const resp = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", "api-key": config.api_key },
+    body: JSON.stringify({ input: text, model: "text-embedding-3-small" })
+  });
+  if (!resp.ok) throw new Error(`Azure embedding error (${resp.status})`);
+  const data = await resp.json() as { data: Array<{ embedding: number[] }> };
+  return new Float32Array(data.data[0].embedding);
+}
+
+// OpenAI Embedding
+async function openaiEmbedding(text: string, config: LLMConfig): Promise<Float32Array> {
+  const resp = await fetch("https://api.openai.com/v1/embeddings", {
+    method: "POST",
+    headers: { "Content-Type": "application/json", "Authorization": `Bearer ${config.api_key}` },
+    body: JSON.stringify({ input: text, model: "text-embedding-3-small" })
+  });
+  if (!resp.ok) throw new Error(`OpenAI embedding error (${resp.status})`);
+  const data = await resp.json() as { data: Array<{ embedding: number[] }> };
+  return new Float32Array(data.data[0].embedding);
+}
+
+// Generate embeddings for all pages that don't have one
+export async function generateMissingEmbeddings(store: Store, config: LLMConfig, onProgress?: (msg: string) => void): Promise<number> {
+  const pages = store.getPagesWithoutEmbeddings();
+  if (!pages.length) return 0;
+
+  onProgress?.(`⏳ ${pages.length}개 페이지 임베딩 생성 중...`);
+  let count = 0;
+
+  for (const page of pages) {
+    try {
+      const text = `${page.title}\n\n${page.content.slice(0, 4000)}`;
+      const embedding = await getEmbedding(text, config);
+      store.saveEmbedding(page.id, embedding, "text-embedding-3-small");
+      count++;
+      if (count % 10 === 0) onProgress?.(`  ${count}/${pages.length} 완료`);
+    } catch (e) {
+      // Skip failed pages silently
+      onProgress?.(`  ⚠ ${page.title} 실패: ${e instanceof Error ? e.message : String(e)}`);
+    }
+  }
+
+  onProgress?.(`✅ ${count}개 임베딩 생성 완료`);
+  return count;
+}
+
+// Semantic search: find pages similar to query text
+export async function semanticSearch(
+  query: string,
+  store: Store,
+  config: LLMConfig,
+  limit: number = 5
+): Promise<Array<{slug: string; title: string; pageType: string; origin: string; similarity: number}>> {
+  const allEmbeddings = store.getAllEmbeddings();
+  if (!allEmbeddings.length) return [];
+
+  // Get query embedding
+  const queryEmbedding = await getEmbedding(query, config);
+
+  // Calculate similarities
+  const results = allEmbeddings.map(page => ({
+    slug: page.slug,
+    title: page.title,
+    pageType: page.pageType,
+    origin: page.origin,
+    similarity: cosineSimilarity(queryEmbedding, page.embedding)
+  }));
+
+  // Sort by similarity descending, return top N
+  return results
+    .sort((a, b) => b.similarity - a.similarity)
+    .slice(0, limit)
+    .filter(r => r.similarity > 0.3); // Minimum threshold
+}

package/src/services/index-generator.ts

@@ -0,0 +1,185 @@
+import type { Store } from "../store";
+import type { LLMConfig } from "../config";
+
+export interface IndexPage {
+  id: number;
+  title: string;
+  slug: string;
+  type: "source" | "concept";
+  linkCount: number;
+}
+
+export interface IndexCategory {
+  name: string;
+  slug: string;
+  description?: string;
+  pages: IndexPage[];
+}
+
+export interface ContentIndex {
+  categories: IndexCategory[];
+  totalPages: number;
+  totalLinks: number;
+  generatedAt: string;
+}
+
+interface IndexConfig {
+  useLLM?: boolean;
+  llmConfig?: LLMConfig;
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s가-힣-]/g, "")
+    .replace(/\s+/g, "-")
+    .replace(/-+/g, "-")
+    .trim();
+}
+
+/**
+ * Simple grouping: group pages by their source document, then orphan concepts separately.
+ */
+function groupBySource(store: Store): IndexCategory[] {
+  const grouped = store.getPagesBySource();
+  const categories: IndexCategory[] = [];
+
+  for (const group of grouped) {
+    categories.push({
+      name: group.sourceTitle,
+      slug: slugify(group.sourceTitle),
+      description: `${group.sourceTitle} 소스 문서에서 생성된 페이지`,
+      pages: group.pages.map((p) => ({
+        id: p.id,
+        title: p.title,
+        slug: p.slug,
+        type: p.page_type as "source" | "concept",
+        linkCount: p.linkCount,
+      })),
+    });
+  }
+
+  // Sort categories alphabetically
+  categories.sort((a, b) => a.name.localeCompare(b.name));
+
+  // Sort pages within each category by link count (descending)
+  for (const cat of categories) {
+    cat.pages.sort((a, b) => b.linkCount - a.linkCount);
+  }
+
+  return categories;
+}
+
+/**
+ * Smart grouping: use LLM to categorize pages into topic clusters.
+ */
+async function groupByLLM(store: Store, llmConfig: LLMConfig): Promise<IndexCategory[]> {
+  const { LLMClient } = await import("../llm-client");
+  const client = new LLMClient(llmConfig);
+
+  const pages = store.listPages();
+  const links = store.getAllLinks();
+
+  // Build link count map
+  const linkCountMap = new Map<number, number>();
+  for (const link of links) {
+    linkCountMap.set(link.to_page_id, (linkCountMap.get(link.to_page_id) || 0) + 1);
+    linkCountMap.set(link.from_page_id, (linkCountMap.get(link.from_page_id) || 0) + 1);
+  }
+
+  const pageTitles = pages.map((p) => `- ${p.title} (${p.page_type})`).join("\n");
+
+  const system = `You are a knowledge organizer. Given a list of wiki page titles, categorize them into 5-10 meaningful topic clusters. Return ONLY a JSON array of objects with: name (category name), description (short description), pages (array of page titles belonging to this category). Every page must be assigned to exactly one category.`;
+
+  const userMessage = `Categorize these wiki pages into topic clusters:\n\n${pageTitles}\n\nReturn JSON array only, no markdown fences.`;
+
+  try {
+    const response = await client.chatComplete(system, userMessage, 4096);
+
+    // Parse the JSON response
+    const cleaned = response.replace(/```json?\s*|```\s*/g, "").trim();
+    const clusters = JSON.parse(cleaned) as Array<{
+      name: string;
+      description?: string;
+      pages: string[];
+    }>;
+
+    // Build page lookup by title
+    const pageByTitle = new Map(pages.map((p) => [p.title, p]));
+
+    const categories: IndexCategory[] = clusters.map((cluster) => ({
+      name: cluster.name,
+      slug: slugify(cluster.name),
+      description: cluster.description,
+      pages: cluster.pages
+        .map((title) => {
+          const page = pageByTitle.get(title);
+          if (!page) return null;
+          return {
+            id: page.id,
+            title: page.title,
+            slug: page.slug,
+            type: page.page_type as "source" | "concept",
+            linkCount: linkCountMap.get(page.id) || 0,
+          };
+        })
+        .filter((p): p is IndexPage => p !== null),
+    }));
+
+    // Add any uncategorized pages
+    const categorizedTitles = new Set(clusters.flatMap((c) => c.pages));
+    const uncategorized = pages.filter((p) => !categorizedTitles.has(p.title));
+    if (uncategorized.length > 0) {
+      categories.push({
+        name: "기타",
+        slug: "etc",
+        description: "분류되지 않은 페이지",
+        pages: uncategorized.map((p) => ({
+          id: p.id,
+          title: p.title,
+          slug: p.slug,
+          type: p.page_type as "source" | "concept",
+          linkCount: linkCountMap.get(p.id) || 0,
+        })),
+      });
+    }
+
+    // Sort
+    categories.sort((a, b) => a.name.localeCompare(b.name));
+    for (const cat of categories) {
+      cat.pages.sort((a, b) => b.linkCount - a.linkCount);
+    }
+
+    return categories;
+  } catch {
+    // Fallback to simple grouping if LLM fails
+    console.warn("LLM categorization failed, falling back to source-based grouping");
+    return groupBySource(store);
+  }
+}
+
+/**
+ * Generate a structured content index for all wiki pages.
+ */
+export async function generateContentIndex(
+  store: Store,
+  config?: IndexConfig
+): Promise<ContentIndex> {
+  const totalPages = store.countPages();
+  const totalLinks = store.getAllLinks().length;
+
+  let categories: IndexCategory[];
+
+  if (config?.useLLM && config.llmConfig?.api_key) {
+    categories = await groupByLLM(store, config.llmConfig);
+  } else {
+    categories = groupBySource(store);
+  }
+
+  return {
+    categories,
+    totalPages,
+    totalLinks,
+    generatedAt: new Date().toISOString(),
+  };
+}

package/src/services/ingest.ts

@@ -1,5 +1,5 @@
 import { Store } from "../store";
-import { type LLMConfig, type Persona } from "../config";
+import { type LLMConfig, type Persona, type WikiSchema } from "../config";
 import { LLMClient, type UsageStats } from "../llm-client";
 
 export interface IngestResult {
@@ -15,10 +15,16 @@ export async function ingestUrl(
   url: string,
   llmConfig: LLMConfig,
   persona: Persona | null,
-  onProgress?: (status: string) => void
+  onProgress?: (status: string) => void,
+  schema?: WikiSchema
 ): Promise<IngestResult> {
   const client = new LLMClient(llmConfig);
   client.resetUsageStats();
+  client.onRetry = (attempt, max, delayMs) => {
+    const delaySec = Math.round(delayMs / 1000);
+    onProgress?.(`⏳ Rate limit — 재시도 ${attempt}/${max}, ${delaySec}초 대기...`);
+    console.log(`\x1b[33m⏳ Rate limit — retry ${attempt}/${max}, waiting ${delaySec}s...\x1b[0m`);
+  };
 
   const { fetchPage } = await import("../ingest/web");
   const { llmChunkDocument, htmlToRawText } = await import("../pipeline/llm-chunker");
@@ -27,15 +33,30 @@ export async function ingestUrl(
   const { title, html } = await fetchPage(url);
 
   const source = store.addSource(url, "web", title, html);
-  const rawText = htmlToRawText(html);
+  const rawText = await htmlToRawText(html);
+
+  if (!rawText || rawText.trim().length < 50) {
+    throw new Error("추출된 텍스트가 너무 짧습니다. 파일 내용을 확인해주세요.");
+  }
 
-  onProgress?.("⏳ LLM 분석 시작...");
-  const { sourceCount, conceptCount } = await llmChunkDocument(rawText, title, source.id, store, 0, persona, client);
+  // Only delete existing pages if NOT resuming (no checkpoints = fresh ingest)
+  if (!store.hasCheckpoints(source.id)) {
+    store.deletePagesBySource(source.id);
+  }
+
+  const isResume = store.hasCheckpoints(source.id);
+  onProgress?.(isResume ? "⏳ LLM 분석 재개..." : "⏳ LLM 분석 시작...");
+  const { sourceCount, conceptCount } = await llmChunkDocument(rawText, title, source.id, store, 0, persona, client, onProgress, schema);
+
+  // Pipeline completed successfully — clear checkpoints for clean future re-ingests
+  store.clearCheckpoints(source.id);
 
   const u = client.getUsageStats();
   const estimatedCostUsd = client.getEstimatedCost();
   store.addUsageLog(source.id, u.totalCalls, u.promptTokens, u.completionTokens, u.totalTokens, estimatedCostUsd);
 
+  store.addActivityLog('ingest', `Ingested ${title}`, 'source', source.id, { url, sourceCount, conceptCount });
+
   return {
     sourceCount,
     conceptCount,
@@ -51,10 +72,16 @@ export async function ingestFile(
   originalName: string,
   llmConfig: LLMConfig,
   persona: Persona | null,
-  onProgress?: (status: string) => void
+  onProgress?: (status: string) => void,
+  schema?: WikiSchema
 ): Promise<IngestResult> {
   const client = new LLMClient(llmConfig);
   client.resetUsageStats();
+  client.onRetry = (attempt, max, delayMs) => {
+    const delaySec = Math.round(delayMs / 1000);
+    onProgress?.(`⏳ Rate limit — 재시도 ${attempt}/${max}, ${delaySec}초 대기...`);
+    console.log(`\x1b[33m⏳ Rate limit — retry ${attempt}/${max}, waiting ${delaySec}s...\x1b[0m`);
+  };
 
   const { llmChunkDocument } = await import("../pipeline/llm-chunker");
 
@@ -63,34 +90,65 @@ export async function ingestFile(
   let title: string;
   let text: string;
 
-  if (ext === "pdf") {
-    const { extractTextFromPdf } = await import("../ingest/pdf");
-    onProgress?.("⏳ PDF 텍스트 추출 중...");
-    ({ title, text } = await extractTextFromPdf(filePath));
-  } else if (ext === "docx") {
-    const { extractTextFromDocx } = await import("../ingest/docx");
-    onProgress?.("⏳ DOCX 텍스트 추출 중...");
-    ({ title, text } = await extractTextFromDocx(filePath));
-  } else if (ext === "pptx") {
-    const { extractTextFromPptx } = await import("../ingest/pptx");
-    onProgress?.("⏳ PPTX 텍스트 추출 중...");
-    ({ title, text } = await extractTextFromPptx(filePath));
-  } else {
-    const { extractWithTextutil } = await import("../ingest/legacy");
-    onProgress?.(`⏳ ${ext.toUpperCase()} 텍스트 추출 중...`);
-    ({ title, text } = await extractWithTextutil(filePath));
+  switch (ext) {
+    case "pdf": {
+      const { extractTextFromPdf } = await import("../ingest/pdf");
+      onProgress?.("⏳ PDF 텍스트 추출 중...");
+      ({ title, text } = await extractTextFromPdf(filePath));
+      break;
+    }
+    case "docx": {
+      const { extractTextFromDocx } = await import("../ingest/docx");
+      onProgress?.("⏳ DOCX 텍스트 추출 중...");
+      ({ title, text } = await extractTextFromDocx(filePath));
+      break;
+    }
+    case "pptx": {
+      const { extractTextFromPptx } = await import("../ingest/pptx");
+      onProgress?.("⏳ PPTX 텍스트 추출 중...");
+      ({ title, text } = await extractTextFromPptx(filePath));
+      break;
+    }
+    case "md": {
+      const { extractTextFromMarkdown } = await import("../ingest/markdown");
+      onProgress?.("⏳ MD 텍스트 추출 중...");
+      const result = extractTextFromMarkdown(filePath);
+      title = result.title;
+      text = result.text;
+      break;
+    }
+    default: {
+      const { extractWithTextutil } = await import("../ingest/legacy");
+      onProgress?.(`⏳ ${ext.toUpperCase()} 텍스트 추출 중...`);
+      ({ title, text } = await extractWithTextutil(filePath));
+      break;
+    }
+  }
+
+  if (!text || text.trim().length < 50) {
+    throw new Error("추출된 텍스트가 너무 짧습니다. 파일 내용을 확인해주세요.");
   }
 
   const source = store.addSource(filePath, ext, title, "(file)");
-  store.deletePagesBySource(source.id);
 
-  onProgress?.("⏳ LLM 분석 시작...");
-  const { sourceCount, conceptCount } = await llmChunkDocument(text, title, source.id, store, 0, persona, client);
+  // Only delete existing pages if NOT resuming (no checkpoints = fresh ingest)
+  if (!store.hasCheckpoints(source.id)) {
+    store.deletePagesBySource(source.id);
+  }
+
+  const isResume = store.hasCheckpoints(source.id);
+  onProgress?.(isResume ? "⏳ LLM 분석 재개..." : "⏳ LLM 분석 시작...");
+  const { sourceCount, conceptCount } = await llmChunkDocument(text, title, source.id, store, 0, persona, client, onProgress, schema);
+
+  // Pipeline completed successfully — clear checkpoints for clean future re-ingests
+  store.clearCheckpoints(source.id);
 
   const u = client.getUsageStats();
   const estimatedCostUsd = client.getEstimatedCost();
   store.addUsageLog(source.id, u.totalCalls, u.promptTokens, u.completionTokens, u.totalTokens, estimatedCostUsd);
 
+  store.addActivityLog('ingest', `Ingested ${originalName}`, 'source', source.id, { filePath, sourceCount, conceptCount });
+
   return {
     sourceCount,
     conceptCount,