@open330/kiwimu 0.4.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/kiwimu +1 -1
- package/package.json +4 -1
- package/personas/namuwiki.json +6 -0
- package/src/build/renderer.ts +49 -2
- package/src/build/static/search.js +33 -2
- package/src/build/static/style.css +84 -1
- package/src/build/templates.ts +297 -167
- package/src/config.ts +35 -29
- package/src/demo/sample-data.ts +70 -0
- package/src/demo/setup.ts +31 -0
- package/src/expand/llm.ts +1 -1
- package/src/index.ts +208 -458
- package/src/ingest/docx.ts +0 -8
- package/src/ingest/legacy.ts +4 -4
- package/src/ingest/pdf.ts +1 -1
- package/src/ingest/pptx.ts +0 -1
- package/src/ingest/web.test.ts +41 -0
- package/src/ingest/web.ts +61 -62
- package/src/llm-client.ts +203 -126
- package/src/pipeline/chunker.test.ts +42 -0
- package/src/pipeline/chunker.ts +1 -48
- package/src/pipeline/llm-chunker.ts +133 -55
- package/src/server.ts +327 -0
- package/src/services/ingest.ts +100 -0
- package/src/store.test.ts +132 -0
- package/src/store.ts +102 -2
- package/src/pipeline/llm-linker.ts +0 -84
package/src/pipeline/chunker.ts
CHANGED
|
@@ -1,63 +1,16 @@
|
|
|
1
|
-
import TurndownService from "turndown";
|
|
2
|
-
import type { Section } from "../ingest/web";
|
|
3
|
-
import type { Store } from "../store";
|
|
4
|
-
|
|
5
|
-
const turndown = new TurndownService({ headingStyle: "atx" });
|
|
6
|
-
turndown.remove(["script", "style"]);
|
|
7
|
-
|
|
8
1
|
export function slugify(text: string): string {
|
|
9
2
|
return text
|
|
10
|
-
.normalize("NFKD")
|
|
11
3
|
.toLowerCase()
|
|
12
4
|
.trim()
|
|
13
|
-
.replace(/[^\w\s
|
|
5
|
+
.replace(/[^\w\s가-힣ㄱ-ㅎㅏ-ㅣ-]/g, "")
|
|
14
6
|
.replace(/[-\s]+/g, "-")
|
|
15
7
|
.replace(/^-|-$/g, "")
|
|
16
8
|
.slice(0, 80);
|
|
17
9
|
}
|
|
18
10
|
|
|
19
|
-
const STOP_TITLES = new Set([
|
|
20
|
-
"introduction", "overview", "summary", "conclusion", "references",
|
|
21
|
-
"bibliography", "appendix", "abstract", "preface", "contents",
|
|
22
|
-
"table of contents", "index", "acknowledgments", "notes",
|
|
23
|
-
]);
|
|
24
|
-
|
|
25
11
|
export function cleanTitle(title: string): string {
|
|
26
12
|
return title
|
|
27
13
|
.replace(/^\s*(Chapter\s+)?\d+(\.\d+)*\s*/i, "")
|
|
28
14
|
.replace(/\s+/g, " ")
|
|
29
15
|
.trim();
|
|
30
16
|
}
|
|
31
|
-
|
|
32
|
-
export function chunkSections(sections: Section[], sourceId: number, store: Store, minWords = 30): number {
|
|
33
|
-
let count = 0;
|
|
34
|
-
|
|
35
|
-
for (const section of sections) {
|
|
36
|
-
const title = cleanTitle(section.title);
|
|
37
|
-
if (!title) continue;
|
|
38
|
-
|
|
39
|
-
const slug = slugify(title);
|
|
40
|
-
if (!slug) continue;
|
|
41
|
-
|
|
42
|
-
const htmlContent = section.htmlParts.join("\n");
|
|
43
|
-
if (!htmlContent.trim()) continue;
|
|
44
|
-
|
|
45
|
-
const content = turndown.turndown(htmlContent).trim();
|
|
46
|
-
const wordCount = content.split(/\s+/).length;
|
|
47
|
-
|
|
48
|
-
if (wordCount < minWords) continue;
|
|
49
|
-
if (STOP_TITLES.has(slug) || STOP_TITLES.has(title.toLowerCase())) {
|
|
50
|
-
if (wordCount < 100) continue;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
const existing = store.getPage(slug);
|
|
54
|
-
if (existing) {
|
|
55
|
-
store.updatePageContent(existing.id, existing.content + "\n\n" + content);
|
|
56
|
-
} else {
|
|
57
|
-
store.addPage(slug, title, content, sourceId, slug);
|
|
58
|
-
count++;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
return count;
|
|
63
|
-
}
|
|
@@ -1,14 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { LLMClient } from "../llm-client";
|
|
2
2
|
import type { Store } from "../store";
|
|
3
3
|
import { slugify } from "./chunker";
|
|
4
4
|
import type { Persona } from "../config";
|
|
5
5
|
|
|
6
6
|
// ── Phase 1: Extract original document structure ──
|
|
7
7
|
|
|
8
|
-
const STRUCTURE_SYSTEM = `You are a document analyzer. Extract the chapter/section structure from this textbook content, preserving the original order and hierarchy.
|
|
9
|
-
|
|
10
|
-
Return valid JSON only. No markdown fences.`;
|
|
11
|
-
|
|
12
8
|
const STRUCTURE_PROMPT = `Extract the document structure from this text. Preserve the original chapter/section ordering.
|
|
13
9
|
|
|
14
10
|
Source: "{sourceTitle}"
|
|
@@ -89,6 +85,25 @@ interface ConceptPage {
|
|
|
89
85
|
suggested_links?: Array<{ text: string; url: string }>;
|
|
90
86
|
}
|
|
91
87
|
|
|
88
|
+
async function parallelMap<T, R>(
|
|
89
|
+
items: T[],
|
|
90
|
+
concurrency: number,
|
|
91
|
+
fn: (item: T, index: number) => Promise<R>
|
|
92
|
+
): Promise<R[]> {
|
|
93
|
+
const results: R[] = new Array(items.length);
|
|
94
|
+
let nextIndex = 0;
|
|
95
|
+
|
|
96
|
+
async function worker() {
|
|
97
|
+
while (nextIndex < items.length) {
|
|
98
|
+
const i = nextIndex++;
|
|
99
|
+
results[i] = await fn(items[i], i);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
|
|
104
|
+
return results;
|
|
105
|
+
}
|
|
106
|
+
|
|
92
107
|
function splitByChapters(text: string): Array<{ chapterHint: string; text: string }> {
|
|
93
108
|
const chapterPattern = /\n(?=(?:CHAPTER\s*\d+|Chapter\s+\d+)[A-Z\s])/g;
|
|
94
109
|
const positions: number[] = [];
|
|
@@ -199,8 +214,17 @@ export async function llmChunkDocument(
|
|
|
199
214
|
sourceId: number,
|
|
200
215
|
store: Store,
|
|
201
216
|
maxChunks: number = 0, // 0 = unlimited
|
|
202
|
-
persona: Persona | null = null
|
|
217
|
+
persona: Persona | null = null,
|
|
218
|
+
llmClient?: LLMClient
|
|
203
219
|
): Promise<{ sourceCount: number; conceptCount: number }> {
|
|
220
|
+
// Use provided client or fall back to deprecated global chatComplete
|
|
221
|
+
const chat = llmClient
|
|
222
|
+
? (system: string, user: string, maxTokens?: number) => llmClient.chatComplete(system, user, maxTokens)
|
|
223
|
+
: async (system: string, user: string, maxTokens?: number) => {
|
|
224
|
+
const { chatComplete } = await import("../llm-client");
|
|
225
|
+
return chatComplete(system, user, maxTokens);
|
|
226
|
+
};
|
|
227
|
+
|
|
204
228
|
let chunks = splitByChapters(rawText);
|
|
205
229
|
if (maxChunks > 0 && chunks.length > maxChunks) {
|
|
206
230
|
console.log(`\x1b[33m⚠ ${chunks.length}개 청크 중 ${maxChunks}개만 처리합니다\x1b[0m`);
|
|
@@ -211,61 +235,56 @@ export async function llmChunkDocument(
|
|
|
211
235
|
}
|
|
212
236
|
console.log(`\x1b[34m🧠 Phase 1: 원본 구조 추출 (${chunks.length}개 청크)...\x1b[0m`);
|
|
213
237
|
|
|
214
|
-
// ── Phase 1: Extract source pages ──
|
|
215
|
-
let
|
|
216
|
-
const
|
|
238
|
+
// ── Phase 1: Extract source pages (parallel LLM calls) ──
|
|
239
|
+
let completedCount = 0;
|
|
240
|
+
const structureSystem = getStructureSystem(persona);
|
|
217
241
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
console.log(` [${i + 1}/${chunks.length}] ${chunk.chapterHint}`);
|
|
242
|
+
const chunkResults = await parallelMap(chunks, 3, async (chunk, i) => {
|
|
243
|
+
console.log(` Phase 1: 처리 중 [${i + 1}/${chunks.length}] ${chunk.chapterHint}...`);
|
|
221
244
|
|
|
222
245
|
const prompt = STRUCTURE_PROMPT
|
|
223
246
|
.replace("{sourceTitle}", sourceTitle)
|
|
224
247
|
.replace("{text}", chunk.text.slice(0, 80000));
|
|
225
248
|
|
|
226
|
-
const structureSystem = getStructureSystem(persona);
|
|
227
249
|
try {
|
|
228
|
-
|
|
250
|
+
let raw = await chat(structureSystem, prompt, 16384);
|
|
229
251
|
if (!raw || raw.trim().length < 10) {
|
|
230
252
|
console.log(` \x1b[33m⚠ 빈 응답, 재시도...\x1b[0m`);
|
|
231
|
-
|
|
232
|
-
if (!
|
|
253
|
+
raw = await chat(structureSystem, prompt, 16384);
|
|
254
|
+
if (!raw || raw.trim().length < 10) {
|
|
233
255
|
console.log(` \x1b[31m✗ 재시도도 빈 응답\x1b[0m`);
|
|
234
|
-
|
|
256
|
+
completedCount++;
|
|
257
|
+
return [] as StructurePage[];
|
|
235
258
|
}
|
|
236
|
-
const sections = parseJSON<StructurePage[]>(retry).filter(s => s.title && s.content && s.content.length > 30);
|
|
237
|
-
// fall through to process sections below
|
|
238
|
-
for (const section of sections) {
|
|
239
|
-
const slug = slugify(section.title);
|
|
240
|
-
if (!slug) continue;
|
|
241
|
-
const existing = store.getPage(slug);
|
|
242
|
-
if (existing) {
|
|
243
|
-
store.updatePageContent(existing.id, existing.content + "\n\n" + section.content);
|
|
244
|
-
} else {
|
|
245
|
-
store.addPage(slug, section.title, section.content, sourceId, slug, "source", orderCounter++);
|
|
246
|
-
sourcePageSummaries.push(`- ${section.title}: ${section.content.slice(0, 150).replace(/\n/g, " ")}`);
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
console.log(` → ${sections.length}개 섹션`);
|
|
250
|
-
continue;
|
|
251
259
|
}
|
|
252
260
|
const sections = parseJSON<StructurePage[]>(raw).filter(s => s.title && s.content && s.content.length > 30);
|
|
261
|
+
completedCount++;
|
|
262
|
+
console.log(` → ${sections.length}개 섹션 (완료 ${completedCount}/${chunks.length})`);
|
|
263
|
+
return sections;
|
|
264
|
+
} catch (e: unknown) {
|
|
265
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
266
|
+
console.log(` \x1b[31m✗ 실패: ${message}\x1b[0m`);
|
|
267
|
+
completedCount++;
|
|
268
|
+
return [] as StructurePage[];
|
|
269
|
+
}
|
|
270
|
+
});
|
|
253
271
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
272
|
+
// Store results sequentially (SQLite writes must be sequential)
|
|
273
|
+
let orderCounter = 0;
|
|
274
|
+
const sourcePageSummaries: string[] = [];
|
|
257
275
|
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
276
|
+
for (const sections of chunkResults) {
|
|
277
|
+
for (const section of sections) {
|
|
278
|
+
const slug = slugify(section.title);
|
|
279
|
+
if (!slug) continue;
|
|
280
|
+
|
|
281
|
+
const existing = store.getPage(slug);
|
|
282
|
+
if (existing) {
|
|
283
|
+
store.updatePageContent(existing.id, existing.content + "\n\n" + section.content);
|
|
284
|
+
} else {
|
|
285
|
+
store.addPage(slug, section.title, section.content, sourceId, slug, "source", orderCounter++);
|
|
286
|
+
sourcePageSummaries.push(`- ${section.title}: ${section.content.slice(0, 150).replace(/\n/g, " ")}`);
|
|
265
287
|
}
|
|
266
|
-
console.log(` → ${sections.length}개 섹션`);
|
|
267
|
-
} catch (e: any) {
|
|
268
|
-
console.log(` \x1b[31m✗ 실패: ${e.message}\x1b[0m`);
|
|
269
288
|
}
|
|
270
289
|
}
|
|
271
290
|
|
|
@@ -292,7 +311,7 @@ export async function llmChunkDocument(
|
|
|
292
311
|
const conceptSystem = getConceptSystem(persona);
|
|
293
312
|
|
|
294
313
|
try {
|
|
295
|
-
const raw = await
|
|
314
|
+
const raw = await chat(conceptSystem, prompt, 16384);
|
|
296
315
|
const concepts = parseJSON<ConceptPage[]>(raw).filter(c => c.title && c.content && c.content.length > 50);
|
|
297
316
|
|
|
298
317
|
for (const concept of concepts) {
|
|
@@ -315,13 +334,67 @@ export async function llmChunkDocument(
|
|
|
315
334
|
conceptCount++;
|
|
316
335
|
}
|
|
317
336
|
console.log(` → ${concepts.length}개 개념`);
|
|
318
|
-
} catch (e:
|
|
319
|
-
|
|
337
|
+
} catch (e: unknown) {
|
|
338
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
339
|
+
console.log(` \x1b[31m✗ 실패: ${message}\x1b[0m`);
|
|
320
340
|
}
|
|
321
341
|
}
|
|
322
342
|
|
|
323
343
|
console.log(`\x1b[32m 📝 ${conceptCount}개 개념 페이지 생성 완료\x1b[0m`);
|
|
324
344
|
|
|
345
|
+
// ── Phase 2.5: Generate quizzes from concept pages ──
|
|
346
|
+
let quizCount = 0;
|
|
347
|
+
try {
|
|
348
|
+
const conceptPagesForQuiz = store.listConceptPages();
|
|
349
|
+
if (conceptPagesForQuiz.length > 0) {
|
|
350
|
+
console.log(`\x1b[34m🧠 Phase 2.5: 퀴즈 생성 (${conceptPagesForQuiz.length}개 개념 페이지)...\x1b[0m`);
|
|
351
|
+
|
|
352
|
+
const quizSystem = `You are a quiz generator for a study wiki. Generate quiz questions based on wiki content.
|
|
353
|
+
Return valid JSON only. No markdown fences.`;
|
|
354
|
+
|
|
355
|
+
await parallelMap(conceptPagesForQuiz, 3, async (page, i) => {
|
|
356
|
+
try {
|
|
357
|
+
const quizPrompt = `Based on this wiki content, generate 2-3 quiz questions in JSON format.
|
|
358
|
+
Types: "fill_blank" (빈칸 채우기), "ox" (OX 퀴즈 - true/false), "short_answer" (단답형)
|
|
359
|
+
|
|
360
|
+
Content title: ${page.title}
|
|
361
|
+
Content:
|
|
362
|
+
${page.content.slice(0, 3000)}
|
|
363
|
+
|
|
364
|
+
Respond with a JSON array only:
|
|
365
|
+
[{"question": "___은 양자역학에서 위치와 운동량을 동시에 측정할 수 없다는 원리이다.", "answer": "불확정성 원리", "type": "fill_blank"}]
|
|
366
|
+
|
|
367
|
+
Rules:
|
|
368
|
+
- For fill_blank: use ___ to mark the blank in the question
|
|
369
|
+
- For ox: question should be a statement, answer should be "O" or "X"
|
|
370
|
+
- For short_answer: question should be answerable in 1-3 words
|
|
371
|
+
- Questions should test understanding, not just recall
|
|
372
|
+
- Write questions in Korean when the content is in Korean`;
|
|
373
|
+
|
|
374
|
+
const raw = await chat(quizSystem, quizPrompt, 2048);
|
|
375
|
+
const quizzes = parseJSON<Array<{ question: string; answer: string; type: string }>>(raw);
|
|
376
|
+
|
|
377
|
+
for (const q of quizzes) {
|
|
378
|
+
if (q.question && q.answer && q.type) {
|
|
379
|
+
store.addQuiz(page.id, q.question, q.answer, q.type);
|
|
380
|
+
quizCount++;
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
} catch (e: unknown) {
|
|
384
|
+
// Quiz generation is non-critical; silently skip failures
|
|
385
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
386
|
+
console.log(` \x1b[33m⚠ 퀴즈 생성 실패 (${page.title}): ${message}\x1b[0m`);
|
|
387
|
+
}
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
console.log(`\x1b[32m 🧩 ${quizCount}개 퀴즈 생성 완료\x1b[0m`);
|
|
391
|
+
}
|
|
392
|
+
} catch (e: unknown) {
|
|
393
|
+
// Phase 2.5 is optional — don't block the pipeline
|
|
394
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
395
|
+
console.log(`\x1b[33m ⚠ 퀴즈 생성 단계 건너뜀: ${message}\x1b[0m`);
|
|
396
|
+
}
|
|
397
|
+
|
|
325
398
|
// ── Phase 3: Resolve wiki links + inject concept links into source pages ──
|
|
326
399
|
console.log(`\x1b[34m🔗 위키 링크 해석 중...\x1b[0m`);
|
|
327
400
|
const allPages = store.listPages();
|
|
@@ -347,31 +420,36 @@ export async function llmChunkDocument(
|
|
|
347
420
|
const srcPages = allPages.filter(p => p.page_type === "source");
|
|
348
421
|
|
|
349
422
|
// Build search terms: full title + key words from title (2+ words long)
|
|
350
|
-
const searchTerms: Array<{ term: string; concept: typeof conceptPages[0] }> = [];
|
|
423
|
+
const searchTerms: Array<{ term: string; concept: typeof conceptPages[0]; regex: RegExp | null }> = [];
|
|
351
424
|
for (const concept of conceptPages) {
|
|
352
|
-
searchTerms.push({ term: concept.title, concept });
|
|
425
|
+
searchTerms.push({ term: concept.title, concept, regex: null });
|
|
353
426
|
// Also try individual significant words from multi-word titles
|
|
354
427
|
const words = concept.title.split(/\s+/).filter(w => w.length >= 4 && !/^(and|the|for|with|from|into)$/i.test(w));
|
|
355
428
|
if (words.length >= 2) {
|
|
356
429
|
// Try pairs of consecutive words
|
|
357
430
|
for (let i = 0; i < words.length - 1; i++) {
|
|
358
|
-
searchTerms.push({ term: `${words[i]} ${words[i + 1]}`, concept });
|
|
431
|
+
searchTerms.push({ term: `${words[i]} ${words[i + 1]}`, concept, regex: null });
|
|
359
432
|
}
|
|
360
433
|
}
|
|
361
434
|
}
|
|
362
435
|
// Sort by term length descending for longest match first
|
|
363
436
|
searchTerms.sort((a, b) => b.term.length - a.term.length);
|
|
364
437
|
|
|
438
|
+
// Pre-compile RegExp objects outside the page loop
|
|
439
|
+
for (const entry of searchTerms) {
|
|
440
|
+
if (entry.term.length < 3) continue;
|
|
441
|
+
const escaped = entry.term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
442
|
+
entry.regex = new RegExp(`(?<!\\[)(?<![\\w/])(${escaped})(?![\\w])(?!\\])(?![^[]*\\])`, "i");
|
|
443
|
+
}
|
|
444
|
+
|
|
365
445
|
for (const srcPage of srcPages) {
|
|
366
446
|
let content = srcPage.content;
|
|
367
447
|
let modified = false;
|
|
368
448
|
const linkedConcepts = new Set<number>();
|
|
369
449
|
|
|
370
|
-
for (const { term, concept } of searchTerms) {
|
|
450
|
+
for (const { term, concept, regex } of searchTerms) {
|
|
371
451
|
if (linkedConcepts.has(concept.id)) continue; // One link per concept per page
|
|
372
|
-
if (term.length < 3) continue;
|
|
373
|
-
const escaped = term.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
374
|
-
const regex = new RegExp(`(?<!\\[)(?<![\\w/])(${escaped})(?![\\w])(?!\\])(?![^[]*\\])`, "i");
|
|
452
|
+
if (term.length < 3 || !regex) continue;
|
|
375
453
|
const match = regex.exec(content);
|
|
376
454
|
if (match) {
|
|
377
455
|
const replacement = `[${match[1]}](/wiki/${concept.slug})`;
|
package/src/server.ts
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
import { join } from "path";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import crypto from "crypto";
|
|
4
|
+
import { DB_FILE, loadConfig, saveConfig, getActivePersona } from "./config";
|
|
5
|
+
import { Store } from "./store";
|
|
6
|
+
import type { KiwiConfig } from "./config";
|
|
7
|
+
|
|
8
|
+
export function startServer(root: string, port: number, host: string): void {
|
|
9
|
+
const config = loadConfig(root);
|
|
10
|
+
const siteDir = join(root, config.build.output_dir);
|
|
11
|
+
|
|
12
|
+
let isProcessing = false;
|
|
13
|
+
let processingStatus = "";
|
|
14
|
+
|
|
15
|
+
const hostname = host;
|
|
16
|
+
const authToken = crypto.randomUUID();
|
|
17
|
+
console.log(`\x1b[32m🥝 Kiwi Mu 서버 시작!\x1b[0m`);
|
|
18
|
+
console.log(` http://${hostname === "0.0.0.0" ? "localhost" : hostname}:${port}`);
|
|
19
|
+
console.log(` 관리 페이지: http://${hostname === "0.0.0.0" ? "localhost" : hostname}:${port}/admin?token=${authToken}`);
|
|
20
|
+
console.log(` 인증 토큰: ${authToken}`);
|
|
21
|
+
if (hostname === "0.0.0.0") console.log(" 네트워크에 공개됨 (0.0.0.0)");
|
|
22
|
+
console.log(" 웹에서 문서 추가 가능합니다.\n");
|
|
23
|
+
|
|
24
|
+
Bun.serve({
|
|
25
|
+
port,
|
|
26
|
+
hostname,
|
|
27
|
+
async fetch(req) {
|
|
28
|
+
const url = new URL(req.url);
|
|
29
|
+
|
|
30
|
+
// ── Auth middleware for /api/* and /admin ──
|
|
31
|
+
if (url.pathname.startsWith("/api/") || url.pathname === "/admin") {
|
|
32
|
+
const authHeader = req.headers.get("Authorization");
|
|
33
|
+
const queryToken = url.searchParams.get("token");
|
|
34
|
+
const bearerToken = authHeader?.startsWith("Bearer ") ? authHeader.slice(7) : null;
|
|
35
|
+
if (bearerToken !== authToken && queryToken !== authToken) {
|
|
36
|
+
return Response.json({ error: "Unauthorized" }, { status: 401 });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// ── API endpoints ──
|
|
41
|
+
|
|
42
|
+
// File upload endpoint
|
|
43
|
+
if (url.pathname === "/api/upload" && req.method === "POST") {
|
|
44
|
+
if (isProcessing) {
|
|
45
|
+
return Response.json({ error: "이미 처리 중입니다", status: processingStatus }, { status: 409 });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const formData = await req.formData();
|
|
49
|
+
const file = formData.get("file") as File | null;
|
|
50
|
+
if (!file) {
|
|
51
|
+
return Response.json({ error: "파일이 필요합니다" }, { status: 400 });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const MAX_UPLOAD_SIZE = 50 * 1024 * 1024; // 50MB
|
|
55
|
+
if (file.size > MAX_UPLOAD_SIZE) {
|
|
56
|
+
return Response.json({ error: "파일 크기가 50MB를 초과합니다" }, { status: 413 });
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const ext = file.name.split(".").pop()?.toLowerCase() || "";
|
|
60
|
+
const supported = ["pdf", "docx", "doc", "pptx", "ppt", "key", "rtf"];
|
|
61
|
+
if (!supported.includes(ext)) {
|
|
62
|
+
return Response.json({ error: `지원하지 않는 형식: .${ext}. 지원: ${supported.join(", ")}` }, { status: 400 });
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Save uploaded file
|
|
66
|
+
const uploadDir = join(root, "uploads");
|
|
67
|
+
const { mkdirSync } = await import("fs");
|
|
68
|
+
mkdirSync(uploadDir, { recursive: true });
|
|
69
|
+
const filePath = join(uploadDir, path.basename(file.name));
|
|
70
|
+
await Bun.write(filePath, await file.arrayBuffer());
|
|
71
|
+
|
|
72
|
+
isProcessing = true;
|
|
73
|
+
processingStatus = "파일 처리 시작...";
|
|
74
|
+
|
|
75
|
+
(async () => {
|
|
76
|
+
const store = new Store(join(root, DB_FILE));
|
|
77
|
+
try {
|
|
78
|
+
const { ingestFile } = await import("./services/ingest");
|
|
79
|
+
const currentConfig = loadConfig(root);
|
|
80
|
+
const currentPersona = getActivePersona(currentConfig);
|
|
81
|
+
|
|
82
|
+
await ingestFile(root, store, filePath, file.name, currentConfig.llm, currentPersona, (status) => {
|
|
83
|
+
processingStatus = status;
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
processingStatus = "빌드 중...";
|
|
87
|
+
const { buildSite } = await import("./build/renderer");
|
|
88
|
+
await buildSite(store, loadConfig(root), root);
|
|
89
|
+
|
|
90
|
+
processingStatus = "완료!";
|
|
91
|
+
} catch (e: unknown) {
|
|
92
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
93
|
+
processingStatus = `오류: ${message}`;
|
|
94
|
+
} finally {
|
|
95
|
+
store.close();
|
|
96
|
+
setTimeout(() => { isProcessing = false; }, 2000);
|
|
97
|
+
}
|
|
98
|
+
})();
|
|
99
|
+
|
|
100
|
+
return Response.json({ ok: true, message: "파일 처리 시작" });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// URL add endpoint
|
|
104
|
+
if (url.pathname === "/api/add" && req.method === "POST") {
|
|
105
|
+
if (isProcessing) {
|
|
106
|
+
return Response.json({ error: "이미 처리 중입니다", status: processingStatus }, { status: 409 });
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const body = await req.json() as { source: string };
|
|
110
|
+
if (!body.source) {
|
|
111
|
+
return Response.json({ error: "source가 필요합니다" }, { status: 400 });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
const { validateUrl } = await import("./ingest/web");
|
|
116
|
+
validateUrl(body.source);
|
|
117
|
+
} catch (e: unknown) {
|
|
118
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
119
|
+
return Response.json({ error: message }, { status: 400 });
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
isProcessing = true;
|
|
123
|
+
processingStatus = "시작 중...";
|
|
124
|
+
|
|
125
|
+
(async () => {
|
|
126
|
+
const store = new Store(join(root, DB_FILE));
|
|
127
|
+
try {
|
|
128
|
+
const { ingestUrl } = await import("./services/ingest");
|
|
129
|
+
const currentConfig = loadConfig(root);
|
|
130
|
+
const currentPersona = getActivePersona(currentConfig);
|
|
131
|
+
|
|
132
|
+
await ingestUrl(root, store, body.source, currentConfig.llm, currentPersona, (status) => {
|
|
133
|
+
processingStatus = status;
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
processingStatus = "빌드 중...";
|
|
137
|
+
const { buildSite } = await import("./build/renderer");
|
|
138
|
+
await buildSite(store, loadConfig(root), root);
|
|
139
|
+
|
|
140
|
+
processingStatus = "완료!";
|
|
141
|
+
} catch (e: unknown) {
|
|
142
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
143
|
+
processingStatus = `오류: ${message}`;
|
|
144
|
+
} finally {
|
|
145
|
+
store.close();
|
|
146
|
+
setTimeout(() => { isProcessing = false; }, 2000);
|
|
147
|
+
}
|
|
148
|
+
})();
|
|
149
|
+
|
|
150
|
+
return Response.json({ ok: true, message: "처리 시작" });
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Admin API - update LLM settings
|
|
154
|
+
if (url.pathname === "/api/settings" && req.method === "POST") {
|
|
155
|
+
const body = await req.json() as Record<string, string | undefined>;
|
|
156
|
+
const currentConfig = loadConfig(root);
|
|
157
|
+
if (body.wiki_name) currentConfig.project.name = body.wiki_name;
|
|
158
|
+
if (body.provider) currentConfig.llm.provider = body.provider;
|
|
159
|
+
if (body.model) currentConfig.llm.model = body.model;
|
|
160
|
+
if (body.api_key !== undefined) currentConfig.llm.api_key = body.api_key ?? "";
|
|
161
|
+
if (body.endpoint !== undefined) currentConfig.llm.endpoint = body.endpoint ?? "";
|
|
162
|
+
saveConfig(root, currentConfig);
|
|
163
|
+
// Reload config for serve
|
|
164
|
+
Object.assign(config, currentConfig);
|
|
165
|
+
|
|
166
|
+
// Auto-rebuild site with new settings
|
|
167
|
+
(async () => {
|
|
168
|
+
const store = new Store(join(root, DB_FILE));
|
|
169
|
+
try {
|
|
170
|
+
const { buildSite } = await import("./build/renderer");
|
|
171
|
+
await buildSite(store, currentConfig, root);
|
|
172
|
+
console.log("\x1b[32m✅ 설정 변경 후 사이트 리빌드 완료\x1b[0m");
|
|
173
|
+
} catch (e: unknown) {
|
|
174
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
175
|
+
console.log(`\x1b[31m리빌드 실패: ${message}\x1b[0m`);
|
|
176
|
+
} finally {
|
|
177
|
+
store.close();
|
|
178
|
+
}
|
|
179
|
+
})();
|
|
180
|
+
|
|
181
|
+
return Response.json({ ok: true });
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (url.pathname === "/api/settings" && req.method === "GET") {
|
|
185
|
+
const currentConfig = loadConfig(root);
|
|
186
|
+
// Mask API key
|
|
187
|
+
const masked = { ...currentConfig.llm, api_key: currentConfig.llm.api_key ? "••••" + currentConfig.llm.api_key.slice(-4) : "" };
|
|
188
|
+
return Response.json(masked);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
// Persona API
|
|
192
|
+
if (url.pathname === "/api/personas" && req.method === "GET") {
|
|
193
|
+
const currentConfig = loadConfig(root);
|
|
194
|
+
return Response.json({
|
|
195
|
+
personas: currentConfig.personas || [],
|
|
196
|
+
active: currentConfig.active_persona || "",
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (url.pathname === "/api/personas" && req.method === "POST") {
|
|
201
|
+
const body = await req.json() as Record<string, unknown>;
|
|
202
|
+
const currentConfig = loadConfig(root);
|
|
203
|
+
if (!currentConfig.personas) currentConfig.personas = [];
|
|
204
|
+
|
|
205
|
+
if (body.action === "add") {
|
|
206
|
+
const persona = body.persona as { name: string; description?: string; system_prompt?: string; content_style?: string };
|
|
207
|
+
const { name, description, system_prompt, content_style } = persona;
|
|
208
|
+
if (!name) return Response.json({ error: "이름이 필요합니다" }, { status: 400 });
|
|
209
|
+
if (currentConfig.personas.find(p => p.name === name)) {
|
|
210
|
+
return Response.json({ error: "이미 존재하는 페르소나입니다" }, { status: 409 });
|
|
211
|
+
}
|
|
212
|
+
currentConfig.personas.push({ name, description: description || "", system_prompt: system_prompt || "", content_style: content_style || "" });
|
|
213
|
+
} else if (body.action === "update") {
|
|
214
|
+
const originalName = body.original_name as string;
|
|
215
|
+
const persona = body.persona as { name: string; description: string; system_prompt: string; content_style: string };
|
|
216
|
+
const idx = currentConfig.personas.findIndex(p => p.name === originalName);
|
|
217
|
+
if (idx === -1) return Response.json({ error: "페르소나를 찾을 수 없습니다" }, { status: 404 });
|
|
218
|
+
currentConfig.personas[idx] = persona;
|
|
219
|
+
if (currentConfig.active_persona === originalName && persona.name !== originalName) {
|
|
220
|
+
currentConfig.active_persona = persona.name;
|
|
221
|
+
}
|
|
222
|
+
} else if (body.action === "delete") {
|
|
223
|
+
const name = body.name as string;
|
|
224
|
+
currentConfig.personas = currentConfig.personas.filter(p => p.name !== name);
|
|
225
|
+
if (currentConfig.active_persona === name) {
|
|
226
|
+
currentConfig.active_persona = currentConfig.personas[0]?.name || "";
|
|
227
|
+
}
|
|
228
|
+
} else if (body.action === "activate") {
|
|
229
|
+
const name = body.name as string;
|
|
230
|
+
if (!currentConfig.personas.find(p => p.name === name)) {
|
|
231
|
+
return Response.json({ error: "페르소나를 찾을 수 없습니다" }, { status: 404 });
|
|
232
|
+
}
|
|
233
|
+
currentConfig.active_persona = name;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
saveConfig(root, currentConfig);
|
|
237
|
+
Object.assign(config, currentConfig);
|
|
238
|
+
return Response.json({ ok: true, personas: currentConfig.personas, active: currentConfig.active_persona });
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Build API
|
|
242
|
+
if (url.pathname === "/api/build" && req.method === "POST") {
|
|
243
|
+
if (isProcessing) {
|
|
244
|
+
return Response.json({ error: "이미 처리 중입니다" }, { status: 409 });
|
|
245
|
+
}
|
|
246
|
+
isProcessing = true;
|
|
247
|
+
processingStatus = "빌드 중...";
|
|
248
|
+
(async () => {
|
|
249
|
+
const store = new Store(join(root, DB_FILE));
|
|
250
|
+
try {
|
|
251
|
+
const { buildSite } = await import("./build/renderer");
|
|
252
|
+
await buildSite(store, loadConfig(root), root);
|
|
253
|
+
processingStatus = "빌드 완료!";
|
|
254
|
+
console.log("\x1b[32m✅ 수동 빌드 완료\x1b[0m");
|
|
255
|
+
} catch (e: unknown) {
|
|
256
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
257
|
+
processingStatus = `빌드 오류: ${message}`;
|
|
258
|
+
} finally {
|
|
259
|
+
store.close();
|
|
260
|
+
setTimeout(() => { isProcessing = false; }, 2000);
|
|
261
|
+
}
|
|
262
|
+
})();
|
|
263
|
+
return Response.json({ ok: true, message: "빌드 시작" });
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
// Admin page
|
|
267
|
+
if (url.pathname === "/admin") {
|
|
268
|
+
const store = new Store(join(root, DB_FILE));
|
|
269
|
+
const sources = store.listSourcesMeta();
|
|
270
|
+
const usage = store.getUsageSummary();
|
|
271
|
+
const configData = loadConfig(root);
|
|
272
|
+
store.close();
|
|
273
|
+
|
|
274
|
+
const { renderAdmin } = await import("./build/templates");
|
|
275
|
+
return new Response(renderAdmin({
|
|
276
|
+
wikiName: configData.project.name,
|
|
277
|
+
sources,
|
|
278
|
+
usage,
|
|
279
|
+
llmConfig: configData.llm,
|
|
280
|
+
personas: configData.personas || [],
|
|
281
|
+
activePersona: configData.active_persona || "",
|
|
282
|
+
authToken,
|
|
283
|
+
}), { headers: { "Content-Type": "text/html" } });
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
if (url.pathname === "/api/status") {
|
|
287
|
+
const store = new Store(join(root, DB_FILE));
|
|
288
|
+
const sources = store.listSourcesMeta();
|
|
289
|
+
const sourcePages = store.listSourcePages();
|
|
290
|
+
const conceptPages = store.listConceptPages();
|
|
291
|
+
const links = store.getAllLinks();
|
|
292
|
+
const usage = store.getUsageSummary();
|
|
293
|
+
store.close();
|
|
294
|
+
|
|
295
|
+
return Response.json({
|
|
296
|
+
processing: isProcessing,
|
|
297
|
+
processingStatus,
|
|
298
|
+
sources: sources.length,
|
|
299
|
+
sourcePages: sourcePages.length,
|
|
300
|
+
conceptPages: conceptPages.length,
|
|
301
|
+
links: links.length,
|
|
302
|
+
usage,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// ── Static file serving ──
|
|
307
|
+
let pathname = url.pathname;
|
|
308
|
+
if (pathname === "/") pathname = "/index.html";
|
|
309
|
+
|
|
310
|
+
const resolved = path.resolve(join(siteDir, pathname));
|
|
311
|
+
if (!resolved.startsWith(path.resolve(siteDir))) {
|
|
312
|
+
return new Response("Forbidden", { status: 403 });
|
|
313
|
+
}
|
|
314
|
+
const staticFile = Bun.file(resolved);
|
|
315
|
+
|
|
316
|
+
if (await staticFile.exists()) {
|
|
317
|
+
const isHtml = pathname.endsWith(".html");
|
|
318
|
+
const cspValue = "default-src 'self'; script-src 'self' 'unsafe-inline' cdn.jsdelivr.net d3js.org; style-src 'self' 'unsafe-inline' cdn.jsdelivr.net fonts.googleapis.com; font-src fonts.gstatic.com; img-src * data:; connect-src 'self'";
|
|
319
|
+
if (isHtml) {
|
|
320
|
+
return new Response(staticFile, { headers: { "Content-Type": "text/html", "Content-Security-Policy": cspValue } });
|
|
321
|
+
}
|
|
322
|
+
return new Response(staticFile);
|
|
323
|
+
}
|
|
324
|
+
return new Response("Not Found", { status: 404 });
|
|
325
|
+
},
|
|
326
|
+
});
|
|
327
|
+
}
|