@chnxthr/knowledge-mcp 0.0.0-stage → 0.3.0

package/dist/plugins/drive/chunker.js

@@ -0,0 +1,356 @@
+/**
+ * Markdown chunking — heading 기반 분할 + token-aware sub-split + overlap.
+ *
+ * 전략:
+ * 1. Front-matter (---...---) 분리 → metadata 추출 (검색 payload 승격)
+ * 2. heading (^#+ ) 위치마다 split → section
+ * 3. 각 section 이 MAX_TOKENS 넘으면 paragraph (\n\n) 단위로 sub-split
+ *    - 표(|...|)·코드블록(```) 은 fence-aware (행/블록 중간에서 안 잘림)
+ * 4. MIN_TOKENS 미만은 다음 청크와 merge
+ * 5. 청크 시작에 OVERLAP_TOKENS 만큼 이전 청크 끝 prepend
+ * 6. 이미지 markdown(![alt](url)) → URL 은 image_urls 로 분리, content 엔 alt 만 (임베딩 노이즈 방지)
+ */
+import { encode } from 'gpt-tokenizer';
+import { parse as parseYaml } from 'yaml';
+const MAX_TOKENS = 800;
+const MIN_TOKENS = 50;
+const OVERLAP_TOKENS = 100;
+// 단일 unit(문단/문장)의 절대 상한 — 초과 시 강제 분할. 임베딩 API 입력 초과(Bad Request) 방지.
+const HARD_MAX_TOKENS = MAX_TOKENS;
+const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n+/;
+export function parseFrontmatter(markdown) {
+    const m = markdown.match(FRONTMATTER_RE);
+    if (!m)
+        return { frontmatter: {}, body: markdown };
+    // 정식 YAML 파싱 — multi-line list(related/source_refs/affects)·inline 배열·따옴표를 문법대로.
+    // (구 단일라인 정규식은 multi-line list 를 빈 값으로 버려 related 의 ~89% 유실 → yaml 파서로 교체)
+    let fm = {};
+    try {
+        const parsed = parseYaml(m[1]);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            fm = parsed;
+        }
+    }
+    catch {
+        // 깨진 YAML frontmatter — 본문은 보존, 메타만 빈값 (fail-safe: 한 문서가 색인 전체를 멈추지 않게)
+        fm = {};
+    }
+    return { frontmatter: fm, body: markdown.slice(m[0].length) };
+}
+/** unknown → 정규화 문자열 (Date·number·bool 방어). 빈 문자열은 null. */
+function asString(v) {
+    if (v === null || v === undefined)
+        return null;
+    if (typeof v === 'string')
+        return v.trim() || null;
+    if (typeof v === 'number' || typeof v === 'boolean')
+        return String(v);
+    if (v instanceof Date)
+        return v.toISOString().slice(0, 10);
+    return null;
+}
+/** unknown → string[]. yaml 배열 그대로 / "[a,b]"·"a,b" 문자열 / 단일값 모두 흡수. */
+function asArray(v) {
+    if (v === null || v === undefined)
+        return [];
+    if (Array.isArray(v))
+        return v.map((x) => String(x).trim()).filter(Boolean);
+    if (typeof v === 'string') {
+        const s = v.trim().replace(/^\[|\]$/g, '');
+        if (!s)
+            return [];
+        return s
+            .split(',')
+            .map((x) => x.trim().replace(/^["']|["']$/g, ''))
+            .filter(Boolean);
+    }
+    return [];
+}
+/** frontmatter(yaml 파싱 결과) → 검색 payload 메타. */
+function parseDocMeta(fm) {
+    return {
+        doc_status: asString(fm.status),
+        doc_tier: asString(fm.tier),
+        doc_date: asString(fm.date),
+        doc_authors: asArray(fm.authors),
+        doc_related: asArray(fm.related),
+        doc_tags: asArray(fm.tags),
+        doc_source_refs: asArray(fm.source_refs),
+        doc_affects: asArray(fm.affects),
+        doc_products: asArray(fm.products),
+        doc_type: asString(fm.type),
+        doc_confidence: asString(fm.confidence),
+    };
+}
+function countTokens(text) {
+    // gpt-tokenizer 는 cl100k_base (text-embedding-3 호환) 기본
+    return encode(text).length;
+}
+// lone surrogate (짝 안 맞는 UTF-16 surrogate) 제거 — 토큰 경계 cut 이 이모지(surrogate pair)를
+// 중간에 쪼개 반쪽만 남으면 임베딩 API 가 invalid 로 400 Bad Request.
+const LONE_SURROGATE_RE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g;
+function stripLoneSurrogates(text) {
+    return text.replace(LONE_SURROGATE_RE, '');
+}
+// ![alt](url) — 이미지 markdown. URL 은 image_urls 로 분리, content 엔 alt 만 (영문 URL 토큰이
+// 한국어 임베딩 벡터를 흔드는 것 방지). alt 가 비면 자리 제거.
+const IMAGE_RE = /!\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/g;
+function extractImages(content) {
+    const urls = [];
+    const cleaned = content.replace(IMAGE_RE, (_full, alt, url) => {
+        urls.push(url);
+        return alt ? alt : '';
+    });
+    return { content: cleaned, image_urls: urls };
+}
+/** 청크 내 markdown table 개수 (separator 행 `|---|` 수로 근사). */
+function countTables(content) {
+    return (content.match(/^\s*\|[\s:|-]+\|\s*$/gm) ?? []).length;
+}
+/** markdown table block 인지 (행 70%+ 가 |...| 형태). */
+function isTableBlock(text) {
+    const lines = text.trim().split('\n');
+    if (lines.length < 2)
+        return false;
+    const pipeLines = lines.filter((l) => /^\s*\|.*\|\s*$/.test(l)).length;
+    return pipeLines >= lines.length * 0.7;
+}
+/** 큰 표를 행 단위로 분할 + 각 조각에 header(첫 행 + separator) prepend (컬럼명 보존). */
+function splitTable(table, maxTokens) {
+    const lines = table.trim().split('\n');
+    const sepIdx = lines.findIndex((l) => /^\s*\|[\s:|-]+\|\s*$/.test(l));
+    const headerEnd = sepIdx >= 0 ? sepIdx + 1 : 1;
+    const header = lines.slice(0, headerEnd).join('\n');
+    const bodyRows = lines.slice(headerEnd);
+    const headerTok = countTokens(header);
+    const out = [];
+    let buf = [header];
+    let bufTok = headerTok;
+    for (const row of bodyRows) {
+        const rt = countTokens(row);
+        if (bufTok + rt > maxTokens && buf.length > 1) {
+            out.push(buf.join('\n'));
+            buf = [header, row];
+            bufTok = headerTok + rt;
+        }
+        else {
+            buf.push(row);
+            bufTok += rt;
+        }
+    }
+    if (buf.length > 1)
+        out.push(buf.join('\n'));
+    return out.length > 0 ? out : [table];
+}
+/** 코드블록(```) 보호하며 \n\n paragraph split. 코드블록은 통째 1 unit. */
+function splitParagraphs(text) {
+    const out = [];
+    for (const seg of text.split(/(```[\s\S]*?```)/g)) {
+        if (seg.startsWith('```')) {
+            if (seg.trim())
+                out.push(seg.trim());
+        }
+        else {
+            for (const p of seg.split(/\n\n+/)) {
+                if (p.trim())
+                    out.push(p.trim());
+            }
+        }
+    }
+    return out;
+}
+/**
+ * 한국어·영어 문장 분리 (보수적). 종결부호/개행 기준. 분리 실패 시 원문 1개 (손실 X).
+ */
+function splitSentences(text) {
+    const parts = text
+        .split(/(?<=[.!?。…])\s+|(?<=[가-힣][다요죠음임함됨까네]\.)\s+|(?<=\n)/)
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+    return parts.length > 0 ? parts : [text];
+}
+/**
+ * 문자 단위 hard cut (문장조차 상한 초과 시 최후 수단). 이진탐색으로 maxTokens 이하 최대 prefix.
+ */
+function hardCutByChars(text, maxTokens) {
+    if (countTokens(text) <= maxTokens)
+        return [text];
+    const out = [];
+    let rest = text;
+    while (rest.length > 0) {
+        if (countTokens(rest) <= maxTokens) {
+            out.push(rest);
+            break;
+        }
+        let lo = 1;
+        let hi = rest.length;
+        let cut = 1;
+        while (lo <= hi) {
+            const mid = (lo + hi) >> 1;
+            if (countTokens(rest.slice(0, mid)) <= maxTokens) {
+                cut = mid;
+                lo = mid + 1;
+            }
+            else {
+                hi = mid - 1;
+            }
+        }
+        out.push(rest.slice(0, cut));
+        rest = rest.slice(cut);
+    }
+    return out;
+}
+/** text 의 마지막 maxTokens 토큰에 해당하는 suffix (overlap 용, 이진탐색). */
+function tailByTokens(text, maxTokens) {
+    if (countTokens(text) <= maxTokens)
+        return text;
+    let lo = 0;
+    let hi = text.length - 1;
+    let cut = text.length - 1;
+    while (lo <= hi) {
+        const mid = (lo + hi) >> 1;
+        if (countTokens(text.slice(mid)) <= maxTokens) {
+            cut = mid;
+            hi = mid - 1;
+        }
+        else {
+            lo = mid + 1;
+        }
+    }
+    return text.slice(cut);
+}
+/**
+ * MAX 초과 unit 강제 분할. 표면 행 단위(header 반복), 아니면 문장 budget → 문자 hard cut.
+ * 모든 반환 unit 은 maxTokens 이하 보장.
+ */
+function splitLongText(text, maxTokens) {
+    if (countTokens(text) <= maxTokens)
+        return [text];
+    // 표는 행 단위 분리 (각 조각 header prepend)
+    if (isTableBlock(text))
+        return splitTable(text, maxTokens);
+    const out = [];
+    let buf = [];
+    let bufTokens = 0;
+    const flush = () => {
+        if (buf.length > 0) {
+            out.push(buf.join(' '));
+            buf = [];
+            bufTokens = 0;
+        }
+    };
+    for (const sentence of splitSentences(text)) {
+        const st = countTokens(sentence);
+        if (st > maxTokens) {
+            flush();
+            out.push(...hardCutByChars(sentence, maxTokens));
+            continue;
+        }
+        if (bufTokens + st > maxTokens && buf.length > 0)
+            flush();
+        buf.push(sentence);
+        bufTokens += st;
+    }
+    flush();
+    return out;
+}
+/** heading line 의 레벨(#개수)+텍스트 추출 — breadcrumb(상위 > 하위) 구성용. */
+function extractHeading(text) {
+    const m = text.match(/^(#+)\s+(.+)$/m);
+    return m ? { level: m[1].length, text: m[2].trim() } : null;
+}
+/**
+ * 전체 markdown → 청크 array. heading context + 이미지/표/코드/frontmatter 메타 보존.
+ */
+export function chunkMarkdown(markdown) {
+    const { frontmatter, body } = parseFrontmatter(markdown);
+    const docMeta = parseDocMeta(frontmatter);
+    if (body.trim().length === 0)
+        return [];
+    const sections = body.split(/\n(?=#+ )/);
+    const chunks = [];
+    const headingStack = []; // index = (heading level - 1) → breadcrumb 상위>하위 맥락 보존
+    let currentHeading = '문서 시작';
+    for (const section of sections) {
+        if (section.trim().length === 0)
+            continue;
+        const h = extractHeading(section);
+        if (h) {
+            headingStack.length = h.level - 1; // 같거나 더 깊은 레벨 잘라내고
+            headingStack[h.level - 1] = h.text; // 현재 레벨 갱신
+            const crumb = headingStack.filter(Boolean).join(' > ');
+            if (crumb)
+                currentHeading = crumb;
+        }
+        const tokens = countTokens(section);
+        if (tokens <= MAX_TOKENS) {
+            chunks.push({ heading: currentHeading, content: section.trim(), tokens });
+            continue;
+        }
+        const paragraphs = splitParagraphs(section).flatMap((p) => countTokens(p) > HARD_MAX_TOKENS ? splitLongText(p, HARD_MAX_TOKENS) : [p]);
+        let buf = [];
+        let bufTokens = 0;
+        for (const para of paragraphs) {
+            const pt = countTokens(para);
+            if (bufTokens + pt > MAX_TOKENS && buf.length > 0) {
+                chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
+                buf = [];
+                bufTokens = 0;
+            }
+            buf.push(para);
+            bufTokens += pt;
+        }
+        if (buf.length > 0) {
+            chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
+        }
+    }
+    // MIN_TOKENS 미만 청크 merge
+    const merged = [];
+    for (const c of chunks) {
+        if (c.tokens < MIN_TOKENS && merged.length > 0) {
+            const prev = merged[merged.length - 1];
+            prev.content = `${prev.content}\n\n${c.content}`;
+            prev.tokens += c.tokens;
+        }
+        else {
+            merged.push({ ...c });
+        }
+    }
+    // 1. 이미지 URL 분리 + surrogate strip + 메타 계산 — overlap *전* 고유 content 기준.
+    //    (overlap 후 계산하면 이전 청크의 이미지/코드가 인접 청크 메타에 오염됨)
+    const withMeta = merged.map((c) => {
+        const tableCount = countTables(c.content);
+        const hasCode = /```/.test(c.content);
+        const { content: imgStripped, image_urls } = extractImages(c.content);
+        return {
+            heading: c.heading,
+            content: stripLoneSurrogates(imgStripped),
+            image_urls,
+            table_count: tableCount,
+            has_code_block: hasCode,
+        };
+    });
+    // 2. Overlap prepend — 이미지 URL 이 이미 분리된 텍스트 suffix (overlap 에 URL 안 섞임).
+    //    ⚠️ tailByTokens 는 토큰 경계로 자르므로 이모지(surrogate pair)를 반토막 낼 수 있다.
+    //    이 lone surrogate 가 Qdrant payload content 에 들어가면 invalid UTF-8 JSON → upsert 400.
+    //    → 아래 3단계에서 최종 content 에 strip 을 한 번 더 적용해 제거한다.
+    for (let i = 1; i < withMeta.length; i++) {
+        const overlap = tailByTokens(withMeta[i - 1].content, OVERLAP_TOKENS);
+        if (overlap)
+            withMeta[i].content = `…${overlap}\n\n${withMeta[i].content}`;
+    }
+    // 3. 최종 RawChunk — overlap 후 재생성된 lone surrogate 재제거(Qdrant 400 방지) + frontmatter 동봉.
+    return withMeta.map((c) => {
+        const content = stripLoneSurrogates(c.content);
+        return {
+            heading: c.heading,
+            content,
+            tokens: countTokens(content),
+            image_urls: c.image_urls,
+            table_count: c.table_count,
+            has_code_block: c.has_code_block,
+            ...docMeta,
+        };
+    });
+}
+//# sourceMappingURL=chunker.js.map

package/dist/plugins/drive/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/plugins/drive/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,qEAAqE;AACrE,MAAM,eAAe,GAAG,UAAU,CAAC;AA8BnC,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAElD,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACzC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAEnD,iFAAiF;IACjF,2EAA2E;IAC3E,IAAI,EAAE,GAA4B,EAAE,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,EAAE,GAAG,MAAiC,CAAC;QACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;QACxE,EAAE,GAAG,EAAE,CAAC;IACV,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAChE,CAAC;AAgBD,4DAA4D;AAC5D,SAAS,QAAQ,CAAC,CAAU;IAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,YAAY,IAAI;QAAE,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AACvE,SAAS,OAAO,CAAC,CAAU;IACzB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClB,OAAO,CAAC;aACL,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,+CAA+C;AAC/C,SAAS,YAAY,CAAC,EAA2B;IAC/C,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC;QAC/B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;QAC1B,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC;QACxC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;QAClC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,uDAAuD;IACvD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,iFAAiF;AACjF,sDAAsD;AACtD,MAAM,iBAAiB,GAAG,yEAAyE,CAAC;AACpG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,yCAAyC;AACzC,MAAM,QAAQ,GAAG,wCAAwC,CAAC;AAC1D,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,GAAW,EAAE,EAAE;QAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED,0DAA0D;AAC1D,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;AAChE,CAAC;AAED,mDAAmD;AACnD,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,OAAO,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;AACzC,CAAC;AAED,sEAAsE;AACtE,SAAS,UAAU,CAAC,KAAa,EAAE,SAAiB;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,SAAS,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACzB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACpB,MAAM,GAAG,SAAS,GAAG,EAAE,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClD,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,CAAC,IAAI,EAAE;oBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,oDAAoD,CAAC;SAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,SAAiB;IACrD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,MAAM;QACR,CAAC;QACD,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjD,GAAG,GAAG,GAAG,CAAC;gBACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8DAA8D;AAC9D,SAAS,YAAY,CAAC,IAAY,EAAE,SAAiB;IACnD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1B,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9C,GAAG,GAAG,GAAG,CAAC;YACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,SAAiB;IACpD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,mCAAmC;IACnC,IAAI,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAE3D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,EAAE,GAAG,SAAS,EAAE,CAAC;YACnB,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QACD,IAAI,SAAS,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,EAAE,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnB,SAAS,IAAI,EAAE,CAAC;IAClB,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+DAA+D;AAC/D,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC;AAQD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC,CAAC,uDAAuD;IAC1F,IAAI,cAAc,GAAG,OAAO,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,EAAE,CAAC;YACN,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,mBAAmB;YACtD,YAAY,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW;YAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvD,IAAI,KAAK;gBAAE,cAAc,GAAG,KAAK,CAAC;QACpC,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,SAAS,GAAG,EAAE,GAAG,UAAU,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC9F,GAAG,GAAG,EAAE,CAAC;gBACT,SAAS,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,CAAC,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,mDAAmD;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtE,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,mBAAmB,CAAC,WAAW,CAAC;YACzC,UAAU;YACV,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,OAAO;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,oEAAoE;IACpE,uFAAuF;IACvF,qDAAqD;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,OAAO;YAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,OAAO,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7E,CAAC;IAED,sFAAsF;IACtF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO;YACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC;YAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,GAAG,OAAO;SACX,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/plugins/drive/clean_text.d.ts

@@ -0,0 +1,13 @@
+/**
+ * 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
+ *
+ * 보수적 규칙만 (의미 손상 방지):
+ *   - 이미지 markdown ![](url)        → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
+ *   - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
+ *   - HTML 태그 (<div>, <br>, <span>...)  → 텍스트 보존하고 태그만 제거
+ *   - pdf-parse 페이지 마커 (-- N of M --) → 제거
+ *   - 과도한 빈 줄·줄끝 공백          → 정규화
+ *
+ * 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
+ */
+export declare function cleanExtractedText(text: string): string;

package/dist/plugins/drive/clean_text.js

@@ -0,0 +1,31 @@
+/**
+ * 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
+ *
+ * 보수적 규칙만 (의미 손상 방지):
+ *   - 이미지 markdown ![](url)        → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
+ *   - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
+ *   - HTML 태그 (<div>, <br>, <span>...)  → 텍스트 보존하고 태그만 제거
+ *   - pdf-parse 페이지 마커 (-- N of M --) → 제거
+ *   - 과도한 빈 줄·줄끝 공백          → 정규화
+ *
+ * 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
+ */
+export function cleanExtractedText(text) {
+    if (!text)
+        return '';
+    let t = text;
+    // 1. 이미지 markdown(![alt](url)) 은 제거하지 않음 — chunker 가 image_urls 로 분리 보존 (2026-05-27)
+    // 2. LaTeX wrapper — \title{X} \section{X} \subsection{X} → X
+    t = t.replace(/\\(?:title|section|subsection|textbf|textit)\s*\{([^}]*)\}/g, '$1');
+    // 3. HTML <br> → 개행, 나머지 태그는 텍스트 보존하고 태그만 제거
+    t = t.replace(/<br\s*\/?>/gi, '\n');
+    t = t.replace(/<\/?[a-zA-Z][^>]*>/g, '');
+    // 4. pdf-parse 페이지 마커 (-- 1 of 22 --) 제거
+    t = t.replace(/^\s*--\s*\d+\s+of\s+\d+\s*--\s*$/gim, '');
+    // 5. HTML entity 일부 복원
+    t = t.replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
+    // 6. 줄끝 공백 + 과도한 빈 줄 정규화
+    t = t.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n');
+    return t.trim();
+}
+//# sourceMappingURL=clean_text.js.map

package/dist/plugins/drive/clean_text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"clean_text.js","sourceRoot":"","sources":["../../../src/plugins/drive/clean_text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,CAAC,GAAG,IAAI,CAAC;IAEb,qFAAqF;IAErF,8DAA8D;IAC9D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,6DAA6D,EAAE,IAAI,CAAC,CAAC;IAEnF,6CAA6C;IAC7C,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAEzC,yCAAyC;IACzC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC;IAEzD,uBAAuB;IACvB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEjG,yBAAyB;IACzB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE5D,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC"}

package/dist/plugins/drive/client.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Google Drive API wrapper.
+ *
+ * 인증 옵션 (우선순위 순):
+ *   1. Service Account JSON (권장 — production-grade)
+ *      - GOOGLE_SERVICE_ACCOUNT_KEY (inline JSON, Railway 친화)
+ *      - GOOGLE_APPLICATION_CREDENTIALS (JSON 파일 path)
+ *
+ *   2. OAuth Refresh Token (fallback, single-user)
+ *      - GOOGLE_OAUTH_CLIENT_ID + GOOGLE_OAUTH_CLIENT_SECRET + GOOGLE_OAUTH_REFRESH_TOKEN
+ *      - Legacy DRIVE_* fallback
+ *
+ * Service Account 사용 시:
+ *   - Drive 폴더에 service account email (xxx@<project>.iam.gserviceaccount.com) viewer 공유 필수
+ *   - refresh_token 만료·rotation 없음
+ *   - server-to-server auth (production standard)
+ */
+import type { drive_v3 } from 'googleapis';
+/**
+ * googleapis 는 불러오는 데만 CPU 1초 이상 걸린다(가장 무거운 의존성). 그래서 처음 쓸 때 불러온다 —
+ * Drive 를 안 쓰는 서버와 `knowledge-mcp keys` 같은 명령이 그만큼 빨리 뜬다.
+ */
+export declare function getDriveClient(): Promise<drive_v3.Drive>;
+/** Google Drive query language 의 single-quote escape. */
+export declare function escapeDriveQuery(s: string): string;

package/dist/plugins/drive/client.js

@@ -0,0 +1,62 @@
+let _client = null;
+const DRIVE_READONLY_SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
+// Drive API 전역 timeout — googleapis 는 명시 timeout 이 없으면 무한에 가깝게 hang 할 수 있어
+// (search/read/list·classifyAndPath 부모 walk가 매달리면 stateless transport 에서 도구 호출
+// 전체가 멈춤). google.options 로 전역 15s 부여. internal-mcp 는 google API 를 drive 만 쓴다.
+const DRIVE_TIMEOUT_MS = Number(process.env.DRIVE_TIMEOUT_MS ?? 15_000);
+/**
+ * googleapis 는 불러오는 데만 CPU 1초 이상 걸린다(가장 무거운 의존성). 그래서 처음 쓸 때 불러온다 —
+ * Drive 를 안 쓰는 서버와 `knowledge-mcp keys` 같은 명령이 그만큼 빨리 뜬다.
+ */
+export async function getDriveClient() {
+    if (_client)
+        return _client;
+    const { google } = await import('googleapis');
+    const { GoogleAuth } = await import('google-auth-library');
+    google.options({ timeout: DRIVE_TIMEOUT_MS });
+    // ── Option 1: Service Account (권장) ────────────────────
+    const serviceAccountJson = process.env.GOOGLE_SERVICE_ACCOUNT_KEY;
+    const credentialsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+    if (serviceAccountJson) {
+        // Inline JSON (Railway env)
+        let parsed;
+        try {
+            parsed = JSON.parse(serviceAccountJson);
+        }
+        catch (e) {
+            throw new Error('[drive] GOOGLE_SERVICE_ACCOUNT_KEY is not valid JSON: ' +
+                (e instanceof Error ? e.message : String(e)));
+        }
+        const auth = new GoogleAuth({
+            credentials: parsed,
+            scopes: [DRIVE_READONLY_SCOPE],
+        });
+        _client = google.drive({ version: 'v3', auth });
+        return _client;
+    }
+    if (credentialsPath) {
+        // GoogleAuth 가 자동 GOOGLE_APPLICATION_CREDENTIALS env 인식
+        const auth = new GoogleAuth({ scopes: [DRIVE_READONLY_SCOPE] });
+        _client = google.drive({ version: 'v3', auth });
+        return _client;
+    }
+    // ── Option 2: OAuth Refresh Token (fallback) ────────────
+    const clientId = process.env.GOOGLE_OAUTH_CLIENT_ID || process.env.DRIVE_CLIENT_ID;
+    const clientSecret = process.env.GOOGLE_OAUTH_CLIENT_SECRET || process.env.DRIVE_CLIENT_SECRET;
+    const refreshToken = process.env.GOOGLE_OAUTH_REFRESH_TOKEN || process.env.DRIVE_REFRESH_TOKEN;
+    if (clientId && clientSecret && refreshToken) {
+        const oauth = new google.auth.OAuth2(clientId, clientSecret);
+        oauth.setCredentials({ refresh_token: refreshToken });
+        _client = google.drive({ version: 'v3', auth: oauth });
+        return _client;
+    }
+    throw new Error('[drive] Drive credentials missing. Use one of:\n' +
+        '  (1) GOOGLE_SERVICE_ACCOUNT_KEY (inline JSON) ← 권장\n' +
+        '  (2) GOOGLE_APPLICATION_CREDENTIALS (JSON file path) ← 권장\n' +
+        '  (3) GOOGLE_OAUTH_CLIENT_ID + _SECRET + _REFRESH_TOKEN ← fallback');
+}
+/** Google Drive query language 의 single-quote escape. */
+export function escapeDriveQuery(s) {
+    return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+}
+//# sourceMappingURL=client.js.map

package/dist/plugins/drive/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../../../src/plugins/drive/client.ts"],"names":[],"mappings":"AAmBA,IAAI,OAAO,GAA0B,IAAI,CAAC;AAE1C,MAAM,oBAAoB,GAAG,gDAAgD,CAAC;AAE9E,2EAA2E;AAC3E,+EAA+E;AAC/E,+EAA+E;AAC/E,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,MAAM,CAAC,CAAC;AAExE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc;IAClC,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9C,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;IAC3D,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAE9C,yDAAyD;IACzD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IAClE,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;IAEnE,IAAI,kBAAkB,EAAE,CAAC;QACvB,4BAA4B;QAC5B,IAAI,MAAM,CAAC;QACX,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QAC1C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,wDAAwD;gBACtD,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC/C,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC;YAC1B,WAAW,EAAE,MAAM;YACnB,MAAM,EAAE,CAAC,oBAAoB,CAAC;SAC/B,CAAC,CAAC;QACH,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACpB,wDAAwD;QACxD,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,2DAA2D;IAC3D,MAAM,QAAQ,GACZ,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IACpE,MAAM,YAAY,GAChB,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC5E,MAAM,YAAY,GAChB,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAE5E,IAAI,QAAQ,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAC7D,KAAK,CAAC,cAAc,CAAC,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,CAAC;QACtD,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACvD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,IAAI,KAAK,CACb,kDAAkD;QAChD,uDAAuD;QACvD,8DAA8D;QAC9D,oEAAoE,CACvE,CAAC;AACJ,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,gBAAgB,CAAC,CAAS;IACxC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACvD,CAAC"}

package/dist/plugins/drive/embeddings.d.ts

@@ -0,0 +1,15 @@
+export declare const EMBEDDING_MODEL = "text-embedding-3-small";
+export declare const EMBEDDING_DIM = 1536;
+interface EmbedPolicy {
+    maxRetries: number;
+    timeoutMs: number;
+}
+export type EmbedTaskType = 'RETRIEVAL_DOCUMENT' | 'RETRIEVAL_QUERY' | 'SEMANTIC_SIMILARITY' | 'CLASSIFICATION' | 'CLUSTERING';
+/**
+ * 여러 텍스트 → embedding 벡터들 (input 순서 보장).
+ * BATCH_SIZE 씩 묶어 batch 호출. policy 기본 = 색인용(patient). 쿼리는 embedQuery 가 fail-fast 사용.
+ */
+export declare function embedTexts(texts: string[], _taskType?: EmbedTaskType, policy?: EmbedPolicy): Promise<number[][]>;
+/** 단일 query → 벡터. 인터랙티브 검색 경로라 fail-fast 정책(QUERY_POLICY) 사용. */
+export declare function embedQuery(query: string): Promise<number[]>;
+export {};

package/dist/plugins/drive/embeddings.js

@@ -0,0 +1,118 @@
+/**
+ * Embedding 헬퍼 — OpenAI `text-embedding-3-small` (2026-05-28 비용 전환).
+ *
+ * 기존 Gemini `gemini-embedding-2` → OpenAI 3-small 전환 이유:
+ *   - 비용 ~1/10 ($0.02/1M vs Gemini ~$0.20/1M)
+ *   - 429 rate limit 여유 (Gemini free tier 한도 잦은 429 → drive-sync 에러 로그)
+ *   - backend 가 이미 3-small 사용(한국어 KG 임베딩 검증됨)
+ *   - 1536 dim 동일 → Qdrant collection 차원 유지 (벡터 공간만 다름 → 전환 시 전체 재색인 필수)
+ *   - input 한도 8191 토큰 → Gemini 대비 큰 청크 Bad Request 완화
+ *
+ * 호출 패턴:
+ *   OpenAI embeddings API 는 `input` 배열 batch 지원 → 한 호출에 여러 청크 (Gemini individual call 대비 빠르고 rate limit ↓).
+ *   429/5xx → exponential backoff + jitter retry. timeout 방어(무응답 hang).
+ */
+const OPENAI_EMBEDDINGS_URL = 'https://api.openai.com/v1/embeddings';
+export const EMBEDDING_MODEL = 'text-embedding-3-small';
+export const EMBEDDING_DIM = 1536;
+// 한 batch 당 input 수 (OpenAI 는 배열 input 지원). 토큰 한도 여유 위해 보수적 128.
+const BATCH_SIZE = Number(process.env.EMBED_BATCH ?? 128);
+const INDEX_POLICY = {
+    maxRetries: Number(process.env.EMBED_INDEX_RETRIES ?? 5),
+    timeoutMs: Number(process.env.EMBED_TIMEOUT_MS ?? 10_000),
+};
+const QUERY_POLICY = {
+    maxRetries: Number(process.env.EMBED_QUERY_RETRIES ?? 2),
+    timeoutMs: Number(process.env.EMBED_QUERY_TIMEOUT_MS ?? 5_000),
+};
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+function withTimeout(p, ms, label) {
+    let t;
+    const timer = new Promise((_, reject) => {
+        t = setTimeout(() => reject(new Error(`${label}: 504 deadline (timeout ${ms}ms)`)), ms);
+    });
+    return Promise.race([p, timer]).finally(() => clearTimeout(t));
+}
+function getApiKey() {
+    const key = process.env.OPENAI_API_KEY;
+    if (!key)
+        throw new Error('[embeddings] OPENAI_API_KEY env missing');
+    return key;
+}
+/** 재시도 가능 여부 (429 rate limit 또는 5xx/timeout). */
+function parseRetryable(status, msg) {
+    const is429 = status === 429 || /rate limit|quota|too many requests/i.test(msg);
+    const is5xx = (typeof status === 'number' && status >= 500 && status <= 599) ||
+        /\b(500|502|503|504)\b|timeout|deadline|ECONNRESET|ETIMEDOUT|fetch failed/i.test(msg);
+    return is429 || is5xx;
+}
+// lone surrogate 방어 — 깨진 이모지 반쪽이 들어오면 400. 빈 입력도 400 → 공백 fallback.
+function sanitize(text) {
+    const s = text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, '');
+    return s.length > 0 ? s : ' ';
+}
+/** input 배열 → 임베딩 배열 (batch 1회 호출 + retry). 순서 보장. policy 로 재시도·timeout 경로별 분리. */
+async function embedBatch(texts, policy) {
+    const apiKey = getApiKey();
+    const inputs = texts.map(sanitize);
+    for (let attempt = 0;; attempt++) {
+        try {
+            const resp = await withTimeout(fetch(OPENAI_EMBEDDINGS_URL, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
+                body: JSON.stringify({ model: EMBEDDING_MODEL, input: inputs, dimensions: EMBEDDING_DIM }),
+            }), policy.timeoutMs, '[embeddings] openai');
+            if (!resp.ok) {
+                const body = await resp.text();
+                const retryable = parseRetryable(resp.status, body);
+                if (!retryable || attempt >= policy.maxRetries) {
+                    throw new Error(`OpenAI embeddings ${resp.status}: ${body.slice(0, 200)}`);
+                }
+                const m = body.match(/try again in ([\d.]+)s/i);
+                const base = m ? Math.ceil(parseFloat(m[1]) * 1000) : Math.min(60_000, 1000 * 2 ** attempt);
+                const wait = Math.round(base + base * 0.3 * Math.random());
+                console.error(`[embeddings] ${resp.status} retry ${attempt + 1}/${policy.maxRetries} — ${wait}ms 대기`);
+                await sleep(wait);
+                continue;
+            }
+            const j = (await resp.json());
+            // index 순 정렬 후 벡터만 (OpenAI 는 보통 정렬돼 오지만 방어)
+            const sorted = [...j.data].sort((a, b) => a.index - b.index);
+            const vecs = sorted.map((d) => d.embedding);
+            if (vecs.length !== inputs.length || vecs[0]?.length !== EMBEDDING_DIM) {
+                throw new Error(`[embeddings] 응답 불일치 — vecs=${vecs.length}/${inputs.length}, dim=${vecs[0]?.length}/${EMBEDDING_DIM}`);
+            }
+            return vecs;
+        }
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (attempt >= policy.maxRetries || !parseRetryable(null, msg))
+                throw err;
+            const base = Math.min(60_000, 1000 * 2 ** attempt);
+            const wait = Math.round(base + base * 0.3 * Math.random());
+            console.error(`[embeddings] 예외 retry ${attempt + 1}/${policy.maxRetries} — ${wait}ms 대기 (${msg.slice(0, 80)})`);
+            await sleep(wait);
+        }
+    }
+}
+/**
+ * 여러 텍스트 → embedding 벡터들 (input 순서 보장).
+ * BATCH_SIZE 씩 묶어 batch 호출. policy 기본 = 색인용(patient). 쿼리는 embedQuery 가 fail-fast 사용.
+ */
+export async function embedTexts(texts, _taskType = 'RETRIEVAL_DOCUMENT', policy = INDEX_POLICY) {
+    if (texts.length === 0)
+        return [];
+    const results = [];
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+        const slice = texts.slice(i, i + BATCH_SIZE);
+        const embeds = await embedBatch(slice, policy);
+        results.push(...embeds);
+    }
+    return results;
+}
+/** 단일 query → 벡터. 인터랙티브 검색 경로라 fail-fast 정책(QUERY_POLICY) 사용. */
+export async function embedQuery(query) {
+    const [vec] = await embedTexts([query], 'RETRIEVAL_QUERY', QUERY_POLICY);
+    return vec;
+}
+//# sourceMappingURL=embeddings.js.map