@ninebix/nmt-system 1.0.0

package/dist/core/language-analyzers.js

@@ -0,0 +1,365 @@
+/**
+ * Language-specific Analyzers
+ * 언어별 형태소 분석 및 청킹 최적화
+ *
+ * @module core/language-analyzers
+ */
+/**
+ * Korean Analyzer
+ * 한국어 형태소 분석기
+ */
+export class KoreanAnalyzer {
+    language = 'ko';
+    // 한국어 조사 목록
+    particles = new Set([
+        '은', '는', '이', '가', '을', '를', '에', '에서', '에게', '께',
+        '의', '로', '으로', '와', '과', '랑', '이랑', '도', '만', '부터',
+        '까지', '보다', '처럼', '같이', '마다', '밖에', '뿐', '조차'
+    ]);
+    // 한국어 어미 목록
+    suffixes = new Set([
+        '다', '니다', '습니다', '세요', '요', '네', '죠', '지', '고',
+        '면', '어서', '아서', '니까', '는데', '던', '었', '았', '겠'
+    ]);
+    // 문장 종결 패턴
+    sentenceEnders = /[.!?。！？]\s*/g;
+    tokenize(text) {
+        const tokens = [];
+        const words = text.split(/\s+/);
+        for (const word of words) {
+            if (!word)
+                continue;
+            // 조사 분리
+            let remaining = word;
+            let foundParticle = false;
+            for (const particle of this.particles) {
+                if (remaining.endsWith(particle) && remaining.length > particle.length) {
+                    const stem = remaining.slice(0, -particle.length);
+                    tokens.push({ text: stem, type: 'word' });
+                    tokens.push({ text: particle, type: 'particle' });
+                    foundParticle = true;
+                    break;
+                }
+            }
+            if (!foundParticle) {
+                // 어미 분리 시도
+                let foundSuffix = false;
+                for (const suffix of this.suffixes) {
+                    if (remaining.endsWith(suffix) && remaining.length > suffix.length) {
+                        const stem = remaining.slice(0, -suffix.length);
+                        tokens.push({ text: stem, type: 'word' });
+                        tokens.push({ text: suffix, type: 'suffix' });
+                        foundSuffix = true;
+                        break;
+                    }
+                }
+                if (!foundSuffix) {
+                    tokens.push({ text: remaining, type: 'word' });
+                }
+            }
+        }
+        return tokens;
+    }
+    splitSentences(text) {
+        return text.split(this.sentenceEnders).filter(s => s.trim().length > 0);
+    }
+    splitPhrases(text) {
+        // 쉼표, 접속사로 분리
+        return text.split(/[,，、]|\s+(그리고|그러나|하지만|그래서|또한)\s+/g)
+            .filter(p => p && p.trim().length > 0);
+    }
+    splitWords(text) {
+        return text.split(/\s+/).filter(w => w.trim().length > 0);
+    }
+    splitMorphemes(text) {
+        return this.tokenize(text).map(t => t.text);
+    }
+}
+/**
+ * English Analyzer
+ * 영어 형태소 분석기
+ */
+export class EnglishAnalyzer {
+    language = 'en';
+    // Common English suffixes
+    suffixes = ['ing', 'ed', 'ly', 'tion', 'sion', 'ness', 'ment', 'able', 'ible'];
+    // Stop words
+    stopWords = new Set([
+        'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+        'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+        'should', 'may', 'might', 'must', 'shall', 'can', 'of', 'at', 'by',
+        'for', 'with', 'about', 'against', 'between', 'into', 'through',
+        'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up',
+        'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',
+        'then', 'once', 'and', 'but', 'or', 'nor', 'so', 'yet'
+    ]);
+    tokenize(text) {
+        const tokens = [];
+        const words = text.toLowerCase().split(/\s+/);
+        for (const word of words) {
+            if (!word)
+                continue;
+            // Remove punctuation
+            const cleanWord = word.replace(/[^\w]/g, '');
+            if (this.stopWords.has(cleanWord)) {
+                tokens.push({ text: cleanWord, type: 'particle' });
+            }
+            else {
+                // Try to find suffix
+                let foundSuffix = false;
+                for (const suffix of this.suffixes) {
+                    if (cleanWord.endsWith(suffix) && cleanWord.length > suffix.length + 2) {
+                        const stem = cleanWord.slice(0, -suffix.length);
+                        tokens.push({ text: stem, type: 'word', lemma: stem });
+                        tokens.push({ text: suffix, type: 'suffix' });
+                        foundSuffix = true;
+                        break;
+                    }
+                }
+                if (!foundSuffix) {
+                    tokens.push({ text: cleanWord, type: 'word' });
+                }
+            }
+        }
+        return tokens;
+    }
+    splitSentences(text) {
+        return text.split(/(?<=[.!?])\s+/).filter(s => s.trim().length > 0);
+    }
+    splitPhrases(text) {
+        return text.split(/[,;:]|\s+(and|but|or|because|although|however)\s+/gi)
+            .filter(p => p && p.trim().length > 0 && p.length > 3);
+    }
+    splitWords(text) {
+        return text.split(/\s+/).filter(w => w.trim().length > 0);
+    }
+    splitMorphemes(text) {
+        return this.tokenize(text).map(t => t.text);
+    }
+}
+/**
+ * Japanese Analyzer
+ * 일본어 형태소 분석기
+ */
+export class JapaneseAnalyzer {
+    language = 'ja';
+    // Unicode ranges
+    hiraganaRange = /[\u3040-\u309F]/;
+    katakanaRange = /[\u30A0-\u30FF]/;
+    kanjiRange = /[\u4E00-\u9FAF]/;
+    tokenize(text) {
+        const tokens = [];
+        let currentToken = '';
+        let currentType = 'word';
+        for (const char of text) {
+            let charType = 'word';
+            if (this.hiraganaRange.test(char)) {
+                charType = 'hiragana';
+            }
+            else if (this.katakanaRange.test(char)) {
+                charType = 'katakana';
+            }
+            else if (this.kanjiRange.test(char)) {
+                charType = 'kanji';
+            }
+            else if (/\s/.test(char)) {
+                charType = 'whitespace';
+            }
+            else if (/[。！？、]/.test(char)) {
+                charType = 'punctuation';
+            }
+            if (charType !== currentType && currentToken) {
+                tokens.push({ text: currentToken, type: currentType });
+                currentToken = '';
+            }
+            if (charType !== 'whitespace') {
+                currentToken += char;
+            }
+            currentType = charType;
+        }
+        if (currentToken) {
+            tokens.push({ text: currentToken, type: currentType });
+        }
+        return tokens;
+    }
+    splitSentences(text) {
+        return text.split(/[。！？]\s*/).filter(s => s.trim().length > 0);
+    }
+    splitPhrases(text) {
+        return text.split(/[、，]\s*/).filter(p => p.trim().length > 0);
+    }
+    splitWords(text) {
+        // Japanese doesn't use spaces, so we split by character type changes
+        const tokens = this.tokenize(text);
+        return tokens.filter(t => t.type !== 'punctuation').map(t => t.text);
+    }
+    splitMorphemes(text) {
+        return this.tokenize(text).map(t => t.text);
+    }
+}
+/**
+ * Chinese Analyzer
+ * 중국어 형태소 분석기
+ */
+export class ChineseAnalyzer {
+    language = 'zh';
+    // Common Chinese particles/function words
+    particles = new Set([
+        '的', '了', '是', '在', '有', '和', '与', '或', '但', '而',
+        '就', '也', '都', '还', '又', '才', '只', '已', '把', '被',
+        '让', '给', '向', '从', '到', '对', '为', '以', '因', '所'
+    ]);
+    tokenize(text) {
+        const tokens = [];
+        for (const char of text) {
+            if (/\s/.test(char))
+                continue;
+            if (/[。！？，、]/.test(char)) {
+                tokens.push({ text: char, type: 'punctuation' });
+            }
+            else if (this.particles.has(char)) {
+                tokens.push({ text: char, type: 'particle' });
+            }
+            else {
+                tokens.push({ text: char, type: 'hanzi' });
+            }
+        }
+        return tokens;
+    }
+    splitSentences(text) {
+        return text.split(/[。！？]\s*/).filter(s => s.trim().length > 0);
+    }
+    splitPhrases(text) {
+        return text.split(/[，、；]\s*/).filter(p => p.trim().length > 0);
+    }
+    splitWords(text) {
+        // Basic bigram approach for Chinese
+        const chars = [...text].filter(c => !/\s/.test(c) && !/[。！？，、；]/.test(c));
+        const words = [];
+        for (let i = 0; i < chars.length; i += 2) {
+            if (i + 1 < chars.length) {
+                words.push(chars[i] + chars[i + 1]);
+            }
+            else {
+                words.push(chars[i]);
+            }
+        }
+        return words;
+    }
+    splitMorphemes(text) {
+        // Each character is a morpheme in Chinese
+        return [...text].filter(c => !/\s/.test(c));
+    }
+}
+/**
+ * Auto-detect language and return appropriate analyzer
+ */
+export function detectLanguage(text) {
+    // Count character types
+    let korean = 0, japanese = 0, chinese = 0, english = 0, total = 0;
+    for (const char of text) {
+        if (/\s/.test(char))
+            continue;
+        total++;
+        // Korean Hangul
+        if (/[\uAC00-\uD7AF\u1100-\u11FF\u3130-\u318F]/.test(char)) {
+            korean++;
+        }
+        // Japanese (Hiragana, Katakana)
+        else if (/[\u3040-\u309F\u30A0-\u30FF]/.test(char)) {
+            japanese++;
+        }
+        // CJK Unified Ideographs (shared by Chinese/Japanese/Korean)
+        else if (/[\u4E00-\u9FAF]/.test(char)) {
+            chinese++; // Assume Chinese if no kana
+        }
+        // Latin
+        else if (/[a-zA-Z]/.test(char)) {
+            english++;
+        }
+    }
+    if (total === 0)
+        return 'en';
+    const koRatio = korean / total;
+    const jaRatio = japanese / total;
+    const zhRatio = chinese / total;
+    const enRatio = english / total;
+    // Determine language
+    if (koRatio > 0.3)
+        return 'ko';
+    if (jaRatio > 0.1)
+        return 'ja'; // Even small amount of kana indicates Japanese
+    if (zhRatio > 0.5)
+        return 'zh';
+    if (enRatio > 0.5)
+        return 'en';
+    // Default fallback
+    if (koRatio > jaRatio && koRatio > zhRatio)
+        return 'ko';
+    if (jaRatio > zhRatio)
+        return 'ja';
+    if (zhRatio > 0)
+        return 'zh';
+    return 'en';
+}
+/**
+ * Get analyzer for language
+ */
+export function getAnalyzer(language, text) {
+    let lang = language;
+    if (language === 'auto' && text) {
+        lang = detectLanguage(text);
+    }
+    switch (lang) {
+        case 'ko':
+            return new KoreanAnalyzer();
+        case 'ja':
+            return new JapaneseAnalyzer();
+        case 'zh':
+            return new ChineseAnalyzer();
+        case 'en':
+        default:
+            return new EnglishAnalyzer();
+    }
+}
+/**
+ * Multi-language tokenizer
+ */
+export class MultiLanguageTokenizer {
+    analyzers = new Map();
+    constructor() {
+        this.analyzers.set('ko', new KoreanAnalyzer());
+        this.analyzers.set('en', new EnglishAnalyzer());
+        this.analyzers.set('ja', new JapaneseAnalyzer());
+        this.analyzers.set('zh', new ChineseAnalyzer());
+    }
+    /**
+     * Tokenize with auto language detection
+     */
+    tokenize(text) {
+        const language = detectLanguage(text);
+        const analyzer = this.analyzers.get(language) || new EnglishAnalyzer();
+        return {
+            language,
+            tokens: analyzer.tokenize(text)
+        };
+    }
+    /**
+     * Split into all levels
+     */
+    analyze(text) {
+        const language = detectLanguage(text);
+        const analyzer = this.analyzers.get(language) || new EnglishAnalyzer();
+        return {
+            language,
+            sentences: analyzer.splitSentences(text),
+            phrases: analyzer.splitPhrases(text),
+            words: analyzer.splitWords(text),
+            morphemes: analyzer.splitMorphemes(text),
+            tokens: analyzer.tokenize(text)
+        };
+    }
+}
+// Export singleton
+export const multiLangTokenizer = new MultiLanguageTokenizer();
+//# sourceMappingURL=language-analyzers.js.map

package/dist/core/language-analyzers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"language-analyzers.js","sourceRoot":"","sources":["../../src/core/language-analyzers.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA0CH;;;GAGG;AACH,MAAM,OAAO,cAAc;IACzB,QAAQ,GAAa,IAAI,CAAC;IAE1B,YAAY;IACJ,SAAS,GAAG,IAAI,GAAG,CAAC;QAC1B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG;QAClD,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI;QACnD,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI;KAC9C,CAAC,CAAC;IAEH,YAAY;IACJ,QAAQ,GAAG,IAAI,GAAG,CAAC;QACzB,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;QAC/C,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;KAChD,CAAC,CAAC;IAEH,WAAW;IACH,cAAc,GAAG,cAAc,CAAC;IAExC,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEhC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,QAAQ;YACR,IAAI,SAAS,GAAG,IAAI,CAAC;YACrB,IAAI,aAAa,GAAG,KAAK,CAAC;YAE1B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtC,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACvE,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAClD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;oBAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;oBAClD,aAAa,GAAG,IAAI,CAAC;oBACrB,MAAM;gBACR,CAAC;YACH,CAAC;YAED,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,WAAW;gBACX,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;wBACnE,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;wBAC1C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;wBAC9C,WAAW,GAAG,IAAI,CAAC;wBACnB,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,YAAY,CAAC,IAAY;QACvB,cAAc;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,mCAAmC,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,eAAe;IAC1B,QAAQ,GAAa,IAAI,CAAC;IAE1B,0BAA0B;IAClB,QAAQ,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAEvF,aAAa;IACL,SAAS,GAAG,IAAI,GAAG,CAAC;QAC1B,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;QACnE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;QACnE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI;QAClE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS;QAC/D,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI;QACjE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS;QACrE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK;KACvD,CAAC,CAAC;IAEH,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,qBAAqB;YACrB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE7C,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACN,qBAAqB;gBACrB,IAAI,WAAW,GAAG,KAAK,CAAC;gBACxB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACnC,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACvE,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBAChD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;wBACvD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;wBAC9C,WAAW,GAAG,IAAI,CAAC;wBACnB,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,WAAW,EAAE,CAAC;oBACjB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,YAAY,CAAC,IAAY;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,qDAAqD,CAAC;aACrE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,gBAAgB;IAC3B,QAAQ,GAAa,IAAI,CAAC;IAE1B,iBAAiB;IACT,aAAa,GAAG,iBAAiB,CAAC;IAClC,aAAa,GAAG,iBAAiB,CAAC;IAClC,UAAU,GAAG,iBAAiB,CAAC;IAEvC,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,IAAI,YAAY,GAAG,EAAE,CAAC;QACtB,IAAI,WAAW,GAAc,MAAM,CAAC;QAEpC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,QAAQ,GAAc,MAAM,CAAC;YAEjC,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,QAAQ,GAAG,UAAU,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzC,QAAQ,GAAG,UAAU,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,QAAQ,GAAG,YAAY,CAAC;YAC1B,CAAC;iBAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/B,QAAQ,GAAG,aAAa,CAAC;YAC3B,CAAC;YAED,IAAI,QAAQ,KAAK,WAAW,IAAI,YAAY,EAAE,CAAC;gBAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBACvD,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YAED,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;gBAC9B,YAAY,IAAI,IAAI,CAAC;YACvB,CAAC;YACD,WAAW,GAAG,QAAQ,CAAC;QACzB,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,YAAY,CAAC,IAAY;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,qEAAqE;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvE,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,eAAe;IAC1B,QAAQ,GAAa,IAAI,CAAC;IAE1B,0CAA0C;IAClC,SAAS,GAAG,IAAI,GAAG,CAAC;QAC1B,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;QAChD,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;QAChD,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;KACjD,CAAC,CAAC;IAEH,QAAQ,CAAC,IAAY;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC9B,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,YAAY,CAAC,IAAY;QACvB,OAAO,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC;IAED,UAAU,CAAC,IAAY;QACrB,oCAAoC;QACpC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAa,EAAE,CAAC;QAE3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,cAAc,CAAC,IAAY;QACzB,0CAA0C;QAC1C,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,wBAAwB;IACxB,IAAI,MAAM,GAAG,CAAC,EAAE,QAAQ,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC;IAElE,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9B,KAAK,EAAE,CAAC;QAER,gBAAgB;QAChB,IAAI,2CAA2C,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3D,MAAM,EAAE,CAAC;QACX,CAAC;QACD,gCAAgC;aAC3B,IAAI,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACnD,QAAQ,EAAE,CAAC;QACb,CAAC;QACD,6DAA6D;aACxD,IAAI,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAE,4BAA4B;QAC1C,CAAC;QACD,QAAQ;aACH,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7B,MAAM,OAAO,GAAG,MAAM,GAAG,KAAK,CAAC;IAC/B,MAAM,OAAO,GAAG,QAAQ,GAAG,KAAK,CAAC;IACjC,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IAChC,MAAM,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IAEhC,qBAAqB;IACrB,IAAI,OAAO,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,OAAO,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC,CAAE,+CAA+C;IAChF,IAAI,OAAO,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAC/B,IAAI,OAAO,GAAG,GAAG;QAAE,OAAO,IAAI,CAAC;IAE/B,mBAAmB;IACnB,IAAI,OAAO,GAAG,OAAO,IAAI,OAAO,GAAG,OAAO;QAAE,OAAO,IAAI,CAAC;IACxD,IAAI,OAAO,GAAG,OAAO;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,OAAO,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAE7B,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,QAA2B,EAAE,IAAa;IACpE,IAAI,IAAI,GAAG,QAAQ,CAAC;IAEpB,IAAI,QAAQ,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC;QAChC,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,IAAI;YACP,OAAO,IAAI,cAAc,EAAE,CAAC;QAC9B,KAAK,IAAI;YACP,OAAO,IAAI,gBAAgB,EAAE,CAAC;QAChC,KAAK,IAAI;YACP,OAAO,IAAI,eAAe,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC;QACV;YACE,OAAO,IAAI,eAAe,EAAE,CAAC;IACjC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACzB,SAAS,GAAoC,IAAI,GAAG,EAAE,CAAC;IAE/D;QACE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,gBAAgB,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,eAAe,EAAE,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY;QACnB,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;QACvE,OAAO;YACL,QAAQ;YACR,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,OAAO,CAAC,IAAY;QAQlB,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;QAEvE,OAAO;YACL,QAAQ;YACR,SAAS,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC;YACxC,OAAO,EAAE,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC;YACpC,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;YAChC,SAAS,EAAE,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC;YACxC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;SAChC,CAAC;IACJ,CAAC;CACF;AAED,mBAAmB;AACnB,MAAM,CAAC,MAAM,kBAAkB,GAAG,IAAI,sBAAsB,EAAE,CAAC"}

package/dist/core/local-embedding.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Local Embedding Service
+ * 로컬 초소형 모델을 사용한 임베딩 생성
+ *
+ * @module core/local-embedding
+ */
+/**
+ * 임베딩 결과
+ */
+interface EmbeddingResult {
+    text: string;
+    embedding: number[];
+    model: string;
+    processingTime: number;
+}
+/**
+ * 배치 임베딩 결과
+ */
+interface BatchEmbeddingResult {
+    results: EmbeddingResult[];
+    totalTime: number;
+    avgTime: number;
+}
+/**
+ * 유사도 결과
+ */
+interface SimilarityResult {
+    text1: string;
+    text2: string;
+    similarity: number;
+    method: 'cosine' | 'dot' | 'euclidean';
+}
+/**
+ * Local Embedding Service
+ */
+export declare class LocalEmbeddingService {
+    private extractor;
+    private modelName;
+    private isLoading;
+    private loadPromise;
+    private cache;
+    private cacheMaxSize;
+    private stats;
+    constructor(modelName?: string);
+    /**
+     * 모델 로드
+     */
+    init(): Promise<void>;
+    /**
+     * 단일 텍스트 임베딩
+     */
+    embed(text: string): Promise<EmbeddingResult>;
+    /**
+     * 배치 임베딩
+     */
+    embedBatch(texts: string[]): Promise<BatchEmbeddingResult>;
+    /**
+     * 두 텍스트 간 유사도 계산
+     */
+    similarity(text1: string, text2: string, method?: 'cosine' | 'dot' | 'euclidean'): Promise<SimilarityResult>;
+    /**
+     * 쿼리와 가장 유사한 텍스트 찾기
+     */
+    findMostSimilar(query: string, candidates: string[], topK?: number): Promise<Array<{
+        text: string;
+        similarity: number;
+    }>>;
+    /**
+     * 코사인 유사도
+     */
+    private cosineSimilarity;
+    /**
+     * 내적
+     */
+    private dotProduct;
+    /**
+     * 유클리드 거리
+     */
+    private euclideanDistance;
+    /**
+     * 캐시에 추가
+     */
+    private addToCache;
+    /**
+     * 통계 반환
+     */
+    getStats(): {
+        cacheSize: number;
+        cacheHitRate: number;
+        avgEmbeddingTime: number;
+        model: string;
+        isLoaded: boolean;
+        totalEmbeddings: number;
+        cacheHits: number;
+        totalTime: number;
+        modelLoadTime: number;
+    };
+    /**
+     * 캐시 클리어
+     */
+    clearCache(): void;
+    /**
+     * 임베딩 차원 반환
+     */
+    getDimension(): number;
+}
+export declare const localEmbedding: LocalEmbeddingService;
+export {};
+//# sourceMappingURL=local-embedding.d.ts.map

package/dist/core/local-embedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-embedding.d.ts","sourceRoot":"","sources":["../../src/core/local-embedding.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH;;GAEG;AACH,UAAU,eAAe;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,UAAU,oBAAoB;IAC5B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,UAAU,gBAAgB;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,QAAQ,GAAG,KAAK,GAAG,WAAW,CAAC;CACxC;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,SAAS,CAAiC;IAClD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,WAAW,CAA8B;IAGjD,OAAO,CAAC,KAAK,CAAoC;IACjD,OAAO,CAAC,YAAY,CAAiB;IAGrC,OAAO,CAAC,KAAK,CAKX;gBAEU,SAAS,GAAE,MAAkC;IAIzD;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0B3B;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAwCnD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAqBhE;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,GAAE,QAAQ,GAAG,KAAK,GAAG,WAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAuB5H;;OAEG;IACG,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,MAAU,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAclI;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAcxB;;OAEG;IACH,OAAO,CAAC,UAAU;IAIlB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,OAAO,CAAC,UAAU;IASlB;;OAEG;IACH,QAAQ;;;;;;;;;;;IAeR;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB;;OAEG;IACH,YAAY,IAAI,MAAM;CAGvB;AAGD,eAAO,MAAM,cAAc,uBAA8B,CAAC"}