@chnxthr/knowledge-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +37 -2
- package/dist/bootstrap.d.ts +1 -0
- package/dist/bootstrap.js +18 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +5 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/auth.d.ts +38 -0
- package/dist/core/auth.js +93 -0
- package/dist/core/auth.js.map +1 -0
- package/dist/core/config.d.ts +259 -0
- package/dist/core/config.js +172 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/context.d.ts +25 -0
- package/dist/core/context.js +37 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +2 -0
- package/dist/core/env.js +5 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/log_mask.d.ts +10 -0
- package/dist/core/log_mask.js +88 -0
- package/dist/core/log_mask.js.map +1 -0
- package/dist/core/oauth.d.ts +46 -0
- package/dist/core/oauth.js +507 -0
- package/dist/core/oauth.js.map +1 -0
- package/dist/core/paths.d.ts +5 -0
- package/dist/core/paths.js +47 -0
- package/dist/core/paths.js.map +1 -0
- package/dist/core/scope.d.ts +28 -0
- package/dist/core/scope.js +54 -0
- package/dist/core/scope.js.map +1 -0
- package/dist/core/upstream.d.ts +30 -0
- package/dist/core/upstream.js +164 -0
- package/dist/core/upstream.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/plugin/define.d.ts +144 -0
- package/dist/plugin/define.js +44 -0
- package/dist/plugin/define.js.map +1 -0
- package/dist/plugin/load.d.ts +2 -0
- package/dist/plugin/load.js +41 -0
- package/dist/plugin/load.js.map +1 -0
- package/dist/plugin/ping.d.ts +9 -0
- package/dist/plugin/ping.js +29 -0
- package/dist/plugin/ping.js.map +1 -0
- package/dist/plugin/registry.d.ts +62 -0
- package/dist/plugin/registry.js +248 -0
- package/dist/plugin/registry.js.map +1 -0
- package/dist/plugin/result.d.ts +20 -0
- package/dist/plugin/result.js +59 -0
- package/dist/plugin/result.js.map +1 -0
- package/dist/plugins/drive/chunker.d.ts +28 -0
- package/dist/plugins/drive/chunker.js +356 -0
- package/dist/plugins/drive/chunker.js.map +1 -0
- package/dist/plugins/drive/clean_text.d.ts +13 -0
- package/dist/plugins/drive/clean_text.js +31 -0
- package/dist/plugins/drive/clean_text.js.map +1 -0
- package/dist/plugins/drive/client.d.ts +94 -0
- package/dist/plugins/drive/client.js +220 -0
- package/dist/plugins/drive/client.js.map +1 -0
- package/dist/plugins/drive/embeddings.d.ts +15 -0
- package/dist/plugins/drive/embeddings.js +118 -0
- package/dist/plugins/drive/embeddings.js.map +1 -0
- package/dist/plugins/drive/exclude.d.ts +11 -0
- package/dist/plugins/drive/exclude.js +36 -0
- package/dist/plugins/drive/exclude.js.map +1 -0
- package/dist/plugins/drive/extract_text.d.ts +26 -0
- package/dist/plugins/drive/extract_text.js +95 -0
- package/dist/plugins/drive/extract_text.js.map +1 -0
- package/dist/plugins/drive/get_token.d.ts +5 -0
- package/dist/plugins/drive/get_token.js +128 -0
- package/dist/plugins/drive/get_token.js.map +1 -0
- package/dist/plugins/drive/index.d.ts +5 -0
- package/dist/plugins/drive/index.js +72 -0
- package/dist/plugins/drive/index.js.map +1 -0
- package/dist/plugins/drive/indexer.d.ts +69 -0
- package/dist/plugins/drive/indexer.js +204 -0
- package/dist/plugins/drive/indexer.js.map +1 -0
- package/dist/plugins/drive/mathpix.d.ts +4 -0
- package/dist/plugins/drive/mathpix.js +102 -0
- package/dist/plugins/drive/mathpix.js.map +1 -0
- package/dist/plugins/drive/ooxml.d.ts +17 -0
- package/dist/plugins/drive/ooxml.js +152 -0
- package/dist/plugins/drive/ooxml.js.map +1 -0
- package/dist/plugins/drive/path.d.ts +40 -0
- package/dist/plugins/drive/path.js +87 -0
- package/dist/plugins/drive/path.js.map +1 -0
- package/dist/plugins/drive/qdrant.d.ts +123 -0
- package/dist/plugins/drive/qdrant.js +148 -0
- package/dist/plugins/drive/qdrant.js.map +1 -0
- package/dist/plugins/drive/reindex.d.ts +2 -0
- package/dist/plugins/drive/reindex.js +263 -0
- package/dist/plugins/drive/reindex.js.map +1 -0
- package/dist/plugins/drive/settings.d.ts +11 -0
- package/dist/plugins/drive/settings.js +13 -0
- package/dist/plugins/drive/settings.js.map +1 -0
- package/dist/plugins/drive/sync.d.ts +16 -0
- package/dist/plugins/drive/sync.js +139 -0
- package/dist/plugins/drive/sync.js.map +1 -0
- package/dist/plugins/drive/sync_state.d.ts +3 -0
- package/dist/plugins/drive/sync_state.js +26 -0
- package/dist/plugins/drive/sync_state.js.map +1 -0
- package/dist/plugins/drive/tools/list_drive_folder.d.ts +104 -0
- package/dist/plugins/drive/tools/list_drive_folder.js +221 -0
- package/dist/plugins/drive/tools/list_drive_folder.js.map +1 -0
- package/dist/plugins/drive/tools/read_drive_file.d.ts +66 -0
- package/dist/plugins/drive/tools/read_drive_file.js +206 -0
- package/dist/plugins/drive/tools/read_drive_file.js.map +1 -0
- package/dist/plugins/drive/tools/search_drive.d.ts +52 -0
- package/dist/plugins/drive/tools/search_drive.js +78 -0
- package/dist/plugins/drive/tools/search_drive.js.map +1 -0
- package/dist/plugins/drive/tools/semantic_search_drive.d.ts +109 -0
- package/dist/plugins/drive/tools/semantic_search_drive.js +228 -0
- package/dist/plugins/drive/tools/semantic_search_drive.js.map +1 -0
- package/dist/plugins/github/index.d.ts +1 -0
- package/dist/plugins/github/index.js +27 -0
- package/dist/plugins/github/index.js.map +1 -0
- package/dist/plugins/github/repo_tree.d.ts +57 -0
- package/dist/plugins/github/repo_tree.js +123 -0
- package/dist/plugins/github/repo_tree.js.map +1 -0
- package/dist/plugins/index.d.ts +11 -0
- package/dist/plugins/index.js +7 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/plugins/railway/index.d.ts +1 -0
- package/dist/plugins/railway/index.js +21 -0
- package/dist/plugins/railway/index.js.map +1 -0
- package/dist/plugins/railway/tools.d.ts +114 -0
- package/dist/plugins/railway/tools.js +165 -0
- package/dist/plugins/railway/tools.js.map +1 -0
- package/dist/plugins/slack/api.d.ts +63 -0
- package/dist/plugins/slack/api.js +280 -0
- package/dist/plugins/slack/api.js.map +1 -0
- package/dist/plugins/slack/index.d.ts +1 -0
- package/dist/plugins/slack/index.js +80 -0
- package/dist/plugins/slack/index.js.map +1 -0
- package/dist/plugins/slack/tools/slack_files.d.ts +87 -0
- package/dist/plugins/slack/tools/slack_files.js +204 -0
- package/dist/plugins/slack/tools/slack_files.js.map +1 -0
- package/dist/plugins/slack/tools/slack_list_channels.d.ts +46 -0
- package/dist/plugins/slack/tools/slack_list_channels.js +197 -0
- package/dist/plugins/slack/tools/slack_list_channels.js.map +1 -0
- package/dist/plugins/slack/tools/slack_read_channel.d.ts +56 -0
- package/dist/plugins/slack/tools/slack_read_channel.js +150 -0
- package/dist/plugins/slack/tools/slack_read_channel.js.map +1 -0
- package/dist/plugins/slack/tools/slack_search.d.ts +53 -0
- package/dist/plugins/slack/tools/slack_search.js +132 -0
- package/dist/plugins/slack/tools/slack_search.js.map +1 -0
- package/dist/run.d.ts +8 -0
- package/dist/run.js +270 -0
- package/dist/run.js.map +1 -0
- package/dist/server/http.d.ts +5 -0
- package/dist/server/http.js +406 -0
- package/dist/server/http.js.map +1 -0
- package/dist/server/identity.d.ts +17 -0
- package/dist/server/identity.js +47 -0
- package/dist/server/identity.js.map +1 -0
- package/dist/server/mcp.d.ts +7 -0
- package/dist/server/mcp.js +55 -0
- package/dist/server/mcp.js.map +1 -0
- package/dist/server/stdio.d.ts +2 -0
- package/dist/server/stdio.js +40 -0
- package/dist/server/stdio.js.map +1 -0
- package/dist/store/index.d.ts +11 -0
- package/dist/store/index.js +31 -0
- package/dist/store/index.js.map +1 -0
- package/dist/store/keys.d.ts +9 -0
- package/dist/store/keys.js +32 -0
- package/dist/store/keys.js.map +1 -0
- package/dist/store/sqlite.d.ts +38 -0
- package/dist/store/sqlite.js +316 -0
- package/dist/store/sqlite.js.map +1 -0
- package/dist/store/supabase.d.ts +35 -0
- package/dist/store/supabase.js +269 -0
- package/dist/store/supabase.js.map +1 -0
- package/dist/store/supabase_client.d.ts +10 -0
- package/dist/store/supabase_client.js +31 -0
- package/dist/store/supabase_client.js.map +1 -0
- package/dist/store/tool_log.d.ts +12 -0
- package/dist/store/tool_log.js +8 -0
- package/dist/store/tool_log.js.map +1 -0
- package/dist/store/types.d.ts +96 -0
- package/dist/store/types.js +2 -0
- package/dist/store/types.js.map +1 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +11 -0
- package/dist/version.js.map +1 -0
- package/package.json +80 -4
- package/sql/supabase.sql +185 -0
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Markdown chunking — heading 기반 분할 + token-aware sub-split + overlap.
|
|
3
|
+
*
|
|
4
|
+
* 전략:
|
|
5
|
+
* 1. Front-matter (---...---) 분리 → metadata 추출 (검색 payload 승격)
|
|
6
|
+
* 2. heading (^#+ ) 위치마다 split → section
|
|
7
|
+
* 3. 각 section 이 MAX_TOKENS 넘으면 paragraph (\n\n) 단위로 sub-split
|
|
8
|
+
* - 표(|...|)·코드블록(```) 은 fence-aware (행/블록 중간에서 안 잘림)
|
|
9
|
+
* 4. MIN_TOKENS 미만은 다음 청크와 merge
|
|
10
|
+
* 5. 청크 시작에 OVERLAP_TOKENS 만큼 이전 청크 끝 prepend
|
|
11
|
+
* 6. 이미지 markdown() → URL 은 image_urls 로 분리, content 엔 alt 만 (임베딩 노이즈 방지)
|
|
12
|
+
*/
|
|
13
|
+
import { encode } from 'gpt-tokenizer';
|
|
14
|
+
import { parse as parseYaml } from 'yaml';
|
|
15
|
+
const MAX_TOKENS = 800;
|
|
16
|
+
const MIN_TOKENS = 50;
|
|
17
|
+
const OVERLAP_TOKENS = 100;
|
|
18
|
+
// 단일 unit(문단/문장)의 절대 상한 — 초과 시 강제 분할. 임베딩 API 입력 초과(Bad Request) 방지.
|
|
19
|
+
const HARD_MAX_TOKENS = MAX_TOKENS;
|
|
20
|
+
const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n+/;
|
|
21
|
+
export function parseFrontmatter(markdown) {
|
|
22
|
+
const m = markdown.match(FRONTMATTER_RE);
|
|
23
|
+
if (!m)
|
|
24
|
+
return { frontmatter: {}, body: markdown };
|
|
25
|
+
// 정식 YAML 파싱 — multi-line list(related/source_refs/affects)·inline 배열·따옴표를 문법대로.
|
|
26
|
+
// (구 단일라인 정규식은 multi-line list 를 빈 값으로 버려 related 의 ~89% 유실 → yaml 파서로 교체)
|
|
27
|
+
let fm = {};
|
|
28
|
+
try {
|
|
29
|
+
const parsed = parseYaml(m[1]);
|
|
30
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
31
|
+
fm = parsed;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
// 깨진 YAML frontmatter — 본문은 보존, 메타만 빈값 (fail-safe: 한 문서가 색인 전체를 멈추지 않게)
|
|
36
|
+
fm = {};
|
|
37
|
+
}
|
|
38
|
+
return { frontmatter: fm, body: markdown.slice(m[0].length) };
|
|
39
|
+
}
|
|
40
|
+
/** unknown → 정규화 문자열 (Date·number·bool 방어). 빈 문자열은 null. */
|
|
41
|
+
function asString(v) {
|
|
42
|
+
if (v === null || v === undefined)
|
|
43
|
+
return null;
|
|
44
|
+
if (typeof v === 'string')
|
|
45
|
+
return v.trim() || null;
|
|
46
|
+
if (typeof v === 'number' || typeof v === 'boolean')
|
|
47
|
+
return String(v);
|
|
48
|
+
if (v instanceof Date)
|
|
49
|
+
return v.toISOString().slice(0, 10);
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
/** unknown → string[]. yaml 배열 그대로 / "[a,b]"·"a,b" 문자열 / 단일값 모두 흡수. */
|
|
53
|
+
function asArray(v) {
|
|
54
|
+
if (v === null || v === undefined)
|
|
55
|
+
return [];
|
|
56
|
+
if (Array.isArray(v))
|
|
57
|
+
return v.map((x) => String(x).trim()).filter(Boolean);
|
|
58
|
+
if (typeof v === 'string') {
|
|
59
|
+
const s = v.trim().replace(/^\[|\]$/g, '');
|
|
60
|
+
if (!s)
|
|
61
|
+
return [];
|
|
62
|
+
return s
|
|
63
|
+
.split(',')
|
|
64
|
+
.map((x) => x.trim().replace(/^["']|["']$/g, ''))
|
|
65
|
+
.filter(Boolean);
|
|
66
|
+
}
|
|
67
|
+
return [];
|
|
68
|
+
}
|
|
69
|
+
/** frontmatter(yaml 파싱 결과) → 검색 payload 메타. */
|
|
70
|
+
function parseDocMeta(fm) {
|
|
71
|
+
return {
|
|
72
|
+
doc_status: asString(fm.status),
|
|
73
|
+
doc_tier: asString(fm.tier),
|
|
74
|
+
doc_date: asString(fm.date),
|
|
75
|
+
doc_authors: asArray(fm.authors),
|
|
76
|
+
doc_related: asArray(fm.related),
|
|
77
|
+
doc_tags: asArray(fm.tags),
|
|
78
|
+
doc_source_refs: asArray(fm.source_refs),
|
|
79
|
+
doc_affects: asArray(fm.affects),
|
|
80
|
+
doc_products: asArray(fm.products),
|
|
81
|
+
doc_type: asString(fm.type),
|
|
82
|
+
doc_confidence: asString(fm.confidence),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
function countTokens(text) {
|
|
86
|
+
// gpt-tokenizer 는 cl100k_base (text-embedding-3 호환) 기본
|
|
87
|
+
return encode(text).length;
|
|
88
|
+
}
|
|
89
|
+
// lone surrogate (짝 안 맞는 UTF-16 surrogate) 제거 — 토큰 경계 cut 이 이모지(surrogate pair)를
|
|
90
|
+
// 중간에 쪼개 반쪽만 남으면 임베딩 API 가 invalid 로 400 Bad Request.
|
|
91
|
+
const LONE_SURROGATE_RE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g;
|
|
92
|
+
function stripLoneSurrogates(text) {
|
|
93
|
+
return text.replace(LONE_SURROGATE_RE, '');
|
|
94
|
+
}
|
|
95
|
+
//  — 이미지 markdown. URL 은 image_urls 로 분리, content 엔 alt 만 (영문 URL 토큰이
|
|
96
|
+
// 한국어 임베딩 벡터를 흔드는 것 방지). alt 가 비면 자리 제거.
|
|
97
|
+
const IMAGE_RE = /!\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/g;
|
|
98
|
+
function extractImages(content) {
|
|
99
|
+
const urls = [];
|
|
100
|
+
const cleaned = content.replace(IMAGE_RE, (_full, alt, url) => {
|
|
101
|
+
urls.push(url);
|
|
102
|
+
return alt ? alt : '';
|
|
103
|
+
});
|
|
104
|
+
return { content: cleaned, image_urls: urls };
|
|
105
|
+
}
|
|
106
|
+
/** 청크 내 markdown table 개수 (separator 행 `|---|` 수로 근사). */
|
|
107
|
+
function countTables(content) {
|
|
108
|
+
return (content.match(/^\s*\|[\s:|-]+\|\s*$/gm) ?? []).length;
|
|
109
|
+
}
|
|
110
|
+
/** markdown table block 인지 (행 70%+ 가 |...| 형태). */
|
|
111
|
+
function isTableBlock(text) {
|
|
112
|
+
const lines = text.trim().split('\n');
|
|
113
|
+
if (lines.length < 2)
|
|
114
|
+
return false;
|
|
115
|
+
const pipeLines = lines.filter((l) => /^\s*\|.*\|\s*$/.test(l)).length;
|
|
116
|
+
return pipeLines >= lines.length * 0.7;
|
|
117
|
+
}
|
|
118
|
+
/** 큰 표를 행 단위로 분할 + 각 조각에 header(첫 행 + separator) prepend (컬럼명 보존). */
|
|
119
|
+
function splitTable(table, maxTokens) {
|
|
120
|
+
const lines = table.trim().split('\n');
|
|
121
|
+
const sepIdx = lines.findIndex((l) => /^\s*\|[\s:|-]+\|\s*$/.test(l));
|
|
122
|
+
const headerEnd = sepIdx >= 0 ? sepIdx + 1 : 1;
|
|
123
|
+
const header = lines.slice(0, headerEnd).join('\n');
|
|
124
|
+
const bodyRows = lines.slice(headerEnd);
|
|
125
|
+
const headerTok = countTokens(header);
|
|
126
|
+
const out = [];
|
|
127
|
+
let buf = [header];
|
|
128
|
+
let bufTok = headerTok;
|
|
129
|
+
for (const row of bodyRows) {
|
|
130
|
+
const rt = countTokens(row);
|
|
131
|
+
if (bufTok + rt > maxTokens && buf.length > 1) {
|
|
132
|
+
out.push(buf.join('\n'));
|
|
133
|
+
buf = [header, row];
|
|
134
|
+
bufTok = headerTok + rt;
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
buf.push(row);
|
|
138
|
+
bufTok += rt;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (buf.length > 1)
|
|
142
|
+
out.push(buf.join('\n'));
|
|
143
|
+
return out.length > 0 ? out : [table];
|
|
144
|
+
}
|
|
145
|
+
/** 코드블록(```) 보호하며 \n\n paragraph split. 코드블록은 통째 1 unit. */
|
|
146
|
+
function splitParagraphs(text) {
|
|
147
|
+
const out = [];
|
|
148
|
+
for (const seg of text.split(/(```[\s\S]*?```)/g)) {
|
|
149
|
+
if (seg.startsWith('```')) {
|
|
150
|
+
if (seg.trim())
|
|
151
|
+
out.push(seg.trim());
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
for (const p of seg.split(/\n\n+/)) {
|
|
155
|
+
if (p.trim())
|
|
156
|
+
out.push(p.trim());
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
return out;
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* 한국어·영어 문장 분리 (보수적). 종결부호/개행 기준. 분리 실패 시 원문 1개 (손실 X).
|
|
164
|
+
*/
|
|
165
|
+
function splitSentences(text) {
|
|
166
|
+
const parts = text
|
|
167
|
+
.split(/(?<=[.!?。…])\s+|(?<=[가-힣][다요죠음임함됨까네]\.)\s+|(?<=\n)/)
|
|
168
|
+
.map((s) => s.trim())
|
|
169
|
+
.filter((s) => s.length > 0);
|
|
170
|
+
return parts.length > 0 ? parts : [text];
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* 문자 단위 hard cut (문장조차 상한 초과 시 최후 수단). 이진탐색으로 maxTokens 이하 최대 prefix.
|
|
174
|
+
*/
|
|
175
|
+
function hardCutByChars(text, maxTokens) {
|
|
176
|
+
if (countTokens(text) <= maxTokens)
|
|
177
|
+
return [text];
|
|
178
|
+
const out = [];
|
|
179
|
+
let rest = text;
|
|
180
|
+
while (rest.length > 0) {
|
|
181
|
+
if (countTokens(rest) <= maxTokens) {
|
|
182
|
+
out.push(rest);
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
let lo = 1;
|
|
186
|
+
let hi = rest.length;
|
|
187
|
+
let cut = 1;
|
|
188
|
+
while (lo <= hi) {
|
|
189
|
+
const mid = (lo + hi) >> 1;
|
|
190
|
+
if (countTokens(rest.slice(0, mid)) <= maxTokens) {
|
|
191
|
+
cut = mid;
|
|
192
|
+
lo = mid + 1;
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
hi = mid - 1;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
out.push(rest.slice(0, cut));
|
|
199
|
+
rest = rest.slice(cut);
|
|
200
|
+
}
|
|
201
|
+
return out;
|
|
202
|
+
}
|
|
203
|
+
/** text 의 마지막 maxTokens 토큰에 해당하는 suffix (overlap 용, 이진탐색). */
|
|
204
|
+
function tailByTokens(text, maxTokens) {
|
|
205
|
+
if (countTokens(text) <= maxTokens)
|
|
206
|
+
return text;
|
|
207
|
+
let lo = 0;
|
|
208
|
+
let hi = text.length - 1;
|
|
209
|
+
let cut = text.length - 1;
|
|
210
|
+
while (lo <= hi) {
|
|
211
|
+
const mid = (lo + hi) >> 1;
|
|
212
|
+
if (countTokens(text.slice(mid)) <= maxTokens) {
|
|
213
|
+
cut = mid;
|
|
214
|
+
hi = mid - 1;
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
lo = mid + 1;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
return text.slice(cut);
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* MAX 초과 unit 강제 분할. 표면 행 단위(header 반복), 아니면 문장 budget → 문자 hard cut.
|
|
224
|
+
* 모든 반환 unit 은 maxTokens 이하 보장.
|
|
225
|
+
*/
|
|
226
|
+
function splitLongText(text, maxTokens) {
|
|
227
|
+
if (countTokens(text) <= maxTokens)
|
|
228
|
+
return [text];
|
|
229
|
+
// 표는 행 단위 분리 (각 조각 header prepend)
|
|
230
|
+
if (isTableBlock(text))
|
|
231
|
+
return splitTable(text, maxTokens);
|
|
232
|
+
const out = [];
|
|
233
|
+
let buf = [];
|
|
234
|
+
let bufTokens = 0;
|
|
235
|
+
const flush = () => {
|
|
236
|
+
if (buf.length > 0) {
|
|
237
|
+
out.push(buf.join(' '));
|
|
238
|
+
buf = [];
|
|
239
|
+
bufTokens = 0;
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
for (const sentence of splitSentences(text)) {
|
|
243
|
+
const st = countTokens(sentence);
|
|
244
|
+
if (st > maxTokens) {
|
|
245
|
+
flush();
|
|
246
|
+
out.push(...hardCutByChars(sentence, maxTokens));
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (bufTokens + st > maxTokens && buf.length > 0)
|
|
250
|
+
flush();
|
|
251
|
+
buf.push(sentence);
|
|
252
|
+
bufTokens += st;
|
|
253
|
+
}
|
|
254
|
+
flush();
|
|
255
|
+
return out;
|
|
256
|
+
}
|
|
257
|
+
/** heading line 의 레벨(#개수)+텍스트 추출 — breadcrumb(상위 > 하위) 구성용. */
|
|
258
|
+
function extractHeading(text) {
|
|
259
|
+
const m = text.match(/^(#+)\s+(.+)$/m);
|
|
260
|
+
return m ? { level: m[1].length, text: m[2].trim() } : null;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* 전체 markdown → 청크 array. heading context + 이미지/표/코드/frontmatter 메타 보존.
|
|
264
|
+
*/
|
|
265
|
+
export function chunkMarkdown(markdown) {
|
|
266
|
+
const { frontmatter, body } = parseFrontmatter(markdown);
|
|
267
|
+
const docMeta = parseDocMeta(frontmatter);
|
|
268
|
+
if (body.trim().length === 0)
|
|
269
|
+
return [];
|
|
270
|
+
const sections = body.split(/\n(?=#+ )/);
|
|
271
|
+
const chunks = [];
|
|
272
|
+
const headingStack = []; // index = (heading level - 1) → breadcrumb 상위>하위 맥락 보존
|
|
273
|
+
let currentHeading = '문서 시작';
|
|
274
|
+
for (const section of sections) {
|
|
275
|
+
if (section.trim().length === 0)
|
|
276
|
+
continue;
|
|
277
|
+
const h = extractHeading(section);
|
|
278
|
+
if (h) {
|
|
279
|
+
headingStack.length = h.level - 1; // 같거나 더 깊은 레벨 잘라내고
|
|
280
|
+
headingStack[h.level - 1] = h.text; // 현재 레벨 갱신
|
|
281
|
+
const crumb = headingStack.filter(Boolean).join(' > ');
|
|
282
|
+
if (crumb)
|
|
283
|
+
currentHeading = crumb;
|
|
284
|
+
}
|
|
285
|
+
const tokens = countTokens(section);
|
|
286
|
+
if (tokens <= MAX_TOKENS) {
|
|
287
|
+
chunks.push({ heading: currentHeading, content: section.trim(), tokens });
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
const paragraphs = splitParagraphs(section).flatMap((p) => countTokens(p) > HARD_MAX_TOKENS ? splitLongText(p, HARD_MAX_TOKENS) : [p]);
|
|
291
|
+
let buf = [];
|
|
292
|
+
let bufTokens = 0;
|
|
293
|
+
for (const para of paragraphs) {
|
|
294
|
+
const pt = countTokens(para);
|
|
295
|
+
if (bufTokens + pt > MAX_TOKENS && buf.length > 0) {
|
|
296
|
+
chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
|
|
297
|
+
buf = [];
|
|
298
|
+
bufTokens = 0;
|
|
299
|
+
}
|
|
300
|
+
buf.push(para);
|
|
301
|
+
bufTokens += pt;
|
|
302
|
+
}
|
|
303
|
+
if (buf.length > 0) {
|
|
304
|
+
chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
// MIN_TOKENS 미만 청크 merge
|
|
308
|
+
const merged = [];
|
|
309
|
+
for (const c of chunks) {
|
|
310
|
+
if (c.tokens < MIN_TOKENS && merged.length > 0) {
|
|
311
|
+
const prev = merged[merged.length - 1];
|
|
312
|
+
prev.content = `${prev.content}\n\n${c.content}`;
|
|
313
|
+
prev.tokens += c.tokens;
|
|
314
|
+
}
|
|
315
|
+
else {
|
|
316
|
+
merged.push({ ...c });
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
// 1. 이미지 URL 분리 + surrogate strip + 메타 계산 — overlap *전* 고유 content 기준.
|
|
320
|
+
// (overlap 후 계산하면 이전 청크의 이미지/코드가 인접 청크 메타에 오염됨)
|
|
321
|
+
const withMeta = merged.map((c) => {
|
|
322
|
+
const tableCount = countTables(c.content);
|
|
323
|
+
const hasCode = /```/.test(c.content);
|
|
324
|
+
const { content: imgStripped, image_urls } = extractImages(c.content);
|
|
325
|
+
return {
|
|
326
|
+
heading: c.heading,
|
|
327
|
+
content: stripLoneSurrogates(imgStripped),
|
|
328
|
+
image_urls,
|
|
329
|
+
table_count: tableCount,
|
|
330
|
+
has_code_block: hasCode,
|
|
331
|
+
};
|
|
332
|
+
});
|
|
333
|
+
// 2. Overlap prepend — 이미지 URL 이 이미 분리된 텍스트 suffix (overlap 에 URL 안 섞임).
|
|
334
|
+
// ⚠️ tailByTokens 는 토큰 경계로 자르므로 이모지(surrogate pair)를 반토막 낼 수 있다.
|
|
335
|
+
// 이 lone surrogate 가 Qdrant payload content 에 들어가면 invalid UTF-8 JSON → upsert 400.
|
|
336
|
+
// → 아래 3단계에서 최종 content 에 strip 을 한 번 더 적용해 제거한다.
|
|
337
|
+
for (let i = 1; i < withMeta.length; i++) {
|
|
338
|
+
const overlap = tailByTokens(withMeta[i - 1].content, OVERLAP_TOKENS);
|
|
339
|
+
if (overlap)
|
|
340
|
+
withMeta[i].content = `…${overlap}\n\n${withMeta[i].content}`;
|
|
341
|
+
}
|
|
342
|
+
// 3. 최종 RawChunk — overlap 후 재생성된 lone surrogate 재제거(Qdrant 400 방지) + frontmatter 동봉.
|
|
343
|
+
return withMeta.map((c) => {
|
|
344
|
+
const content = stripLoneSurrogates(c.content);
|
|
345
|
+
return {
|
|
346
|
+
heading: c.heading,
|
|
347
|
+
content,
|
|
348
|
+
tokens: countTokens(content),
|
|
349
|
+
image_urls: c.image_urls,
|
|
350
|
+
table_count: c.table_count,
|
|
351
|
+
has_code_block: c.has_code_block,
|
|
352
|
+
...docMeta,
|
|
353
|
+
};
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/plugins/drive/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,qEAAqE;AACrE,MAAM,eAAe,GAAG,UAAU,CAAC;AA8BnC,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAElD,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACzC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAEnD,iFAAiF;IACjF,2EAA2E;IAC3E,IAAI,EAAE,GAA4B,EAAE,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,EAAE,GAAG,MAAiC,CAAC;QACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;QACxE,EAAE,GAAG,EAAE,CAAC;IACV,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAChE,CAAC;AAgBD,4DAA4D;AAC5D,SAAS,QAAQ,CAAC,CAAU;IAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,YAAY,IAAI;QAAE,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AACvE,SAAS,OAAO,CAAC,CAAU;IACzB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClB,OAAO,CAAC;aACL,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,+CAA+C;AAC/C,SAAS,YAAY,CAAC,EAA2B;IAC/C,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC;QAC/B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;QAC1B,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC;QACxC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;QAClC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,uDAAuD;IACvD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,iFAAiF;AACjF,sDAAsD;AACtD,MAAM,iBAAiB,GAAG,yEAAyE,CAAC;AACpG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,yCAAyC;AACzC,MAAM,QAAQ,GAAG,wCAAwC,CAAC;AAC1D,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,GAAW,EAAE,EAAE;QAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED,0DAA0D;AAC1D,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;AAChE,CAAC;AAED,mDAAmD;AACnD,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,OAAO,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;AACzC,CAAC;AAED,sEAAsE;AACtE,SAAS,UAAU,CAAC,KAAa,EAAE,SAAiB;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,SAAS,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACzB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACpB,MAAM,GAAG,SAAS,GAAG,EAAE,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClD,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,CAAC,IAAI,EAAE;oBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,oDAAoD,CAAC;SAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,SAAiB;IACrD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,MAAM;QACR,CAAC;QACD,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjD,GAAG,GAAG,GAAG,CAAC;gBACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8DAA8D;AAC9D,SAAS,YAAY,CAAC,IAAY,EAAE,SAAiB;IACnD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1B,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9C,GAAG,GAAG,GAAG,CAAC;YACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,SAAiB;IACpD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,mCAAmC;IACnC,IAAI,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAE3D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,EAAE,GAAG,SAAS,EAAE,CAAC;YACnB,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QACD,IAAI,SAAS,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,EAAE,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnB,SAAS,IAAI,EAAE,CAAC;IAClB,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+DAA+D;AAC/D,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC;AAQD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC,CAAC,uDAAuD;IAC1F,IAAI,cAAc,GAAG,OAAO,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,EAAE,CAAC;YACN,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,mBAAmB;YACtD,YAAY,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW;YAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvD,IAAI,KAAK;gBAAE,cAAc,GAAG,KAAK,CAAC;QACpC,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,SAAS,GAAG,EAAE,GAAG,UAAU,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC9F,GAAG,GAAG,EAAE,CAAC;gBACT,SAAS,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,CAAC,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,mDAAmD;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtE,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,mBAAmB,CAAC,WAAW,CAAC;YACzC,UAAU;YACV,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,OAAO;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,oEAAoE;IACpE,uFAAuF;IACvF,qDAAqD;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,OAAO;YAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,OAAO,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7E,CAAC;IAED,sFAAsF;IACtF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO;YACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC;YAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,GAAG,OAAO;SACX,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
|
|
3
|
+
*
|
|
4
|
+
* 보수적 규칙만 (의미 손상 방지):
|
|
5
|
+
* - 이미지 markdown  → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
|
|
6
|
+
* - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
|
|
7
|
+
* - HTML 태그 (<div>, <br>, <span>...) → 텍스트 보존하고 태그만 제거
|
|
8
|
+
* - pdf-parse 페이지 마커 (-- N of M --) → 제거
|
|
9
|
+
* - 과도한 빈 줄·줄끝 공백 → 정규화
|
|
10
|
+
*
|
|
11
|
+
* 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
|
|
12
|
+
*/
|
|
13
|
+
export declare function cleanExtractedText(text: string): string;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
|
|
3
|
+
*
|
|
4
|
+
* 보수적 규칙만 (의미 손상 방지):
|
|
5
|
+
* - 이미지 markdown  → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
|
|
6
|
+
* - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
|
|
7
|
+
* - HTML 태그 (<div>, <br>, <span>...) → 텍스트 보존하고 태그만 제거
|
|
8
|
+
* - pdf-parse 페이지 마커 (-- N of M --) → 제거
|
|
9
|
+
* - 과도한 빈 줄·줄끝 공백 → 정규화
|
|
10
|
+
*
|
|
11
|
+
* 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
|
|
12
|
+
*/
|
|
13
|
+
export function cleanExtractedText(text) {
|
|
14
|
+
if (!text)
|
|
15
|
+
return '';
|
|
16
|
+
let t = text;
|
|
17
|
+
// 1. 이미지 markdown() 은 제거하지 않음 — chunker 가 image_urls 로 분리 보존 (2026-05-27)
|
|
18
|
+
// 2. LaTeX wrapper — \title{X} \section{X} \subsection{X} → X
|
|
19
|
+
t = t.replace(/\\(?:title|section|subsection|textbf|textit)\s*\{([^}]*)\}/g, '$1');
|
|
20
|
+
// 3. HTML <br> → 개행, 나머지 태그는 텍스트 보존하고 태그만 제거
|
|
21
|
+
t = t.replace(/<br\s*\/?>/gi, '\n');
|
|
22
|
+
t = t.replace(/<\/?[a-zA-Z][^>]*>/g, '');
|
|
23
|
+
// 4. pdf-parse 페이지 마커 (-- 1 of 22 --) 제거
|
|
24
|
+
t = t.replace(/^\s*--\s*\d+\s+of\s+\d+\s*--\s*$/gim, '');
|
|
25
|
+
// 5. HTML entity 일부 복원
|
|
26
|
+
t = t.replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
27
|
+
// 6. 줄끝 공백 + 과도한 빈 줄 정규화
|
|
28
|
+
t = t.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n');
|
|
29
|
+
return t.trim();
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=clean_text.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"clean_text.js","sourceRoot":"","sources":["../../../src/plugins/drive/clean_text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,CAAC,GAAG,IAAI,CAAC;IAEb,qFAAqF;IAErF,8DAA8D;IAC9D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,6DAA6D,EAAE,IAAI,CAAC,CAAC;IAEnF,6CAA6C;IAC7C,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAEzC,yCAAyC;IACzC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC;IAEzD,uBAAuB;IACvB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEjG,yBAAyB;IACzB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE5D,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
export interface DriveFile {
|
|
2
|
+
id?: string | null;
|
|
3
|
+
name?: string | null;
|
|
4
|
+
mimeType?: string | null;
|
|
5
|
+
parents?: string[] | null;
|
|
6
|
+
modifiedTime?: string | null;
|
|
7
|
+
size?: string | null;
|
|
8
|
+
webViewLink?: string | null;
|
|
9
|
+
trashed?: boolean | null;
|
|
10
|
+
owners?: Array<{
|
|
11
|
+
emailAddress?: string | null;
|
|
12
|
+
displayName?: string | null;
|
|
13
|
+
}> | null;
|
|
14
|
+
[field: string]: unknown;
|
|
15
|
+
}
|
|
16
|
+
export interface DriveChange {
|
|
17
|
+
removed?: boolean | null;
|
|
18
|
+
fileId?: string | null;
|
|
19
|
+
file?: DriveFile | null;
|
|
20
|
+
}
|
|
21
|
+
export interface DriveChangeList {
|
|
22
|
+
nextPageToken?: string | null;
|
|
23
|
+
newStartPageToken?: string | null;
|
|
24
|
+
changes?: DriveChange[];
|
|
25
|
+
}
|
|
26
|
+
export interface DriveFileList {
|
|
27
|
+
files?: DriveFile[];
|
|
28
|
+
nextPageToken?: string | null;
|
|
29
|
+
}
|
|
30
|
+
type Params = Record<string, string | number | boolean | undefined>;
|
|
31
|
+
type ResponseType = 'json' | 'text' | 'arraybuffer';
|
|
32
|
+
export interface DriveClient {
|
|
33
|
+
files: {
|
|
34
|
+
list(params: Params): Promise<{
|
|
35
|
+
data: DriveFileList;
|
|
36
|
+
}>;
|
|
37
|
+
/** Metadata, or the file body with `alt: 'media'` and a response type. */
|
|
38
|
+
get(params: Params & {
|
|
39
|
+
fileId: string;
|
|
40
|
+
}, opts?: {
|
|
41
|
+
responseType?: ResponseType;
|
|
42
|
+
}): Promise<{
|
|
43
|
+
data: any;
|
|
44
|
+
}>;
|
|
45
|
+
export(params: Params & {
|
|
46
|
+
fileId: string;
|
|
47
|
+
mimeType: string;
|
|
48
|
+
}, opts?: {
|
|
49
|
+
responseType?: ResponseType;
|
|
50
|
+
}): Promise<{
|
|
51
|
+
data: any;
|
|
52
|
+
}>;
|
|
53
|
+
};
|
|
54
|
+
changes: {
|
|
55
|
+
getStartPageToken(params?: Params): Promise<{
|
|
56
|
+
data: {
|
|
57
|
+
startPageToken?: string | null;
|
|
58
|
+
};
|
|
59
|
+
}>;
|
|
60
|
+
list(params: Params & {
|
|
61
|
+
pageToken: string;
|
|
62
|
+
}): Promise<{
|
|
63
|
+
data: DriveChangeList;
|
|
64
|
+
}>;
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export declare class DriveApiError extends Error {
|
|
68
|
+
/** HTTP status (0 = network or timeout). googleapis put the same value in `code`. */
|
|
69
|
+
readonly code: number;
|
|
70
|
+
constructor(message: string,
|
|
71
|
+
/** HTTP status (0 = network or timeout). googleapis put the same value in `code`. */
|
|
72
|
+
code: number);
|
|
73
|
+
}
|
|
74
|
+
type Credentials = {
|
|
75
|
+
kind: 'service_account';
|
|
76
|
+
clientEmail: string;
|
|
77
|
+
privateKey: string;
|
|
78
|
+
tokenUri: string;
|
|
79
|
+
} | {
|
|
80
|
+
kind: 'refresh_token';
|
|
81
|
+
clientId: string;
|
|
82
|
+
clientSecret: string;
|
|
83
|
+
refreshToken: string;
|
|
84
|
+
tokenUri: string;
|
|
85
|
+
};
|
|
86
|
+
/** Which credentials the env provides. Throws with the options when none are set. */
|
|
87
|
+
export declare function resolveCredentials(env?: NodeJS.ProcessEnv): Credentials;
|
|
88
|
+
/** Builds a client. `apiBase` and credentials are injectable for tests; production uses env and Google URLs. */
|
|
89
|
+
export declare function createDriveClient(creds: Credentials, apiBase?: string): DriveClient;
|
|
90
|
+
/** The process-wide Drive client, built from env on first use. */
|
|
91
|
+
export declare function getDriveClient(): Promise<DriveClient>;
|
|
92
|
+
/** Google Drive query language 의 single-quote escape. */
|
|
93
|
+
export declare function escapeDriveQuery(s: string): string;
|
|
94
|
+
export {};
|