@chnxthr/knowledge-mcp 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +37 -2
  3. package/dist/bootstrap.d.ts +1 -0
  4. package/dist/bootstrap.js +18 -0
  5. package/dist/bootstrap.js.map +1 -0
  6. package/dist/cli.d.ts +2 -0
  7. package/dist/cli.js +5 -0
  8. package/dist/cli.js.map +1 -0
  9. package/dist/core/auth.d.ts +38 -0
  10. package/dist/core/auth.js +93 -0
  11. package/dist/core/auth.js.map +1 -0
  12. package/dist/core/config.d.ts +259 -0
  13. package/dist/core/config.js +172 -0
  14. package/dist/core/config.js.map +1 -0
  15. package/dist/core/context.d.ts +25 -0
  16. package/dist/core/context.js +37 -0
  17. package/dist/core/context.js.map +1 -0
  18. package/dist/core/env.d.ts +2 -0
  19. package/dist/core/env.js +5 -0
  20. package/dist/core/env.js.map +1 -0
  21. package/dist/core/log_mask.d.ts +10 -0
  22. package/dist/core/log_mask.js +88 -0
  23. package/dist/core/log_mask.js.map +1 -0
  24. package/dist/core/oauth.d.ts +46 -0
  25. package/dist/core/oauth.js +507 -0
  26. package/dist/core/oauth.js.map +1 -0
  27. package/dist/core/paths.d.ts +5 -0
  28. package/dist/core/paths.js +47 -0
  29. package/dist/core/paths.js.map +1 -0
  30. package/dist/core/scope.d.ts +28 -0
  31. package/dist/core/scope.js +54 -0
  32. package/dist/core/scope.js.map +1 -0
  33. package/dist/core/upstream.d.ts +30 -0
  34. package/dist/core/upstream.js +164 -0
  35. package/dist/core/upstream.js.map +1 -0
  36. package/dist/index.d.ts +23 -0
  37. package/dist/index.js +21 -0
  38. package/dist/index.js.map +1 -0
  39. package/dist/plugin/define.d.ts +144 -0
  40. package/dist/plugin/define.js +44 -0
  41. package/dist/plugin/define.js.map +1 -0
  42. package/dist/plugin/load.d.ts +2 -0
  43. package/dist/plugin/load.js +41 -0
  44. package/dist/plugin/load.js.map +1 -0
  45. package/dist/plugin/ping.d.ts +9 -0
  46. package/dist/plugin/ping.js +29 -0
  47. package/dist/plugin/ping.js.map +1 -0
  48. package/dist/plugin/registry.d.ts +62 -0
  49. package/dist/plugin/registry.js +248 -0
  50. package/dist/plugin/registry.js.map +1 -0
  51. package/dist/plugin/result.d.ts +20 -0
  52. package/dist/plugin/result.js +59 -0
  53. package/dist/plugin/result.js.map +1 -0
  54. package/dist/plugins/drive/chunker.d.ts +28 -0
  55. package/dist/plugins/drive/chunker.js +356 -0
  56. package/dist/plugins/drive/chunker.js.map +1 -0
  57. package/dist/plugins/drive/clean_text.d.ts +13 -0
  58. package/dist/plugins/drive/clean_text.js +31 -0
  59. package/dist/plugins/drive/clean_text.js.map +1 -0
  60. package/dist/plugins/drive/client.d.ts +94 -0
  61. package/dist/plugins/drive/client.js +220 -0
  62. package/dist/plugins/drive/client.js.map +1 -0
  63. package/dist/plugins/drive/embeddings.d.ts +15 -0
  64. package/dist/plugins/drive/embeddings.js +118 -0
  65. package/dist/plugins/drive/embeddings.js.map +1 -0
  66. package/dist/plugins/drive/exclude.d.ts +11 -0
  67. package/dist/plugins/drive/exclude.js +36 -0
  68. package/dist/plugins/drive/exclude.js.map +1 -0
  69. package/dist/plugins/drive/extract_text.d.ts +26 -0
  70. package/dist/plugins/drive/extract_text.js +95 -0
  71. package/dist/plugins/drive/extract_text.js.map +1 -0
  72. package/dist/plugins/drive/get_token.d.ts +5 -0
  73. package/dist/plugins/drive/get_token.js +128 -0
  74. package/dist/plugins/drive/get_token.js.map +1 -0
  75. package/dist/plugins/drive/index.d.ts +5 -0
  76. package/dist/plugins/drive/index.js +72 -0
  77. package/dist/plugins/drive/index.js.map +1 -0
  78. package/dist/plugins/drive/indexer.d.ts +69 -0
  79. package/dist/plugins/drive/indexer.js +204 -0
  80. package/dist/plugins/drive/indexer.js.map +1 -0
  81. package/dist/plugins/drive/mathpix.d.ts +4 -0
  82. package/dist/plugins/drive/mathpix.js +102 -0
  83. package/dist/plugins/drive/mathpix.js.map +1 -0
  84. package/dist/plugins/drive/ooxml.d.ts +17 -0
  85. package/dist/plugins/drive/ooxml.js +152 -0
  86. package/dist/plugins/drive/ooxml.js.map +1 -0
  87. package/dist/plugins/drive/path.d.ts +40 -0
  88. package/dist/plugins/drive/path.js +87 -0
  89. package/dist/plugins/drive/path.js.map +1 -0
  90. package/dist/plugins/drive/qdrant.d.ts +123 -0
  91. package/dist/plugins/drive/qdrant.js +148 -0
  92. package/dist/plugins/drive/qdrant.js.map +1 -0
  93. package/dist/plugins/drive/reindex.d.ts +2 -0
  94. package/dist/plugins/drive/reindex.js +263 -0
  95. package/dist/plugins/drive/reindex.js.map +1 -0
  96. package/dist/plugins/drive/settings.d.ts +11 -0
  97. package/dist/plugins/drive/settings.js +13 -0
  98. package/dist/plugins/drive/settings.js.map +1 -0
  99. package/dist/plugins/drive/sync.d.ts +16 -0
  100. package/dist/plugins/drive/sync.js +139 -0
  101. package/dist/plugins/drive/sync.js.map +1 -0
  102. package/dist/plugins/drive/sync_state.d.ts +3 -0
  103. package/dist/plugins/drive/sync_state.js +26 -0
  104. package/dist/plugins/drive/sync_state.js.map +1 -0
  105. package/dist/plugins/drive/tools/list_drive_folder.d.ts +104 -0
  106. package/dist/plugins/drive/tools/list_drive_folder.js +221 -0
  107. package/dist/plugins/drive/tools/list_drive_folder.js.map +1 -0
  108. package/dist/plugins/drive/tools/read_drive_file.d.ts +66 -0
  109. package/dist/plugins/drive/tools/read_drive_file.js +206 -0
  110. package/dist/plugins/drive/tools/read_drive_file.js.map +1 -0
  111. package/dist/plugins/drive/tools/search_drive.d.ts +52 -0
  112. package/dist/plugins/drive/tools/search_drive.js +78 -0
  113. package/dist/plugins/drive/tools/search_drive.js.map +1 -0
  114. package/dist/plugins/drive/tools/semantic_search_drive.d.ts +109 -0
  115. package/dist/plugins/drive/tools/semantic_search_drive.js +228 -0
  116. package/dist/plugins/drive/tools/semantic_search_drive.js.map +1 -0
  117. package/dist/plugins/github/index.d.ts +1 -0
  118. package/dist/plugins/github/index.js +27 -0
  119. package/dist/plugins/github/index.js.map +1 -0
  120. package/dist/plugins/github/repo_tree.d.ts +57 -0
  121. package/dist/plugins/github/repo_tree.js +123 -0
  122. package/dist/plugins/github/repo_tree.js.map +1 -0
  123. package/dist/plugins/index.d.ts +11 -0
  124. package/dist/plugins/index.js +7 -0
  125. package/dist/plugins/index.js.map +1 -0
  126. package/dist/plugins/railway/index.d.ts +1 -0
  127. package/dist/plugins/railway/index.js +21 -0
  128. package/dist/plugins/railway/index.js.map +1 -0
  129. package/dist/plugins/railway/tools.d.ts +114 -0
  130. package/dist/plugins/railway/tools.js +165 -0
  131. package/dist/plugins/railway/tools.js.map +1 -0
  132. package/dist/plugins/slack/api.d.ts +63 -0
  133. package/dist/plugins/slack/api.js +280 -0
  134. package/dist/plugins/slack/api.js.map +1 -0
  135. package/dist/plugins/slack/index.d.ts +1 -0
  136. package/dist/plugins/slack/index.js +80 -0
  137. package/dist/plugins/slack/index.js.map +1 -0
  138. package/dist/plugins/slack/tools/slack_files.d.ts +87 -0
  139. package/dist/plugins/slack/tools/slack_files.js +204 -0
  140. package/dist/plugins/slack/tools/slack_files.js.map +1 -0
  141. package/dist/plugins/slack/tools/slack_list_channels.d.ts +46 -0
  142. package/dist/plugins/slack/tools/slack_list_channels.js +197 -0
  143. package/dist/plugins/slack/tools/slack_list_channels.js.map +1 -0
  144. package/dist/plugins/slack/tools/slack_read_channel.d.ts +56 -0
  145. package/dist/plugins/slack/tools/slack_read_channel.js +150 -0
  146. package/dist/plugins/slack/tools/slack_read_channel.js.map +1 -0
  147. package/dist/plugins/slack/tools/slack_search.d.ts +53 -0
  148. package/dist/plugins/slack/tools/slack_search.js +132 -0
  149. package/dist/plugins/slack/tools/slack_search.js.map +1 -0
  150. package/dist/run.d.ts +8 -0
  151. package/dist/run.js +270 -0
  152. package/dist/run.js.map +1 -0
  153. package/dist/server/http.d.ts +5 -0
  154. package/dist/server/http.js +406 -0
  155. package/dist/server/http.js.map +1 -0
  156. package/dist/server/identity.d.ts +17 -0
  157. package/dist/server/identity.js +47 -0
  158. package/dist/server/identity.js.map +1 -0
  159. package/dist/server/mcp.d.ts +7 -0
  160. package/dist/server/mcp.js +55 -0
  161. package/dist/server/mcp.js.map +1 -0
  162. package/dist/server/stdio.d.ts +2 -0
  163. package/dist/server/stdio.js +40 -0
  164. package/dist/server/stdio.js.map +1 -0
  165. package/dist/store/index.d.ts +11 -0
  166. package/dist/store/index.js +31 -0
  167. package/dist/store/index.js.map +1 -0
  168. package/dist/store/keys.d.ts +9 -0
  169. package/dist/store/keys.js +32 -0
  170. package/dist/store/keys.js.map +1 -0
  171. package/dist/store/sqlite.d.ts +38 -0
  172. package/dist/store/sqlite.js +316 -0
  173. package/dist/store/sqlite.js.map +1 -0
  174. package/dist/store/supabase.d.ts +35 -0
  175. package/dist/store/supabase.js +269 -0
  176. package/dist/store/supabase.js.map +1 -0
  177. package/dist/store/supabase_client.d.ts +10 -0
  178. package/dist/store/supabase_client.js +31 -0
  179. package/dist/store/supabase_client.js.map +1 -0
  180. package/dist/store/tool_log.d.ts +12 -0
  181. package/dist/store/tool_log.js +8 -0
  182. package/dist/store/tool_log.js.map +1 -0
  183. package/dist/store/types.d.ts +96 -0
  184. package/dist/store/types.js +2 -0
  185. package/dist/store/types.js.map +1 -0
  186. package/dist/version.d.ts +1 -0
  187. package/dist/version.js +11 -0
  188. package/dist/version.js.map +1 -0
  189. package/package.json +80 -4
  190. package/sql/supabase.sql +185 -0
@@ -0,0 +1,356 @@
1
+ /**
2
+ * Markdown chunking — heading 기반 분할 + token-aware sub-split + overlap.
3
+ *
4
+ * 전략:
5
+ * 1. Front-matter (---...---) 분리 → metadata 추출 (검색 payload 승격)
6
+ * 2. heading (^#+ ) 위치마다 split → section
7
+ * 3. 각 section 이 MAX_TOKENS 넘으면 paragraph (\n\n) 단위로 sub-split
8
+ * - 표(|...|)·코드블록(```) 은 fence-aware (행/블록 중간에서 안 잘림)
9
+ * 4. MIN_TOKENS 미만은 다음 청크와 merge
10
+ * 5. 청크 시작에 OVERLAP_TOKENS 만큼 이전 청크 끝 prepend
11
+ * 6. 이미지 markdown(![alt](url)) → URL 은 image_urls 로 분리, content 엔 alt 만 (임베딩 노이즈 방지)
12
+ */
13
+ import { encode } from 'gpt-tokenizer';
14
+ import { parse as parseYaml } from 'yaml';
15
+ const MAX_TOKENS = 800;
16
+ const MIN_TOKENS = 50;
17
+ const OVERLAP_TOKENS = 100;
18
+ // 단일 unit(문단/문장)의 절대 상한 — 초과 시 강제 분할. 임베딩 API 입력 초과(Bad Request) 방지.
19
+ const HARD_MAX_TOKENS = MAX_TOKENS;
20
+ const FRONTMATTER_RE = /^---\n([\s\S]*?)\n---\n+/;
21
+ export function parseFrontmatter(markdown) {
22
+ const m = markdown.match(FRONTMATTER_RE);
23
+ if (!m)
24
+ return { frontmatter: {}, body: markdown };
25
+ // 정식 YAML 파싱 — multi-line list(related/source_refs/affects)·inline 배열·따옴표를 문법대로.
26
+ // (구 단일라인 정규식은 multi-line list 를 빈 값으로 버려 related 의 ~89% 유실 → yaml 파서로 교체)
27
+ let fm = {};
28
+ try {
29
+ const parsed = parseYaml(m[1]);
30
+ if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
31
+ fm = parsed;
32
+ }
33
+ }
34
+ catch {
35
+ // 깨진 YAML frontmatter — 본문은 보존, 메타만 빈값 (fail-safe: 한 문서가 색인 전체를 멈추지 않게)
36
+ fm = {};
37
+ }
38
+ return { frontmatter: fm, body: markdown.slice(m[0].length) };
39
+ }
40
+ /** unknown → 정규화 문자열 (Date·number·bool 방어). 빈 문자열은 null. */
41
+ function asString(v) {
42
+ if (v === null || v === undefined)
43
+ return null;
44
+ if (typeof v === 'string')
45
+ return v.trim() || null;
46
+ if (typeof v === 'number' || typeof v === 'boolean')
47
+ return String(v);
48
+ if (v instanceof Date)
49
+ return v.toISOString().slice(0, 10);
50
+ return null;
51
+ }
52
+ /** unknown → string[]. yaml 배열 그대로 / "[a,b]"·"a,b" 문자열 / 단일값 모두 흡수. */
53
+ function asArray(v) {
54
+ if (v === null || v === undefined)
55
+ return [];
56
+ if (Array.isArray(v))
57
+ return v.map((x) => String(x).trim()).filter(Boolean);
58
+ if (typeof v === 'string') {
59
+ const s = v.trim().replace(/^\[|\]$/g, '');
60
+ if (!s)
61
+ return [];
62
+ return s
63
+ .split(',')
64
+ .map((x) => x.trim().replace(/^["']|["']$/g, ''))
65
+ .filter(Boolean);
66
+ }
67
+ return [];
68
+ }
69
+ /** frontmatter(yaml 파싱 결과) → 검색 payload 메타. */
70
+ function parseDocMeta(fm) {
71
+ return {
72
+ doc_status: asString(fm.status),
73
+ doc_tier: asString(fm.tier),
74
+ doc_date: asString(fm.date),
75
+ doc_authors: asArray(fm.authors),
76
+ doc_related: asArray(fm.related),
77
+ doc_tags: asArray(fm.tags),
78
+ doc_source_refs: asArray(fm.source_refs),
79
+ doc_affects: asArray(fm.affects),
80
+ doc_products: asArray(fm.products),
81
+ doc_type: asString(fm.type),
82
+ doc_confidence: asString(fm.confidence),
83
+ };
84
+ }
85
+ function countTokens(text) {
86
+ // gpt-tokenizer 는 cl100k_base (text-embedding-3 호환) 기본
87
+ return encode(text).length;
88
+ }
89
+ // lone surrogate (짝 안 맞는 UTF-16 surrogate) 제거 — 토큰 경계 cut 이 이모지(surrogate pair)를
90
+ // 중간에 쪼개 반쪽만 남으면 임베딩 API 가 invalid 로 400 Bad Request.
91
+ const LONE_SURROGATE_RE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g;
92
+ function stripLoneSurrogates(text) {
93
+ return text.replace(LONE_SURROGATE_RE, '');
94
+ }
95
+ // ![alt](url) — 이미지 markdown. URL 은 image_urls 로 분리, content 엔 alt 만 (영문 URL 토큰이
96
+ // 한국어 임베딩 벡터를 흔드는 것 방지). alt 가 비면 자리 제거.
97
+ const IMAGE_RE = /!\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)/g;
98
+ function extractImages(content) {
99
+ const urls = [];
100
+ const cleaned = content.replace(IMAGE_RE, (_full, alt, url) => {
101
+ urls.push(url);
102
+ return alt ? alt : '';
103
+ });
104
+ return { content: cleaned, image_urls: urls };
105
+ }
106
+ /** 청크 내 markdown table 개수 (separator 행 `|---|` 수로 근사). */
107
+ function countTables(content) {
108
+ return (content.match(/^\s*\|[\s:|-]+\|\s*$/gm) ?? []).length;
109
+ }
110
+ /** markdown table block 인지 (행 70%+ 가 |...| 형태). */
111
+ function isTableBlock(text) {
112
+ const lines = text.trim().split('\n');
113
+ if (lines.length < 2)
114
+ return false;
115
+ const pipeLines = lines.filter((l) => /^\s*\|.*\|\s*$/.test(l)).length;
116
+ return pipeLines >= lines.length * 0.7;
117
+ }
118
+ /** 큰 표를 행 단위로 분할 + 각 조각에 header(첫 행 + separator) prepend (컬럼명 보존). */
119
+ function splitTable(table, maxTokens) {
120
+ const lines = table.trim().split('\n');
121
+ const sepIdx = lines.findIndex((l) => /^\s*\|[\s:|-]+\|\s*$/.test(l));
122
+ const headerEnd = sepIdx >= 0 ? sepIdx + 1 : 1;
123
+ const header = lines.slice(0, headerEnd).join('\n');
124
+ const bodyRows = lines.slice(headerEnd);
125
+ const headerTok = countTokens(header);
126
+ const out = [];
127
+ let buf = [header];
128
+ let bufTok = headerTok;
129
+ for (const row of bodyRows) {
130
+ const rt = countTokens(row);
131
+ if (bufTok + rt > maxTokens && buf.length > 1) {
132
+ out.push(buf.join('\n'));
133
+ buf = [header, row];
134
+ bufTok = headerTok + rt;
135
+ }
136
+ else {
137
+ buf.push(row);
138
+ bufTok += rt;
139
+ }
140
+ }
141
+ if (buf.length > 1)
142
+ out.push(buf.join('\n'));
143
+ return out.length > 0 ? out : [table];
144
+ }
145
+ /** 코드블록(```) 보호하며 \n\n paragraph split. 코드블록은 통째 1 unit. */
146
+ function splitParagraphs(text) {
147
+ const out = [];
148
+ for (const seg of text.split(/(```[\s\S]*?```)/g)) {
149
+ if (seg.startsWith('```')) {
150
+ if (seg.trim())
151
+ out.push(seg.trim());
152
+ }
153
+ else {
154
+ for (const p of seg.split(/\n\n+/)) {
155
+ if (p.trim())
156
+ out.push(p.trim());
157
+ }
158
+ }
159
+ }
160
+ return out;
161
+ }
162
+ /**
163
+ * 한국어·영어 문장 분리 (보수적). 종결부호/개행 기준. 분리 실패 시 원문 1개 (손실 X).
164
+ */
165
+ function splitSentences(text) {
166
+ const parts = text
167
+ .split(/(?<=[.!?。…])\s+|(?<=[가-힣][다요죠음임함됨까네]\.)\s+|(?<=\n)/)
168
+ .map((s) => s.trim())
169
+ .filter((s) => s.length > 0);
170
+ return parts.length > 0 ? parts : [text];
171
+ }
172
+ /**
173
+ * 문자 단위 hard cut (문장조차 상한 초과 시 최후 수단). 이진탐색으로 maxTokens 이하 최대 prefix.
174
+ */
175
+ function hardCutByChars(text, maxTokens) {
176
+ if (countTokens(text) <= maxTokens)
177
+ return [text];
178
+ const out = [];
179
+ let rest = text;
180
+ while (rest.length > 0) {
181
+ if (countTokens(rest) <= maxTokens) {
182
+ out.push(rest);
183
+ break;
184
+ }
185
+ let lo = 1;
186
+ let hi = rest.length;
187
+ let cut = 1;
188
+ while (lo <= hi) {
189
+ const mid = (lo + hi) >> 1;
190
+ if (countTokens(rest.slice(0, mid)) <= maxTokens) {
191
+ cut = mid;
192
+ lo = mid + 1;
193
+ }
194
+ else {
195
+ hi = mid - 1;
196
+ }
197
+ }
198
+ out.push(rest.slice(0, cut));
199
+ rest = rest.slice(cut);
200
+ }
201
+ return out;
202
+ }
203
+ /** text 의 마지막 maxTokens 토큰에 해당하는 suffix (overlap 용, 이진탐색). */
204
+ function tailByTokens(text, maxTokens) {
205
+ if (countTokens(text) <= maxTokens)
206
+ return text;
207
+ let lo = 0;
208
+ let hi = text.length - 1;
209
+ let cut = text.length - 1;
210
+ while (lo <= hi) {
211
+ const mid = (lo + hi) >> 1;
212
+ if (countTokens(text.slice(mid)) <= maxTokens) {
213
+ cut = mid;
214
+ hi = mid - 1;
215
+ }
216
+ else {
217
+ lo = mid + 1;
218
+ }
219
+ }
220
+ return text.slice(cut);
221
+ }
222
+ /**
223
+ * MAX 초과 unit 강제 분할. 표면 행 단위(header 반복), 아니면 문장 budget → 문자 hard cut.
224
+ * 모든 반환 unit 은 maxTokens 이하 보장.
225
+ */
226
+ function splitLongText(text, maxTokens) {
227
+ if (countTokens(text) <= maxTokens)
228
+ return [text];
229
+ // 표는 행 단위 분리 (각 조각 header prepend)
230
+ if (isTableBlock(text))
231
+ return splitTable(text, maxTokens);
232
+ const out = [];
233
+ let buf = [];
234
+ let bufTokens = 0;
235
+ const flush = () => {
236
+ if (buf.length > 0) {
237
+ out.push(buf.join(' '));
238
+ buf = [];
239
+ bufTokens = 0;
240
+ }
241
+ };
242
+ for (const sentence of splitSentences(text)) {
243
+ const st = countTokens(sentence);
244
+ if (st > maxTokens) {
245
+ flush();
246
+ out.push(...hardCutByChars(sentence, maxTokens));
247
+ continue;
248
+ }
249
+ if (bufTokens + st > maxTokens && buf.length > 0)
250
+ flush();
251
+ buf.push(sentence);
252
+ bufTokens += st;
253
+ }
254
+ flush();
255
+ return out;
256
+ }
257
+ /** heading line 의 레벨(#개수)+텍스트 추출 — breadcrumb(상위 > 하위) 구성용. */
258
+ function extractHeading(text) {
259
+ const m = text.match(/^(#+)\s+(.+)$/m);
260
+ return m ? { level: m[1].length, text: m[2].trim() } : null;
261
+ }
262
+ /**
263
+ * 전체 markdown → 청크 array. heading context + 이미지/표/코드/frontmatter 메타 보존.
264
+ */
265
+ export function chunkMarkdown(markdown) {
266
+ const { frontmatter, body } = parseFrontmatter(markdown);
267
+ const docMeta = parseDocMeta(frontmatter);
268
+ if (body.trim().length === 0)
269
+ return [];
270
+ const sections = body.split(/\n(?=#+ )/);
271
+ const chunks = [];
272
+ const headingStack = []; // index = (heading level - 1) → breadcrumb 상위>하위 맥락 보존
273
+ let currentHeading = '문서 시작';
274
+ for (const section of sections) {
275
+ if (section.trim().length === 0)
276
+ continue;
277
+ const h = extractHeading(section);
278
+ if (h) {
279
+ headingStack.length = h.level - 1; // 같거나 더 깊은 레벨 잘라내고
280
+ headingStack[h.level - 1] = h.text; // 현재 레벨 갱신
281
+ const crumb = headingStack.filter(Boolean).join(' > ');
282
+ if (crumb)
283
+ currentHeading = crumb;
284
+ }
285
+ const tokens = countTokens(section);
286
+ if (tokens <= MAX_TOKENS) {
287
+ chunks.push({ heading: currentHeading, content: section.trim(), tokens });
288
+ continue;
289
+ }
290
+ const paragraphs = splitParagraphs(section).flatMap((p) => countTokens(p) > HARD_MAX_TOKENS ? splitLongText(p, HARD_MAX_TOKENS) : [p]);
291
+ let buf = [];
292
+ let bufTokens = 0;
293
+ for (const para of paragraphs) {
294
+ const pt = countTokens(para);
295
+ if (bufTokens + pt > MAX_TOKENS && buf.length > 0) {
296
+ chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
297
+ buf = [];
298
+ bufTokens = 0;
299
+ }
300
+ buf.push(para);
301
+ bufTokens += pt;
302
+ }
303
+ if (buf.length > 0) {
304
+ chunks.push({ heading: currentHeading, content: buf.join('\n\n').trim(), tokens: bufTokens });
305
+ }
306
+ }
307
+ // MIN_TOKENS 미만 청크 merge
308
+ const merged = [];
309
+ for (const c of chunks) {
310
+ if (c.tokens < MIN_TOKENS && merged.length > 0) {
311
+ const prev = merged[merged.length - 1];
312
+ prev.content = `${prev.content}\n\n${c.content}`;
313
+ prev.tokens += c.tokens;
314
+ }
315
+ else {
316
+ merged.push({ ...c });
317
+ }
318
+ }
319
+ // 1. 이미지 URL 분리 + surrogate strip + 메타 계산 — overlap *전* 고유 content 기준.
320
+ // (overlap 후 계산하면 이전 청크의 이미지/코드가 인접 청크 메타에 오염됨)
321
+ const withMeta = merged.map((c) => {
322
+ const tableCount = countTables(c.content);
323
+ const hasCode = /```/.test(c.content);
324
+ const { content: imgStripped, image_urls } = extractImages(c.content);
325
+ return {
326
+ heading: c.heading,
327
+ content: stripLoneSurrogates(imgStripped),
328
+ image_urls,
329
+ table_count: tableCount,
330
+ has_code_block: hasCode,
331
+ };
332
+ });
333
+ // 2. Overlap prepend — 이미지 URL 이 이미 분리된 텍스트 suffix (overlap 에 URL 안 섞임).
334
+ // ⚠️ tailByTokens 는 토큰 경계로 자르므로 이모지(surrogate pair)를 반토막 낼 수 있다.
335
+ // 이 lone surrogate 가 Qdrant payload content 에 들어가면 invalid UTF-8 JSON → upsert 400.
336
+ // → 아래 3단계에서 최종 content 에 strip 을 한 번 더 적용해 제거한다.
337
+ for (let i = 1; i < withMeta.length; i++) {
338
+ const overlap = tailByTokens(withMeta[i - 1].content, OVERLAP_TOKENS);
339
+ if (overlap)
340
+ withMeta[i].content = `…${overlap}\n\n${withMeta[i].content}`;
341
+ }
342
+ // 3. 최종 RawChunk — overlap 후 재생성된 lone surrogate 재제거(Qdrant 400 방지) + frontmatter 동봉.
343
+ return withMeta.map((c) => {
344
+ const content = stripLoneSurrogates(c.content);
345
+ return {
346
+ heading: c.heading,
347
+ content,
348
+ tokens: countTokens(content),
349
+ image_urls: c.image_urls,
350
+ table_count: c.table_count,
351
+ has_code_block: c.has_code_block,
352
+ ...docMeta,
353
+ };
354
+ });
355
+ }
356
+ //# sourceMappingURL=chunker.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../../src/plugins/drive/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,qEAAqE;AACrE,MAAM,eAAe,GAAG,UAAU,CAAC;AA8BnC,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAElD,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACzC,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAEnD,iFAAiF;IACjF,2EAA2E;IAC3E,IAAI,EAAE,GAA4B,EAAE,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,EAAE,GAAG,MAAiC,CAAC;QACzC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;QACxE,EAAE,GAAG,EAAE,CAAC;IACV,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAChE,CAAC;AAgBD,4DAA4D;AAC5D,SAAS,QAAQ,CAAC,CAAU;IAC1B,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC/C,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;IACnD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,YAAY,IAAI;QAAE,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC;AACd,CAAC;AAED,uEAAuE;AACvE,SAAS,OAAO,CAAC,CAAU;IACzB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5E,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;QAClB,OAAO,CAAC;aACL,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;aAChD,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,+CAA+C;AAC/C,SAAS,YAAY,CAAC,EAA2B;IAC/C,OAAO;QACL,UAAU,EAAE,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC;QAC/B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC;QAC1B,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC;QACxC,WAAW,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC;QAClC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;QAC3B,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC,UAAU,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,uDAAuD;IACvD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AAC7B,CAAC;AAED,iFAAiF;AACjF,sDAAsD;AACtD,MAAM,iBAAiB,GAAG,yEAAyE,CAAC;AACpG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,iFAAiF;AACjF,yCAAyC;AACzC,MAAM,QAAQ,GAAG,wCAAwC,CAAC;AAC1D,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,GAAW,EAAE,GAAW,EAAE,EAAE;QAC5E,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AAChD,CAAC;AAED,0DAA0D;AAC1D,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;AAChE,CAAC;AAED,mDAAmD;AACnD,SAAS,YAAY,CAAC,IAAY;IAChC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACvE,OAAO,SAAS,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;AACzC,CAAC;AAED,sEAAsE;AACtE,SAAS,UAAU,CAAC,KAAa,EAAE,SAAiB;IAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,CAAC,MAAM,CAAC,CAAC;IAC7B,IAAI,MAAM,GAAG,SAAS,CAAC;IACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACzB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YACpB,MAAM,GAAG,SAAS,GAAG,EAAE,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACd,MAAM,IAAI,EAAE,CAAC;QACf,CAAC;IACH,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AACxC,CAAC;AAED,4DAA4D;AAC5D,SAAS,eAAe,CAAC,IAAY;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClD,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,GAAG,CAAC,IAAI,EAAE;gBAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,IAAI,CAAC,CAAC,IAAI,EAAE;oBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,KAAK,GAAG,IAAI;SACf,KAAK,CAAC,oDAAoD,CAAC;SAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY,EAAE,SAAiB;IACrD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC;YACnC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,MAAM;QACR,CAAC;QACD,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;YAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjD,GAAG,GAAG,GAAG,CAAC;gBACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;YACf,CAAC;QACH,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,8DAA8D;AAC9D,SAAS,YAAY,CAAC,IAAY,EAAE,SAAiB;IACnD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAChD,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1B,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QAChB,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,SAAS,EAAE,CAAC;YAC9C,GAAG,GAAG,GAAG,CAAC;YACV,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,IAAY,EAAE,SAAiB;IACpD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAClD,mCAAmC;IACnC,IAAI,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAE3D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,GAAG,GAAa,EAAE,CAAC;IACvB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YACxB,GAAG,GAAG,EAAE,CAAC;YACT,SAAS,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5C,MAAM,EAAE,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjC,IAAI,EAAE,GAAG,SAAS,EAAE,CAAC;YACnB,KAAK,EAAE,CAAC;YACR,GAAG,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;YACjD,SAAS;QACX,CAAC;QACD,IAAI,SAAS,GAAG,EAAE,GAAG,SAAS,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,EAAE,CAAC;QAC1D,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnB,SAAS,IAAI,EAAE,CAAC;IAClB,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+DAA+D;AAC/D,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9D,CAAC;AAQD;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAExC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IACzC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAa,EAAE,CAAC,CAAC,uDAAuD;IAC1F,IAAI,cAAc,GAAG,OAAO,CAAC;IAE7B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAE1C,MAAM,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,CAAC,EAAE,CAAC;YACN,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,mBAAmB;YACtD,YAAY,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW;YAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvD,IAAI,KAAK;gBAAE,cAAc,GAAG,KAAK,CAAC;QACpC,CAAC;QAED,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,MAAM,IAAI,UAAU,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,SAAS;QACX,CAAC;QAED,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CACxD,WAAW,CAAC,CAAC,CAAC,GAAG,eAAe,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC3E,CAAC;QACF,IAAI,GAAG,GAAa,EAAE,CAAC;QACvB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,SAAS,GAAG,EAAE,GAAG,UAAU,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC9F,GAAG,GAAG,EAAE,CAAC;gBACT,SAAS,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,SAAS,IAAI,EAAE,CAAC;QAClB,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;QAChG,CAAC;IACH,CAAC;IAED,yBAAyB;IACzB,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,MAAM,GAAG,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACvC,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,CAAC,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;YACjD,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IAED,uEAAuE;IACvE,mDAAmD;IACnD,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtE,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO,EAAE,mBAAmB,CAAC,WAAW,CAAC;YACzC,UAAU;YACV,WAAW,EAAE,UAAU;YACvB,cAAc,EAAE,OAAO;SACxB,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,yEAAyE;IACzE,oEAAoE;IACpE,uFAAuF;IACvF,qDAAqD;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACtE,IAAI,OAAO;YAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,OAAO,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC7E,CAAC;IAED,sFAAsF;IACtF,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,OAAO,GAAG,mBAAmB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/C,OAAO;YACL,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,OAAO;YACP,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC;YAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,cAAc,EAAE,CAAC,CAAC,cAAc;YAChC,GAAG,OAAO;SACX,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,13 @@
1
+ /**
2
+ * 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
3
+ *
4
+ * 보수적 규칙만 (의미 손상 방지):
5
+ * - 이미지 markdown ![](url) → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
6
+ * - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
7
+ * - HTML 태그 (<div>, <br>, <span>...) → 텍스트 보존하고 태그만 제거
8
+ * - pdf-parse 페이지 마커 (-- N of M --) → 제거
9
+ * - 과도한 빈 줄·줄끝 공백 → 정규화
10
+ *
11
+ * 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
12
+ */
13
+ export declare function cleanExtractedText(text: string): string;
@@ -0,0 +1,31 @@
1
+ /**
2
+ * 추출 텍스트 정제 — 임베딩/검색 품질을 위해 추출기 노이즈 제거.
3
+ *
4
+ * 보수적 규칙만 (의미 손상 방지):
5
+ * - 이미지 markdown ![](url) → 보존 (chunker 가 image_urls 로 분리. 2026-05-27 RAG 개선)
6
+ * - LaTeX wrapper (\title{}, \section{}) → 내용만 보존
7
+ * - HTML 태그 (<div>, <br>, <span>...) → 텍스트 보존하고 태그만 제거
8
+ * - pdf-parse 페이지 마커 (-- N of M --) → 제거
9
+ * - 과도한 빈 줄·줄끝 공백 → 정규화
10
+ *
11
+ * 수식 delimiter($...$) 는 보존 (실제 수식 가능성). 표(| ... |) 보존.
12
+ */
13
+ export function cleanExtractedText(text) {
14
+ if (!text)
15
+ return '';
16
+ let t = text;
17
+ // 1. 이미지 markdown(![alt](url)) 은 제거하지 않음 — chunker 가 image_urls 로 분리 보존 (2026-05-27)
18
+ // 2. LaTeX wrapper — \title{X} \section{X} \subsection{X} → X
19
+ t = t.replace(/\\(?:title|section|subsection|textbf|textit)\s*\{([^}]*)\}/g, '$1');
20
+ // 3. HTML <br> → 개행, 나머지 태그는 텍스트 보존하고 태그만 제거
21
+ t = t.replace(/<br\s*\/?>/gi, '\n');
22
+ t = t.replace(/<\/?[a-zA-Z][^>]*>/g, '');
23
+ // 4. pdf-parse 페이지 마커 (-- 1 of 22 --) 제거
24
+ t = t.replace(/^\s*--\s*\d+\s+of\s+\d+\s*--\s*$/gim, '');
25
+ // 5. HTML entity 일부 복원
26
+ t = t.replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
27
+ // 6. 줄끝 공백 + 과도한 빈 줄 정규화
28
+ t = t.replace(/[ \t]+\n/g, '\n').replace(/\n{3,}/g, '\n\n');
29
+ return t.trim();
30
+ }
31
+ //# sourceMappingURL=clean_text.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"clean_text.js","sourceRoot":"","sources":["../../../src/plugins/drive/clean_text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,CAAC,GAAG,IAAI,CAAC;IAEb,qFAAqF;IAErF,8DAA8D;IAC9D,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,6DAA6D,EAAE,IAAI,CAAC,CAAC;IAEnF,6CAA6C;IAC7C,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;IAEzC,yCAAyC;IACzC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC;IAEzD,uBAAuB;IACvB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAEjG,yBAAyB;IACzB,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE5D,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC"}
@@ -0,0 +1,94 @@
1
+ export interface DriveFile {
2
+ id?: string | null;
3
+ name?: string | null;
4
+ mimeType?: string | null;
5
+ parents?: string[] | null;
6
+ modifiedTime?: string | null;
7
+ size?: string | null;
8
+ webViewLink?: string | null;
9
+ trashed?: boolean | null;
10
+ owners?: Array<{
11
+ emailAddress?: string | null;
12
+ displayName?: string | null;
13
+ }> | null;
14
+ [field: string]: unknown;
15
+ }
16
+ export interface DriveChange {
17
+ removed?: boolean | null;
18
+ fileId?: string | null;
19
+ file?: DriveFile | null;
20
+ }
21
+ export interface DriveChangeList {
22
+ nextPageToken?: string | null;
23
+ newStartPageToken?: string | null;
24
+ changes?: DriveChange[];
25
+ }
26
+ export interface DriveFileList {
27
+ files?: DriveFile[];
28
+ nextPageToken?: string | null;
29
+ }
30
+ type Params = Record<string, string | number | boolean | undefined>;
31
+ type ResponseType = 'json' | 'text' | 'arraybuffer';
32
+ export interface DriveClient {
33
+ files: {
34
+ list(params: Params): Promise<{
35
+ data: DriveFileList;
36
+ }>;
37
+ /** Metadata, or the file body with `alt: 'media'` and a response type. */
38
+ get(params: Params & {
39
+ fileId: string;
40
+ }, opts?: {
41
+ responseType?: ResponseType;
42
+ }): Promise<{
43
+ data: any;
44
+ }>;
45
+ export(params: Params & {
46
+ fileId: string;
47
+ mimeType: string;
48
+ }, opts?: {
49
+ responseType?: ResponseType;
50
+ }): Promise<{
51
+ data: any;
52
+ }>;
53
+ };
54
+ changes: {
55
+ getStartPageToken(params?: Params): Promise<{
56
+ data: {
57
+ startPageToken?: string | null;
58
+ };
59
+ }>;
60
+ list(params: Params & {
61
+ pageToken: string;
62
+ }): Promise<{
63
+ data: DriveChangeList;
64
+ }>;
65
+ };
66
+ }
67
+ export declare class DriveApiError extends Error {
68
+ /** HTTP status (0 = network or timeout). googleapis put the same value in `code`. */
69
+ readonly code: number;
70
+ constructor(message: string,
71
+ /** HTTP status (0 = network or timeout). googleapis put the same value in `code`. */
72
+ code: number);
73
+ }
74
+ type Credentials = {
75
+ kind: 'service_account';
76
+ clientEmail: string;
77
+ privateKey: string;
78
+ tokenUri: string;
79
+ } | {
80
+ kind: 'refresh_token';
81
+ clientId: string;
82
+ clientSecret: string;
83
+ refreshToken: string;
84
+ tokenUri: string;
85
+ };
86
+ /** Which credentials the env provides. Throws with the options when none are set. */
87
+ export declare function resolveCredentials(env?: NodeJS.ProcessEnv): Credentials;
88
+ /** Builds a client. `apiBase` and credentials are injectable for tests; production uses env and Google URLs. */
89
+ export declare function createDriveClient(creds: Credentials, apiBase?: string): DriveClient;
90
+ /** The process-wide Drive client, built from env on first use. */
91
+ export declare function getDriveClient(): Promise<DriveClient>;
92
+ /** Google Drive query language 의 single-quote escape. */
93
+ export declare function escapeDriveQuery(s: string): string;
94
+ export {};