@chnxthr/knowledge-mcp 0.0.0-stage → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +37 -2
- package/dist/bootstrap.d.ts +1 -0
- package/dist/bootstrap.js +18 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +5 -0
- package/dist/cli.js.map +1 -0
- package/dist/core/auth.d.ts +38 -0
- package/dist/core/auth.js +93 -0
- package/dist/core/auth.js.map +1 -0
- package/dist/core/config.d.ts +259 -0
- package/dist/core/config.js +172 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/context.d.ts +25 -0
- package/dist/core/context.js +37 -0
- package/dist/core/context.js.map +1 -0
- package/dist/core/env.d.ts +2 -0
- package/dist/core/env.js +5 -0
- package/dist/core/env.js.map +1 -0
- package/dist/core/log_mask.d.ts +10 -0
- package/dist/core/log_mask.js +88 -0
- package/dist/core/log_mask.js.map +1 -0
- package/dist/core/oauth.d.ts +46 -0
- package/dist/core/oauth.js +507 -0
- package/dist/core/oauth.js.map +1 -0
- package/dist/core/paths.d.ts +5 -0
- package/dist/core/paths.js +47 -0
- package/dist/core/paths.js.map +1 -0
- package/dist/core/scope.d.ts +28 -0
- package/dist/core/scope.js +54 -0
- package/dist/core/scope.js.map +1 -0
- package/dist/core/upstream.d.ts +30 -0
- package/dist/core/upstream.js +164 -0
- package/dist/core/upstream.js.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +1 -0
- package/dist/plugin/define.d.ts +144 -0
- package/dist/plugin/define.js +44 -0
- package/dist/plugin/define.js.map +1 -0
- package/dist/plugin/load.d.ts +2 -0
- package/dist/plugin/load.js +41 -0
- package/dist/plugin/load.js.map +1 -0
- package/dist/plugin/ping.d.ts +9 -0
- package/dist/plugin/ping.js +29 -0
- package/dist/plugin/ping.js.map +1 -0
- package/dist/plugin/registry.d.ts +62 -0
- package/dist/plugin/registry.js +248 -0
- package/dist/plugin/registry.js.map +1 -0
- package/dist/plugin/result.d.ts +20 -0
- package/dist/plugin/result.js +59 -0
- package/dist/plugin/result.js.map +1 -0
- package/dist/plugins/drive/chunker.d.ts +28 -0
- package/dist/plugins/drive/chunker.js +356 -0
- package/dist/plugins/drive/chunker.js.map +1 -0
- package/dist/plugins/drive/clean_text.d.ts +13 -0
- package/dist/plugins/drive/clean_text.js +31 -0
- package/dist/plugins/drive/clean_text.js.map +1 -0
- package/dist/plugins/drive/client.d.ts +94 -0
- package/dist/plugins/drive/client.js +220 -0
- package/dist/plugins/drive/client.js.map +1 -0
- package/dist/plugins/drive/embeddings.d.ts +15 -0
- package/dist/plugins/drive/embeddings.js +118 -0
- package/dist/plugins/drive/embeddings.js.map +1 -0
- package/dist/plugins/drive/exclude.d.ts +11 -0
- package/dist/plugins/drive/exclude.js +36 -0
- package/dist/plugins/drive/exclude.js.map +1 -0
- package/dist/plugins/drive/extract_text.d.ts +26 -0
- package/dist/plugins/drive/extract_text.js +95 -0
- package/dist/plugins/drive/extract_text.js.map +1 -0
- package/dist/plugins/drive/get_token.d.ts +5 -0
- package/dist/plugins/drive/get_token.js +128 -0
- package/dist/plugins/drive/get_token.js.map +1 -0
- package/dist/plugins/drive/index.d.ts +5 -0
- package/dist/plugins/drive/index.js +72 -0
- package/dist/plugins/drive/index.js.map +1 -0
- package/dist/plugins/drive/indexer.d.ts +69 -0
- package/dist/plugins/drive/indexer.js +204 -0
- package/dist/plugins/drive/indexer.js.map +1 -0
- package/dist/plugins/drive/mathpix.d.ts +4 -0
- package/dist/plugins/drive/mathpix.js +102 -0
- package/dist/plugins/drive/mathpix.js.map +1 -0
- package/dist/plugins/drive/ooxml.d.ts +17 -0
- package/dist/plugins/drive/ooxml.js +152 -0
- package/dist/plugins/drive/ooxml.js.map +1 -0
- package/dist/plugins/drive/path.d.ts +40 -0
- package/dist/plugins/drive/path.js +87 -0
- package/dist/plugins/drive/path.js.map +1 -0
- package/dist/plugins/drive/qdrant.d.ts +123 -0
- package/dist/plugins/drive/qdrant.js +148 -0
- package/dist/plugins/drive/qdrant.js.map +1 -0
- package/dist/plugins/drive/reindex.d.ts +2 -0
- package/dist/plugins/drive/reindex.js +263 -0
- package/dist/plugins/drive/reindex.js.map +1 -0
- package/dist/plugins/drive/settings.d.ts +11 -0
- package/dist/plugins/drive/settings.js +13 -0
- package/dist/plugins/drive/settings.js.map +1 -0
- package/dist/plugins/drive/sync.d.ts +16 -0
- package/dist/plugins/drive/sync.js +139 -0
- package/dist/plugins/drive/sync.js.map +1 -0
- package/dist/plugins/drive/sync_state.d.ts +3 -0
- package/dist/plugins/drive/sync_state.js +26 -0
- package/dist/plugins/drive/sync_state.js.map +1 -0
- package/dist/plugins/drive/tools/list_drive_folder.d.ts +104 -0
- package/dist/plugins/drive/tools/list_drive_folder.js +221 -0
- package/dist/plugins/drive/tools/list_drive_folder.js.map +1 -0
- package/dist/plugins/drive/tools/read_drive_file.d.ts +66 -0
- package/dist/plugins/drive/tools/read_drive_file.js +206 -0
- package/dist/plugins/drive/tools/read_drive_file.js.map +1 -0
- package/dist/plugins/drive/tools/search_drive.d.ts +52 -0
- package/dist/plugins/drive/tools/search_drive.js +78 -0
- package/dist/plugins/drive/tools/search_drive.js.map +1 -0
- package/dist/plugins/drive/tools/semantic_search_drive.d.ts +109 -0
- package/dist/plugins/drive/tools/semantic_search_drive.js +228 -0
- package/dist/plugins/drive/tools/semantic_search_drive.js.map +1 -0
- package/dist/plugins/github/index.d.ts +1 -0
- package/dist/plugins/github/index.js +27 -0
- package/dist/plugins/github/index.js.map +1 -0
- package/dist/plugins/github/repo_tree.d.ts +57 -0
- package/dist/plugins/github/repo_tree.js +123 -0
- package/dist/plugins/github/repo_tree.js.map +1 -0
- package/dist/plugins/index.d.ts +11 -0
- package/dist/plugins/index.js +7 -0
- package/dist/plugins/index.js.map +1 -0
- package/dist/plugins/railway/index.d.ts +1 -0
- package/dist/plugins/railway/index.js +21 -0
- package/dist/plugins/railway/index.js.map +1 -0
- package/dist/plugins/railway/tools.d.ts +114 -0
- package/dist/plugins/railway/tools.js +165 -0
- package/dist/plugins/railway/tools.js.map +1 -0
- package/dist/plugins/slack/api.d.ts +63 -0
- package/dist/plugins/slack/api.js +280 -0
- package/dist/plugins/slack/api.js.map +1 -0
- package/dist/plugins/slack/index.d.ts +1 -0
- package/dist/plugins/slack/index.js +80 -0
- package/dist/plugins/slack/index.js.map +1 -0
- package/dist/plugins/slack/tools/slack_files.d.ts +87 -0
- package/dist/plugins/slack/tools/slack_files.js +204 -0
- package/dist/plugins/slack/tools/slack_files.js.map +1 -0
- package/dist/plugins/slack/tools/slack_list_channels.d.ts +46 -0
- package/dist/plugins/slack/tools/slack_list_channels.js +197 -0
- package/dist/plugins/slack/tools/slack_list_channels.js.map +1 -0
- package/dist/plugins/slack/tools/slack_read_channel.d.ts +56 -0
- package/dist/plugins/slack/tools/slack_read_channel.js +150 -0
- package/dist/plugins/slack/tools/slack_read_channel.js.map +1 -0
- package/dist/plugins/slack/tools/slack_search.d.ts +53 -0
- package/dist/plugins/slack/tools/slack_search.js +132 -0
- package/dist/plugins/slack/tools/slack_search.js.map +1 -0
- package/dist/run.d.ts +8 -0
- package/dist/run.js +270 -0
- package/dist/run.js.map +1 -0
- package/dist/server/http.d.ts +5 -0
- package/dist/server/http.js +406 -0
- package/dist/server/http.js.map +1 -0
- package/dist/server/identity.d.ts +17 -0
- package/dist/server/identity.js +47 -0
- package/dist/server/identity.js.map +1 -0
- package/dist/server/mcp.d.ts +7 -0
- package/dist/server/mcp.js +55 -0
- package/dist/server/mcp.js.map +1 -0
- package/dist/server/stdio.d.ts +2 -0
- package/dist/server/stdio.js +40 -0
- package/dist/server/stdio.js.map +1 -0
- package/dist/store/index.d.ts +11 -0
- package/dist/store/index.js +31 -0
- package/dist/store/index.js.map +1 -0
- package/dist/store/keys.d.ts +9 -0
- package/dist/store/keys.js +32 -0
- package/dist/store/keys.js.map +1 -0
- package/dist/store/sqlite.d.ts +38 -0
- package/dist/store/sqlite.js +316 -0
- package/dist/store/sqlite.js.map +1 -0
- package/dist/store/supabase.d.ts +35 -0
- package/dist/store/supabase.js +269 -0
- package/dist/store/supabase.js.map +1 -0
- package/dist/store/supabase_client.d.ts +10 -0
- package/dist/store/supabase_client.js +31 -0
- package/dist/store/supabase_client.js.map +1 -0
- package/dist/store/tool_log.d.ts +12 -0
- package/dist/store/tool_log.js +8 -0
- package/dist/store/tool_log.js.map +1 -0
- package/dist/store/types.d.ts +96 -0
- package/dist/store/types.js +2 -0
- package/dist/store/types.js.map +1 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +11 -0
- package/dist/version.js.map +1 -0
- package/package.json +80 -4
- package/sql/supabase.sql +185 -0
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Google OAuth Refresh Token 발급 자동화.
|
|
3
|
+
*
|
|
4
|
+
* 사용법:
|
|
5
|
+
* 1. .env 에 GOOGLE_OAUTH_CLIENT_ID + GOOGLE_OAUTH_CLIENT_SECRET 채워둠
|
|
6
|
+
* 2. (OAuth Client type 이 Web 이면) Cloud Console → Credentials →
|
|
7
|
+
* OAuth Client → Authorized redirect URIs 에 추가:
|
|
8
|
+
* http://localhost:3500/callback
|
|
9
|
+
* 3. knowledge-mcp drive get-token
|
|
10
|
+
* 4. browser opens -> sign in with your Google account + allow
|
|
11
|
+
* 5. callback 받으면 콘솔에 refresh token 출력
|
|
12
|
+
* 6. .env 에 GOOGLE_OAUTH_REFRESH_TOKEN= 라인 추가
|
|
13
|
+
*/
|
|
14
|
+
import http from 'http';
|
|
15
|
+
import { URL } from 'url';
|
|
16
|
+
import { execFile } from 'child_process';
|
|
17
|
+
const PORT = 3500;
|
|
18
|
+
const REDIRECT_URI = `http://localhost:${PORT}/callback`;
|
|
19
|
+
const SCOPE = 'https://www.googleapis.com/auth/drive.readonly';
|
|
20
|
+
/**
|
|
21
|
+
* `knowledge-mcp drive get-token` — opens Google sign-in and prints a Drive refresh token.
|
|
22
|
+
* The token is written with process.stdout.write: console output is masked and would hide it.
|
|
23
|
+
*/
|
|
24
|
+
export async function runGetToken() {
|
|
25
|
+
const clientId = process.env.GOOGLE_OAUTH_CLIENT_ID;
|
|
26
|
+
const clientSecret = process.env.GOOGLE_OAUTH_CLIENT_SECRET;
|
|
27
|
+
if (!clientId || !clientSecret) {
|
|
28
|
+
console.error('❌ GOOGLE_OAUTH_CLIENT_ID / CLIENT_SECRET 가 .env 에 없습니다.');
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
const authUrl = 'https://accounts.google.com/o/oauth2/v2/auth?' +
|
|
32
|
+
new URLSearchParams({
|
|
33
|
+
client_id: clientId,
|
|
34
|
+
redirect_uri: REDIRECT_URI,
|
|
35
|
+
response_type: 'code',
|
|
36
|
+
scope: SCOPE,
|
|
37
|
+
access_type: 'offline',
|
|
38
|
+
prompt: 'consent', // refresh_token 강제 발급
|
|
39
|
+
}).toString();
|
|
40
|
+
console.log('\n========================================');
|
|
41
|
+
console.log(' OAuth Refresh Token 발급');
|
|
42
|
+
console.log('========================================\n');
|
|
43
|
+
console.log('1. Local callback server: http://localhost:' + PORT);
|
|
44
|
+
console.log('2. 브라우저 자동 열림 (또는 아래 URL 수동 접속):');
|
|
45
|
+
console.log('\n' + authUrl + '\n');
|
|
46
|
+
console.log('3. Sign in with your Google account and allow access');
|
|
47
|
+
console.log('4. 자동 redirect 후 콘솔에 refresh token 출력\n');
|
|
48
|
+
const server = http.createServer(async (req, res) => {
|
|
49
|
+
if (!req.url?.startsWith('/callback')) {
|
|
50
|
+
res.writeHead(404);
|
|
51
|
+
res.end();
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
55
|
+
const code = url.searchParams.get('code');
|
|
56
|
+
const error = url.searchParams.get('error');
|
|
57
|
+
if (error) {
|
|
58
|
+
res.writeHead(400, { 'Content-Type': 'text/plain; charset=utf-8' });
|
|
59
|
+
res.end(`OAuth error: ${error}`);
|
|
60
|
+
console.error('\n❌ OAuth error:', error);
|
|
61
|
+
process.exit(1);
|
|
62
|
+
}
|
|
63
|
+
if (!code) {
|
|
64
|
+
res.writeHead(400);
|
|
65
|
+
res.end('No code received');
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
const tokenRes = await fetch('https://oauth2.googleapis.com/token', {
|
|
70
|
+
method: 'POST',
|
|
71
|
+
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
|
72
|
+
body: new URLSearchParams({
|
|
73
|
+
grant_type: 'authorization_code',
|
|
74
|
+
code,
|
|
75
|
+
client_id: clientId,
|
|
76
|
+
client_secret: clientSecret,
|
|
77
|
+
redirect_uri: REDIRECT_URI,
|
|
78
|
+
}),
|
|
79
|
+
});
|
|
80
|
+
const tokens = (await tokenRes.json());
|
|
81
|
+
if (!tokenRes.ok)
|
|
82
|
+
throw new Error(`${tokens.error ?? tokenRes.status} ${tokens.error_description ?? ''}`.trim());
|
|
83
|
+
const refreshToken = tokens.refresh_token;
|
|
84
|
+
if (!refreshToken) {
|
|
85
|
+
console.error('\n❌ refresh_token 미발급. 이미 권한 부여한 경우 일 수 있음.');
|
|
86
|
+
console.error(' 해결: https://myaccount.google.com/connections 에서');
|
|
87
|
+
console.error(' client 제거 후 다시 시도.\n');
|
|
88
|
+
res.writeHead(400);
|
|
89
|
+
res.end('No refresh_token in response');
|
|
90
|
+
process.exit(1);
|
|
91
|
+
}
|
|
92
|
+
console.log('\n========================================');
|
|
93
|
+
console.log(' ✅ 발급 성공');
|
|
94
|
+
console.log('========================================\n');
|
|
95
|
+
console.log('.env 에 다음 라인 추가:\n');
|
|
96
|
+
process.stdout.write(`GOOGLE_OAUTH_REFRESH_TOKEN=${refreshToken}\n\n`);
|
|
97
|
+
res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
|
|
98
|
+
res.end(`<!doctype html><html><body style="font-family:sans-serif;padding:40px;">
|
|
99
|
+
<h2>✅ Refresh Token 발급 완료</h2>
|
|
100
|
+
<p>터미널에서 token 을 확인하고 <code>.env</code> 에 추가하세요.</p>
|
|
101
|
+
<p>이 창은 닫으셔도 됩니다.</p>
|
|
102
|
+
</body></html>`);
|
|
103
|
+
setTimeout(() => {
|
|
104
|
+
server.close();
|
|
105
|
+
process.exit(0);
|
|
106
|
+
}, 1000);
|
|
107
|
+
}
|
|
108
|
+
catch (e) {
|
|
109
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
110
|
+
console.error('\n❌ Token exchange failed:', msg);
|
|
111
|
+
res.writeHead(500);
|
|
112
|
+
res.end(`Error: ${msg}`);
|
|
113
|
+
process.exit(1);
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
server.listen(PORT, () => {
|
|
117
|
+
console.log(`✓ Server listening on :${PORT}\n`);
|
|
118
|
+
// 브라우저 자동 열기 (macOS open, Linux xdg-open, Windows start)
|
|
119
|
+
const opener = process.platform === 'darwin' ? ['open', [authUrl]] : process.platform === 'win32' ? ['cmd', ['/c', 'start', '', authUrl]] : ['xdg-open', [authUrl]];
|
|
120
|
+
execFile(opener[0], opener[1], (err) => {
|
|
121
|
+
if (err) {
|
|
122
|
+
console.log('(브라우저 자동 열기 실패. 위 URL 을 수동 복사하세요)');
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
});
|
|
126
|
+
await new Promise(() => { }); // 콜백이 process.exit 로 끝낸다
|
|
127
|
+
}
|
|
128
|
+
//# sourceMappingURL=get_token.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get_token.js","sourceRoot":"","sources":["../../../src/plugins/drive/get_token.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,GAAG,EAAE,MAAM,KAAK,CAAC;AAC1B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,MAAM,IAAI,GAAG,IAAI,CAAC;AAClB,MAAM,YAAY,GAAG,oBAAoB,IAAI,WAAW,CAAC;AACzD,MAAM,KAAK,GAAG,gDAAgD,CAAC;AAE/D;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC;IACpD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;IAE5D,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,yDAAyD,CAAC,CAAC;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,OAAO,GACX,+CAA+C;QAC/C,IAAI,eAAe,CAAC;YAClB,SAAS,EAAE,QAAQ;YACnB,YAAY,EAAE,YAAY;YAC1B,aAAa,EAAE,MAAM;YACrB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,SAAS;YACtB,MAAM,EAAE,SAAS,EAAE,sBAAsB;SAC1C,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEhB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;IAC1D,OAAO,CAAC,GAAG,CAAC,6CAA6C,GAAG,IAAI,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,kCAAkC,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;IACpE,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;IAEvD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QAClD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YACtC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,oBAAoB,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,IAAI,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE5C,IAAI,KAAK,EAAE,CAAC;YACV,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,2BAA2B,EAAE,CAAC,CAAC;YACpE,GAAG,CAAC,GAAG,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,qCAAqC,EAAE;gBAClE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,mCAAmC,EAAE;gBAChE,IAAI,EAAE,IAAI,eAAe,CAAC;oBACxB,UAAU,EAAE,oBAAoB;oBAChC,IAAI;oBACJ,SAAS,EAAE,QAAQ;oBACnB,aAAa,EAAE,YAAY;oBAC3B,YAAY,EAAE,YAAY;iBAC3B,CAAC;aACH,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2E,CAAC;YACjH,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,IAAI,MAAM,CAAC,iBAAiB,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACjH,MAAM,YAAY,GAAG,MAAM,CAAC,aAAa,CAAC;YAE1C,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,OAAO,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;gBACpE,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAC/C,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;gBACnB,GAAG,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;gBACxC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YAC1D,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,8BAA8B,YAAY,MAAM,CAAC,CAAC;YAEvE,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,0BAA0B,EAAE,CAAC,CAAC;YACnE,GAAG,CAAC,GAAG,CAAC;;;;qBAIO,CAAC,CAAC;YAEjB,UAAU,CAAC,GAAG,EAAE;gBACd,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACX,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACvD,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,CAAC,CAAC;YACjD,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACnB,GAAG,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QACvB,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,IAAI,CAAC,CAAC;QAChD,yDAAyD;QACzD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACpK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAW,EAAE,MAAM,CAAC,CAAC,CAAa,EAAE,CAAC,GAAG,EAAE,EAAE;YAC3D,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;YACnD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,OAAO,CAAQ,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;AAC/D,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Drive API credentials present. Matches client.ts. */
|
|
2
|
+
export declare function driveConfigured(): boolean;
|
|
3
|
+
/** semantic_search_drive — Qdrant (vectors) and OpenAI (embeddings). Independent of the Drive API credentials. */
|
|
4
|
+
export declare function semanticConfigured(): boolean;
|
|
5
|
+
export declare const drivePlugin: import("../../plugin/define.js").Plugin;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Google Drive plugin — keyword search, read, folder listing, and semantic (vector) search.
|
|
3
|
+
*
|
|
4
|
+
* Credentials (any one):
|
|
5
|
+
* GOOGLE_SERVICE_ACCOUNT_KEY (JSON) | GOOGLE_APPLICATION_CREDENTIALS (file path)
|
|
6
|
+
* | GOOGLE_OAUTH_CLIENT_ID + GOOGLE_OAUTH_CLIENT_SECRET + GOOGLE_OAUTH_REFRESH_TOKEN
|
|
7
|
+
* Semantic search also needs QDRANT_URL and OPENAI_API_KEY, and an index (`knowledge-mcp drive reindex`).
|
|
8
|
+
* Scope comes from the config file: drive.roots (folders to search) and drive.excludeFolders.
|
|
9
|
+
*/
|
|
10
|
+
import { definePlugin } from '../../plugin/define.js';
|
|
11
|
+
import { isSet as has } from '../../core/env.js';
|
|
12
|
+
import { searchDriveTool } from './tools/search_drive.js';
|
|
13
|
+
import { readDriveFileTool } from './tools/read_drive_file.js';
|
|
14
|
+
import { listDriveFolderTool } from './tools/list_drive_folder.js';
|
|
15
|
+
import { semanticSearchDriveTool } from './tools/semantic_search_drive.js';
|
|
16
|
+
/** Drive API credentials present. Matches client.ts. */
|
|
17
|
+
export function driveConfigured() {
|
|
18
|
+
if (has(process.env.GOOGLE_SERVICE_ACCOUNT_KEY))
|
|
19
|
+
return true;
|
|
20
|
+
if (has(process.env.GOOGLE_APPLICATION_CREDENTIALS))
|
|
21
|
+
return true;
|
|
22
|
+
const id = process.env.GOOGLE_OAUTH_CLIENT_ID || process.env.DRIVE_CLIENT_ID;
|
|
23
|
+
const secret = process.env.GOOGLE_OAUTH_CLIENT_SECRET || process.env.DRIVE_CLIENT_SECRET;
|
|
24
|
+
const refresh = process.env.GOOGLE_OAUTH_REFRESH_TOKEN || process.env.DRIVE_REFRESH_TOKEN;
|
|
25
|
+
return has(id) && has(secret) && has(refresh);
|
|
26
|
+
}
|
|
27
|
+
/** semantic_search_drive — Qdrant (vectors) and OpenAI (embeddings). Independent of the Drive API credentials. */
|
|
28
|
+
export function semanticConfigured() {
|
|
29
|
+
return has(process.env.QDRANT_URL) && has(process.env.OPENAI_API_KEY);
|
|
30
|
+
}
|
|
31
|
+
const DRIVE_CREDS = 'set GOOGLE_SERVICE_ACCOUNT_KEY, GOOGLE_APPLICATION_CREDENTIALS or GOOGLE_OAUTH_CLIENT_ID/SECRET/REFRESH_TOKEN';
|
|
32
|
+
const driveOnly = () => driveConfigured() || DRIVE_CREDS;
|
|
33
|
+
export const drivePlugin = definePlugin({
|
|
34
|
+
id: 'drive',
|
|
35
|
+
description: 'Google Drive search, read and semantic search',
|
|
36
|
+
// On when either half works: the Drive API tools, or semantic search over an existing index.
|
|
37
|
+
enabled: () => driveConfigured() || semanticConfigured() || `${DRIVE_CREDS} (or QDRANT_URL + OPENAI_API_KEY)`,
|
|
38
|
+
tools: [
|
|
39
|
+
{ ...searchDriveTool, enabled: driveOnly },
|
|
40
|
+
{ ...readDriveFileTool, enabled: driveOnly },
|
|
41
|
+
{ ...listDriveFolderTool, enabled: driveOnly },
|
|
42
|
+
{ ...semanticSearchDriveTool, enabled: () => semanticConfigured() || 'missing env QDRANT_URL, OPENAI_API_KEY' },
|
|
43
|
+
],
|
|
44
|
+
instructions: 'Drive: meaning/concept questions → semantic_search_drive; keyword or file name → search_drive; ' +
|
|
45
|
+
'"list all" / folder contents → list_drive_folder(folderName). When a chunk lacks context, read_drive_file with the File ID ' +
|
|
46
|
+
'(fileIds=[...] reads several at once).',
|
|
47
|
+
async start({ log }) {
|
|
48
|
+
// Incremental Drive → Qdrant sync. Needs both halves and is skipped with DRIVE_SYNC_ENABLED=false.
|
|
49
|
+
if (!process.env.QDRANT_URL || process.env.DRIVE_SYNC_ENABLED === 'false' || !driveConfigured())
|
|
50
|
+
return;
|
|
51
|
+
const { ensureCollection } = await import('./qdrant.js');
|
|
52
|
+
const { startDriveSyncScheduler } = await import('./sync.js');
|
|
53
|
+
try {
|
|
54
|
+
await ensureCollection(); // a fresh deploy otherwise fails its first sync silently
|
|
55
|
+
}
|
|
56
|
+
catch (e) {
|
|
57
|
+
log('ensureCollection failed (sync keeps trying):', e);
|
|
58
|
+
}
|
|
59
|
+
startDriveSyncScheduler(parseInt(process.env.DRIVE_SYNC_INTERVAL_MIN || '30', 10));
|
|
60
|
+
},
|
|
61
|
+
commands: {
|
|
62
|
+
reindex: {
|
|
63
|
+
description: 'full Drive → Qdrant index (--dry-run, --recreate, --root <id>, --dev-only, --biz-only)',
|
|
64
|
+
run: async (args) => (await import('./reindex.js')).runReindex(args),
|
|
65
|
+
},
|
|
66
|
+
'get-token': {
|
|
67
|
+
description: 'sign in with Google and print a Drive refresh token (GOOGLE_OAUTH_CLIENT_ID/SECRET)',
|
|
68
|
+
run: async () => (await import('./get_token.js')).runGetToken(),
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/plugins/drive/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,KAAK,IAAI,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAE3E,wDAAwD;AACxD,MAAM,UAAU,eAAe;IAC7B,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7D,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;QAAE,OAAO,IAAI,CAAC;IACjE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC7E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACzF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IAC1F,OAAO,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;AAChD,CAAC;AAED,kHAAkH;AAClH,MAAM,UAAU,kBAAkB;IAChC,OAAO,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,WAAW,GAAG,+GAA+G,CAAC;AACpI,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,eAAe,EAAE,IAAI,WAAW,CAAC;AAEzD,MAAM,CAAC,MAAM,WAAW,GAAG,YAAY,CAAC;IACtC,EAAE,EAAE,OAAO;IACX,WAAW,EAAE,+CAA+C;IAC5D,6FAA6F;IAC7F,OAAO,EAAE,GAAG,EAAE,CAAC,eAAe,EAAE,IAAI,kBAAkB,EAAE,IAAI,GAAG,WAAW,mCAAmC;IAC7G,KAAK,EAAE;QACL,EAAE,GAAG,eAAe,EAAE,OAAO,EAAE,SAAS,EAAE;QAC1C,EAAE,GAAG,iBAAiB,EAAE,OAAO,EAAE,SAAS,EAAE;QAC5C,EAAE,GAAG,mBAAmB,EAAE,OAAO,EAAE,SAAS,EAAE;QAC9C,EAAE,GAAG,uBAAuB,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,kBAAkB,EAAE,IAAI,wCAAwC,EAAE;KAChH;IACD,YAAY,EACV,iGAAiG;QACjG,6HAA6H;QAC7H,wCAAwC;IAC1C,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE;QACjB,mGAAmG;QACnG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,IAAI,CAAC,eAAe,EAAE;YAAE,OAAO;QACxG,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QACzD,MAAM,EAAE,uBAAuB,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,gBAAgB,EAAE,CAAC,CAAC,yDAAyD;QACrF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,GAAG,CAAC,8CAA8C,EAAE,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,uBAAuB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;IACrF,CAAC;IACD,QAAQ,EAAE;QACR,OAAO,EAAE;YACP,WAAW,EAAE,wFAAwF;YACrG,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC;SACrE;QACD,WAAW,EAAE;YACX,WAAW,EAAE,qFAAqF;YAClG,GAAG,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,WAAW,EAAE;SAChE;KACF;CACF,CAAC,CAAC"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drive 파일 1개 → Qdrant 색인/삭제 공통 로직.
|
|
3
|
+
* 전체 reindex (reindex.ts) 와 증분 sync (sync.ts) 가 공유.
|
|
4
|
+
*/
|
|
5
|
+
import type { DriveClient } from './client.js';
|
|
6
|
+
import { type RawChunk } from './chunker.js';
|
|
7
|
+
import { type ChunkPayload } from './qdrant.js';
|
|
8
|
+
import { DRIVE_ROOTS, MAX_FILE_BYTES } from './settings.js';
|
|
9
|
+
export { DRIVE_ROOTS, MAX_FILE_BYTES };
|
|
10
|
+
/** 추출 지원 mimeType (extract_text.ts 와 일치). 스캔 PDF/이미지는 Mathpix OCR. 풀 reindex·증분 sync 공용 SSOT. */
|
|
11
|
+
export declare const SUPPORTED_MIMES: Set<string>;
|
|
12
|
+
/** 점진적 chunk ID — file_id + index 면 항상 같은 ID (idempotent upsert). 색인 전 경로 공용. */
|
|
13
|
+
export declare function makeChunkId(fileId: string, chunkIndex: number): number;
|
|
14
|
+
/** path + source -> file_type / cluster. cluster = top-level folder (generic). */
|
|
15
|
+
export declare function classifyFile(path: string, source: 'dev' | 'biz'): {
|
|
16
|
+
type: ChunkPayload['file_type'];
|
|
17
|
+
cluster: string | null;
|
|
18
|
+
};
|
|
19
|
+
export interface IndexableFile {
|
|
20
|
+
id: string;
|
|
21
|
+
name: string;
|
|
22
|
+
path: string;
|
|
23
|
+
modifiedTime: string;
|
|
24
|
+
source: 'dev' | 'biz';
|
|
25
|
+
mimeType: string;
|
|
26
|
+
/** Drive 메타의 바이트 크기. Google 문서는 없음. */
|
|
27
|
+
size?: number;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* 청크의 임베딩 입력 텍스트 — heading breadcrumb(상위>하위 섹션 맥락)를 본문 앞에 붙인다.
|
|
31
|
+
* (contextual chunk header) 부모 heading 은 청크 본문에 없어(다른 섹션) 벡터가 섹션 맥락을 잃는데,
|
|
32
|
+
* 이를 임베딩에 포함하면 recall 이 오른다. 저장·표시용 content(payload)는 그대로 두고 임베딩 입력에만 적용.
|
|
33
|
+
* '문서 시작'(첫 heading 이전 placeholder)은 의미 없어 skip. ★증분(indexFile)·전체(reindex.ts) 공용 SSOT —
|
|
34
|
+
* 두 경로가 반드시 같은 입력을 임베딩해야 같은 청크가 같은 벡터를 가진다(벡터 공간 정합).
|
|
35
|
+
*/
|
|
36
|
+
export declare function embedInputForChunk(chunk: RawChunk): string;
|
|
37
|
+
/** 추출 텍스트 → sha256 (무변경 재색인 skip 판정용). */
|
|
38
|
+
export declare function contentHash(text: string): string;
|
|
39
|
+
export interface BizMeta {
|
|
40
|
+
doc_category: string | null;
|
|
41
|
+
doc_customer: string | null;
|
|
42
|
+
path_tags: string[];
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* biz 소스(PDF/docx — frontmatter 없음)의 경로/파일명에서 메타 파생.
|
|
46
|
+
* EXTRACTED — 사람이 폴더로 분류한 사실(LLM 추론 아님). dev 소스는 frontmatter 가 SSOT 라 빈 값.
|
|
47
|
+
* doc_category : 최상위 폴더(NN_ 제거) 예) "01_고객사/…" → "고객사"
|
|
48
|
+
* doc_customer : 01_고객사/{name} 예) "01_고객사/Acme/…" → "Acme"
|
|
49
|
+
* path_tags : 폴더 세그먼트(NN_ 제거) + 파일명 토큰 (검색 태그)
|
|
50
|
+
*/
|
|
51
|
+
export declare function deriveBizMeta(path: string, source: 'dev' | 'biz'): BizMeta;
|
|
52
|
+
/**
|
|
53
|
+
* chunk + 파일/분류/경로 메타 → ChunkPayload. reindex(reindex.ts)·증분(indexFile) 공유 SSOT.
|
|
54
|
+
* dev 메타는 chunk(frontmatter 파싱)에, biz 메타는 bizMeta(경로 파싱)에 담겨 들어온다.
|
|
55
|
+
*/
|
|
56
|
+
export declare function buildChunkPayload(chunk: RawChunk, file: IndexableFile, idx: number, chunkCount: number, classification: {
|
|
57
|
+
type: ChunkPayload['file_type'];
|
|
58
|
+
cluster: string | null;
|
|
59
|
+
}, bizMeta: BizMeta, hash: string): ChunkPayload;
|
|
60
|
+
/**
|
|
61
|
+
* 파일 1개 형식별 추출 → chunk → embed → Qdrant upsert. 색인된 chunk 수 반환 (skip 시 0).
|
|
62
|
+
*
|
|
63
|
+
* - EFF-1: 추출 텍스트 sha256 이 기존 색인과 동일하면 embed/upsert 생략(무변경 skip).
|
|
64
|
+
* - B-1: 재색인 시 청크 수가 줄면 옛 청크(chunk_index >= 새 수)를 삭제해 유령 청크(폐기된 사실) 잔류 방지.
|
|
65
|
+
* - B-9: 증분 경로는 wait:true 로 색인 직후 검색 가시성 + 실패 인지.
|
|
66
|
+
*/
|
|
67
|
+
export declare function indexFile(drive: DriveClient, file: IndexableFile): Promise<number>;
|
|
68
|
+
/** 특정 파일의 모든 chunk 를 Qdrant 에서 삭제 (파일 삭제·trashed 시). */
|
|
69
|
+
export declare function deleteFileVectors(fileId: string): Promise<void>;
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { createHash } from 'crypto';
|
|
2
|
+
import { chunkMarkdown } from './chunker.js';
|
|
3
|
+
import { embedTexts } from './embeddings.js';
|
|
4
|
+
import { extractText } from './extract_text.js';
|
|
5
|
+
import { getQdrantClient, DRIVE_COLLECTION } from './qdrant.js';
|
|
6
|
+
import { DRIVE_ROOTS, MAX_FILE_BYTES } from './settings.js';
|
|
7
|
+
export { DRIVE_ROOTS, MAX_FILE_BYTES };
|
|
8
|
+
/** 추출 지원 mimeType (extract_text.ts 와 일치). 스캔 PDF/이미지는 Mathpix OCR. 풀 reindex·증분 sync 공용 SSOT. */
|
|
9
|
+
export const SUPPORTED_MIMES = new Set([
|
|
10
|
+
'text/markdown',
|
|
11
|
+
'text/plain',
|
|
12
|
+
'application/vnd.google-apps.document',
|
|
13
|
+
'application/vnd.google-apps.spreadsheet',
|
|
14
|
+
'application/vnd.google-apps.presentation',
|
|
15
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document', // .docx
|
|
16
|
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation', // .pptx
|
|
17
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', // .xlsx
|
|
18
|
+
'application/pdf',
|
|
19
|
+
'image/png',
|
|
20
|
+
'image/jpeg',
|
|
21
|
+
'image/jpg',
|
|
22
|
+
'image/webp',
|
|
23
|
+
]);
|
|
24
|
+
/** 점진적 chunk ID — file_id + index 면 항상 같은 ID (idempotent upsert). 색인 전 경로 공용. */
|
|
25
|
+
export function makeChunkId(fileId, chunkIndex) {
|
|
26
|
+
const hash = createHash('md5').update(`${fileId}:${chunkIndex}`).digest();
|
|
27
|
+
return Number(hash.readBigUInt64BE(0) & 0x7fffffffffffffffn);
|
|
28
|
+
}
|
|
29
|
+
/** path + source -> file_type / cluster. cluster = top-level folder (generic). */
|
|
30
|
+
export function classifyFile(path, source) {
|
|
31
|
+
const m = path.match(/^([^/]+)\//);
|
|
32
|
+
const topFolder = m ? m[1] : null;
|
|
33
|
+
return { type: source === 'biz' ? 'biz' : 'other', cluster: topFolder };
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* 청크의 임베딩 입력 텍스트 — heading breadcrumb(상위>하위 섹션 맥락)를 본문 앞에 붙인다.
|
|
37
|
+
* (contextual chunk header) 부모 heading 은 청크 본문에 없어(다른 섹션) 벡터가 섹션 맥락을 잃는데,
|
|
38
|
+
* 이를 임베딩에 포함하면 recall 이 오른다. 저장·표시용 content(payload)는 그대로 두고 임베딩 입력에만 적용.
|
|
39
|
+
* '문서 시작'(첫 heading 이전 placeholder)은 의미 없어 skip. ★증분(indexFile)·전체(reindex.ts) 공용 SSOT —
|
|
40
|
+
* 두 경로가 반드시 같은 입력을 임베딩해야 같은 청크가 같은 벡터를 가진다(벡터 공간 정합).
|
|
41
|
+
*/
|
|
42
|
+
export function embedInputForChunk(chunk) {
|
|
43
|
+
const h = chunk.heading?.trim();
|
|
44
|
+
if (!h || h === '문서 시작')
|
|
45
|
+
return chunk.content;
|
|
46
|
+
return `${h}\n\n${chunk.content}`;
|
|
47
|
+
}
|
|
48
|
+
/** 추출 텍스트 → sha256 (무변경 재색인 skip 판정용). */
|
|
49
|
+
export function contentHash(text) {
|
|
50
|
+
return createHash('sha256').update(text).digest('hex');
|
|
51
|
+
}
|
|
52
|
+
// 폴더/파일명 앞 번호 prefix("01_", "06_", "0316_") 제거용.
|
|
53
|
+
const NN_PREFIX = /^\d{1,4}[_-]/;
|
|
54
|
+
/**
|
|
55
|
+
* biz 소스(PDF/docx — frontmatter 없음)의 경로/파일명에서 메타 파생.
|
|
56
|
+
* EXTRACTED — 사람이 폴더로 분류한 사실(LLM 추론 아님). dev 소스는 frontmatter 가 SSOT 라 빈 값.
|
|
57
|
+
* doc_category : 최상위 폴더(NN_ 제거) 예) "01_고객사/…" → "고객사"
|
|
58
|
+
* doc_customer : 01_고객사/{name} 예) "01_고객사/Acme/…" → "Acme"
|
|
59
|
+
* path_tags : 폴더 세그먼트(NN_ 제거) + 파일명 토큰 (검색 태그)
|
|
60
|
+
*/
|
|
61
|
+
export function deriveBizMeta(path, source) {
|
|
62
|
+
if (source !== 'biz')
|
|
63
|
+
return { doc_category: null, doc_customer: null, path_tags: [] };
|
|
64
|
+
const segs = path.split('/').filter(Boolean);
|
|
65
|
+
const folders = segs.slice(0, -1); // 마지막 = 파일명
|
|
66
|
+
const fileName = segs[segs.length - 1] ?? '';
|
|
67
|
+
const strip = (s) => s.replace(NN_PREFIX, '');
|
|
68
|
+
const category = folders[0] ? strip(folders[0]) : null;
|
|
69
|
+
const customer = folders[0] && folders[0].includes('고객사') && folders[1] ? folders[1] : null;
|
|
70
|
+
const folderTags = folders.map(strip);
|
|
71
|
+
const nameTokens = fileName
|
|
72
|
+
.replace(/\.[a-z0-9]+$/i, '') // 확장자 제거
|
|
73
|
+
.split(/[_\-\s]+/)
|
|
74
|
+
.map((t) => t.trim())
|
|
75
|
+
.filter((t) => t.length >= 2);
|
|
76
|
+
const path_tags = [...new Set([...folderTags, ...nameTokens])].filter(Boolean);
|
|
77
|
+
return { doc_category: category, doc_customer: customer, path_tags };
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* chunk + 파일/분류/경로 메타 → ChunkPayload. reindex(reindex.ts)·증분(indexFile) 공유 SSOT.
|
|
81
|
+
* dev 메타는 chunk(frontmatter 파싱)에, biz 메타는 bizMeta(경로 파싱)에 담겨 들어온다.
|
|
82
|
+
*/
|
|
83
|
+
export function buildChunkPayload(chunk, file, idx, chunkCount, classification, bizMeta, hash) {
|
|
84
|
+
const tags = [...new Set([...chunk.doc_tags, ...bizMeta.path_tags])];
|
|
85
|
+
return {
|
|
86
|
+
chunk_id: `${file.id}_${idx}`,
|
|
87
|
+
file_id: file.id,
|
|
88
|
+
file_name: file.name,
|
|
89
|
+
file_path: file.path,
|
|
90
|
+
heading: chunk.heading,
|
|
91
|
+
chunk_index: idx,
|
|
92
|
+
chunk_count: chunkCount,
|
|
93
|
+
content: chunk.content,
|
|
94
|
+
modified_time: file.modifiedTime,
|
|
95
|
+
content_hash: hash,
|
|
96
|
+
file_type: classification.type,
|
|
97
|
+
cluster: classification.cluster,
|
|
98
|
+
source: file.source,
|
|
99
|
+
image_urls: chunk.image_urls,
|
|
100
|
+
has_image: chunk.image_urls.length > 0,
|
|
101
|
+
table_count: chunk.table_count,
|
|
102
|
+
has_code_block: chunk.has_code_block,
|
|
103
|
+
doc_status: chunk.doc_status,
|
|
104
|
+
doc_tier: chunk.doc_tier,
|
|
105
|
+
doc_date: chunk.doc_date,
|
|
106
|
+
doc_authors: chunk.doc_authors,
|
|
107
|
+
doc_related: chunk.doc_related,
|
|
108
|
+
doc_source_refs: chunk.doc_source_refs,
|
|
109
|
+
doc_affects: chunk.doc_affects,
|
|
110
|
+
doc_products: chunk.doc_products,
|
|
111
|
+
doc_type: chunk.doc_type,
|
|
112
|
+
doc_confidence: chunk.doc_confidence,
|
|
113
|
+
doc_tags: tags,
|
|
114
|
+
doc_category: bizMeta.doc_category,
|
|
115
|
+
doc_customer: bizMeta.doc_customer,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* 파일의 기존 색인 상태(청크0 payload) 조회 — content_hash·청크 수.
|
|
120
|
+
* 못 찾으면 null (미색인). 조회 실패 시 null 취급(안전 — 재색인 진행).
|
|
121
|
+
*/
|
|
122
|
+
async function getIndexedState(fileId) {
|
|
123
|
+
try {
|
|
124
|
+
const got = await (await getQdrantClient()).retrieve(DRIVE_COLLECTION, {
|
|
125
|
+
ids: [makeChunkId(fileId, 0)],
|
|
126
|
+
with_payload: true,
|
|
127
|
+
with_vector: false,
|
|
128
|
+
});
|
|
129
|
+
if (!got || got.length === 0)
|
|
130
|
+
return null;
|
|
131
|
+
const p = got[0].payload;
|
|
132
|
+
return {
|
|
133
|
+
content_hash: p?.content_hash ?? null,
|
|
134
|
+
chunk_count: typeof p?.chunk_count === 'number' ? p.chunk_count : 0,
|
|
135
|
+
file_path: String(p?.file_path ?? ''),
|
|
136
|
+
file_name: String(p?.file_name ?? ''),
|
|
137
|
+
source: String(p?.source ?? ''),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return null; // 조회 실패 = 미확인 → 재색인 진행(안전)
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* 파일 1개 형식별 추출 → chunk → embed → Qdrant upsert. 색인된 chunk 수 반환 (skip 시 0).
|
|
146
|
+
*
|
|
147
|
+
* - EFF-1: 추출 텍스트 sha256 이 기존 색인과 동일하면 embed/upsert 생략(무변경 skip).
|
|
148
|
+
* - B-1: 재색인 시 청크 수가 줄면 옛 청크(chunk_index >= 새 수)를 삭제해 유령 청크(폐기된 사실) 잔류 방지.
|
|
149
|
+
* - B-9: 증분 경로는 wait:true 로 색인 직후 검색 가시성 + 실패 인지.
|
|
150
|
+
*/
|
|
151
|
+
export async function indexFile(drive, file) {
|
|
152
|
+
if (file.size && file.size > MAX_FILE_BYTES) {
|
|
153
|
+
console.error(`[index] skip (too large: ${file.size}B > ${MAX_FILE_BYTES}B): ${file.path}`);
|
|
154
|
+
return 0;
|
|
155
|
+
}
|
|
156
|
+
const { text, needsOcr } = await extractText(drive, file.id, file.mimeType);
|
|
157
|
+
// needsOcr=true 는 이제 미지원 형식만 (PDF 스캔본·이미지는 extract_text 가 Mathpix 로 처리).
|
|
158
|
+
// 미지원 또는 추출 결과 빈 파일 → 색인 skip.
|
|
159
|
+
if (needsOcr || text.trim().length === 0)
|
|
160
|
+
return 0;
|
|
161
|
+
const hash = contentHash(text);
|
|
162
|
+
// EFF-1: 본문·경로·이름이 모두 그대로면 embed/upsert 생략 (sharing 등 메타 change 비용 차단).
|
|
163
|
+
// 옮기거나 이름만 바꾼 파일은 다시 색인한다 — payload 의 file_path 가 직원별 폴더 scope 판정 기준이라
|
|
164
|
+
// 옛 경로가 남으면 옮긴 뒤에도 옛 폴더 권한자에게 검색된다. 경로 태그(doc_tags)도 청크마다 다시 만든다.
|
|
165
|
+
const prev = await getIndexedState(file.id);
|
|
166
|
+
if (prev &&
|
|
167
|
+
prev.content_hash === hash &&
|
|
168
|
+
prev.file_path === file.path &&
|
|
169
|
+
prev.file_name === file.name &&
|
|
170
|
+
prev.source === file.source) {
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
173
|
+
const chunks = chunkMarkdown(text);
|
|
174
|
+
if (chunks.length === 0)
|
|
175
|
+
return 0;
|
|
176
|
+
const classification = classifyFile(file.path, file.source);
|
|
177
|
+
const bizMeta = deriveBizMeta(file.path, file.source);
|
|
178
|
+
// heading breadcrumb 포함 임베딩(embedInputForChunk) — 저장 content 는 그대로, 임베딩 입력만 섹션 맥락 부여.
|
|
179
|
+
const vectors = await embedTexts(chunks.map(embedInputForChunk), 'RETRIEVAL_DOCUMENT');
|
|
180
|
+
const points = chunks.map((c, idx) => ({
|
|
181
|
+
id: makeChunkId(file.id, idx),
|
|
182
|
+
vector: vectors[idx],
|
|
183
|
+
payload: buildChunkPayload(c, file, idx, chunks.length, classification, bizMeta, hash),
|
|
184
|
+
}));
|
|
185
|
+
const client = await getQdrantClient();
|
|
186
|
+
// 새 청크 0..M-1 upsert (wait:true — 단일 파일이라 비용 작고, 직후 검색 가시성·실패 인지 확보)
|
|
187
|
+
await client.upsert(DRIVE_COLLECTION, { wait: true, points });
|
|
188
|
+
// B-1: 줄어든 경우 옛 청크(index >= M)를 결정적 chunk ID 로 직접 삭제.
|
|
189
|
+
// (Qdrant range 필터는 비인덱싱 payload 에서 400 Bad Request → ID 삭제로 우회. upsert 와 disjoint 집합)
|
|
190
|
+
if (prev && prev.chunk_count > chunks.length) {
|
|
191
|
+
const orphanIds = [];
|
|
192
|
+
for (let i = chunks.length; i < prev.chunk_count; i++)
|
|
193
|
+
orphanIds.push(makeChunkId(file.id, i));
|
|
194
|
+
await client.delete(DRIVE_COLLECTION, { wait: true, points: orphanIds });
|
|
195
|
+
}
|
|
196
|
+
return chunks.length;
|
|
197
|
+
}
|
|
198
|
+
/** 특정 파일의 모든 chunk 를 Qdrant 에서 삭제 (파일 삭제·trashed 시). */
|
|
199
|
+
export async function deleteFileVectors(fileId) {
|
|
200
|
+
await (await getQdrantClient()).delete(DRIVE_COLLECTION, {
|
|
201
|
+
filter: { must: [{ key: 'file_id', match: { value: fileId } }] },
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../../src/plugins/drive/indexer.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,aAAa,EAAiB,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAqB,MAAM,aAAa,CAAC;AAEnF,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE5D,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;AAEvC,iGAAiG;AACjG,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAS;IAC7C,eAAe;IACf,YAAY;IACZ,sCAAsC;IACtC,yCAAyC;IACzC,0CAA0C;IAC1C,yEAAyE,EAAE,QAAQ;IACnF,2EAA2E,EAAE,QAAQ;IACrF,mEAAmE,EAAE,QAAQ;IAC7E,iBAAiB;IACjB,WAAW;IACX,YAAY;IACZ,WAAW;IACX,YAAY;CACb,CAAC,CAAC;AAGH,iFAAiF;AACjF,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,UAAkB;IAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;IAC1E,OAAO,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAAC,CAAC;AAC/D,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,MAAqB;IAErB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAClC,OAAO,EAAE,IAAI,EAAE,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;AAC1E,CAAC;AAaD;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAe;IAChD,MAAM,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;IAChC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IAC9C,OAAO,GAAG,CAAC,OAAO,KAAK,CAAC,OAAO,EAAE,CAAC;AACpC,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAQD,iDAAiD;AACjD,MAAM,SAAS,GAAG,cAAc,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,MAAqB;IAC/D,IAAI,MAAM,KAAK,KAAK;QAAE,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IACvF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY;IAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACvD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5F,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,UAAU,GAAG,QAAQ;SACxB,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,SAAS;SACtC,KAAK,CAAC,UAAU,CAAC;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAChC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/E,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAAe,EACf,IAAmB,EACnB,GAAW,EACX,UAAkB,EAClB,cAA2E,EAC3E,OAAgB,EAChB,IAAY;IAEZ,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO;QACL,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE;QAC7B,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,SAAS,EAAE,IAAI,CAAC,IAAI;QACpB,SAAS,EAAE,IAAI,CAAC,IAAI;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,WAAW,EAAE,GAAG;QAChB,WAAW,EAAE,UAAU;QACvB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,aAAa,EAAE,IAAI,CAAC,YAAY;QAChC,YAAY,EAAE,IAAI;QAClB,SAAS,EAAE,cAAc,CAAC,IAAI;QAC9B,OAAO,EAAE,cAAc,CAAC,OAAO;QAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;QACtC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,eAAe,EAAE,KAAK,CAAC,eAAe;QACtC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,QAAQ,EAAE,IAAI;QACd,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,YAAY,EAAE,OAAO,CAAC,YAAY;KACnC,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,eAAe,CAC5B,MAAc;IAEd,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC,QAAQ,CAAC,gBAAgB,EAAE;YACrE,GAAG,EAAE,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC7B,YAAY,EAAE,IAAI;YAClB,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,OAA8C,CAAC;QAChE,OAAO;YACL,YAAY,EAAG,CAAC,EAAE,YAAuB,IAAI,IAAI;YACjD,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAC,WAAsB,CAAC,CAAC,CAAC,CAAC;YAC/E,SAAS,EAAE,MAAM,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;YACrC,SAAS,EAAE,MAAM,CAAC,CAAC,EAAE,SAAS,IAAI,EAAE,CAAC;YACrC,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,IAAI,EAAE,CAAC;SAChC,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,CAAC,2BAA2B;IAC1C,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAkB,EAAE,IAAmB;IACrE,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,cAAc,EAAE,CAAC;QAC5C,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,IAAI,OAAO,cAAc,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5F,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC5E,yEAAyE;IACzE,+BAA+B;IAC/B,IAAI,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEnD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAE/B,wEAAwE;IACxE,uEAAuE;IACvE,kEAAkE;IAClE,MAAM,IAAI,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5C,IACE,IAAI;QACJ,IAAI,CAAC,YAAY,KAAK,IAAI;QAC1B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,IAAI;QAC5B,IAAI,CAAC,SAAS,KAAK,IAAI,CAAC,IAAI;QAC5B,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAC3B,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAElC,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,sFAAsF;IACtF,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,kBAAkB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QACrC,EAAE,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC;QAC7B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC;QACpB,OAAO,EAAE,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,CAAC;KACvF,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAG,MAAM,eAAe,EAAE,CAAC;IACvC,qEAAqE;IACrE,MAAM,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9D,sDAAsD;IACtD,4FAA4F;IAC5F,IAAI,IAAI,IAAI,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE;YAAE,SAAS,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC/F,MAAM,MAAM,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,MAAc;IACpD,MAAM,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC,MAAM,CAAC,gBAAgB,EAAE;QACvD,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE;KACjE,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** PDF (스캔본) → markdown. async upload + poll + .md 다운로드. */
|
|
2
|
+
export declare function mathpixPdfToMarkdown(buffer: Buffer, maxWaitSec?: number): Promise<string>;
|
|
3
|
+
/** 단일 이미지 → markdown. v3/text (즉시). */
|
|
4
|
+
export declare function mathpixImageToMarkdown(buffer: Buffer, mimeType: string): Promise<string>;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mathpix OCR — 스캔 PDF / 이미지 → markdown.
|
|
3
|
+
*
|
|
4
|
+
* 키: MATHPIX_APP_ID / MATHPIX_APP_KEY.
|
|
5
|
+
*
|
|
6
|
+
* - PDF (스캔본): v3/pdf (async) — upload → poll → /pdf/{id}.md
|
|
7
|
+
* - 이미지: v3/text (즉시)
|
|
8
|
+
*
|
|
9
|
+
* 수식·표 구조 보존 (Mathpix Markdown) → chunk 시 표 안 깨짐 → 검색 품질 ↑.
|
|
10
|
+
*/
|
|
11
|
+
const MATHPIX_API = 'https://api.mathpix.com/v3';
|
|
12
|
+
function mathpixHeaders() {
|
|
13
|
+
const appId = process.env.MATHPIX_APP_ID;
|
|
14
|
+
const appKey = process.env.MATHPIX_APP_KEY;
|
|
15
|
+
if (!appId || !appKey)
|
|
16
|
+
throw new Error('[mathpix] MATHPIX_APP_ID / MATHPIX_APP_KEY 미설정');
|
|
17
|
+
return { app_id: appId, app_key: appKey };
|
|
18
|
+
}
|
|
19
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
20
|
+
/**
|
|
21
|
+
* timeout·재시도 감싼 fetch. 예전엔 어떤 fetch 도 timeout·재시도가 없어, status 폴링이 네트워크
|
|
22
|
+
* 정체 시 deadline 까지 매달리고 일시 5xx 에 전체 OCR 이 실패했다. AbortSignal.timeout + 5xx/네트워크
|
|
23
|
+
* 짧은 재시도로 안정화(OCR 은 유저 대기 경로).
|
|
24
|
+
*/
|
|
25
|
+
async function mpFetch(url, init, timeoutMs = 30_000, retries = 2) {
|
|
26
|
+
for (let attempt = 0;; attempt++) {
|
|
27
|
+
try {
|
|
28
|
+
const resp = await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
|
|
29
|
+
if (resp.status >= 500 && resp.status <= 599 && attempt < retries) {
|
|
30
|
+
await sleep(1000 * (attempt + 1));
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
return resp;
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
// timeout(TimeoutError)·네트워크 오류 — 재시도, 소진 시 throw
|
|
37
|
+
if (attempt < retries) {
|
|
38
|
+
await sleep(1000 * (attempt + 1));
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
throw new Error(`[mathpix] fetch 실패(${url.split('/').slice(-1)[0]}): ${e instanceof Error ? e.message : String(e)}`);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** PDF (스캔본) → markdown. async upload + poll + .md 다운로드. */
|
|
46
|
+
export async function mathpixPdfToMarkdown(buffer, maxWaitSec = 180) {
|
|
47
|
+
const headers = mathpixHeaders();
|
|
48
|
+
// 1. Upload
|
|
49
|
+
const form = new FormData();
|
|
50
|
+
form.append('file', new Blob([buffer], { type: 'application/pdf' }), 'doc.pdf');
|
|
51
|
+
form.append('options_json', JSON.stringify({
|
|
52
|
+
conversion_formats: { md: true },
|
|
53
|
+
math_inline_delimiters: ['$', '$'],
|
|
54
|
+
rm_spaces: false,
|
|
55
|
+
enable_tables_fallback: true,
|
|
56
|
+
include_diagram_text: true, // 다이어그램 내부 텍스트도 OCR (figure URL 은 .md 에 기본 포함)
|
|
57
|
+
}));
|
|
58
|
+
const upResp = await mpFetch(`${MATHPIX_API}/pdf`, { method: 'POST', headers, body: form }, 60_000);
|
|
59
|
+
if (!upResp.ok)
|
|
60
|
+
throw new Error(`[mathpix] upload 실패 HTTP ${upResp.status}: ${(await upResp.text()).slice(0, 200)}`);
|
|
61
|
+
const { pdf_id } = (await upResp.json());
|
|
62
|
+
if (!pdf_id)
|
|
63
|
+
throw new Error('[mathpix] pdf_id 없음');
|
|
64
|
+
// 2. Poll (2s 간격)
|
|
65
|
+
const deadline = Date.now() + maxWaitSec * 1000;
|
|
66
|
+
while (Date.now() < deadline) {
|
|
67
|
+
await sleep(2000);
|
|
68
|
+
const stResp = await mpFetch(`${MATHPIX_API}/pdf/${pdf_id}`, { headers });
|
|
69
|
+
const stData = (await stResp.json());
|
|
70
|
+
if (stData.status === 'completed') {
|
|
71
|
+
// 3. .md 다운로드
|
|
72
|
+
const mdResp = await mpFetch(`${MATHPIX_API}/pdf/${pdf_id}.md`, { headers });
|
|
73
|
+
const md = await mdResp.text();
|
|
74
|
+
if (!md.trim())
|
|
75
|
+
throw new Error('[mathpix] completed but empty markdown');
|
|
76
|
+
return md;
|
|
77
|
+
}
|
|
78
|
+
if (stData.status === 'error')
|
|
79
|
+
throw new Error('[mathpix] 처리 error');
|
|
80
|
+
}
|
|
81
|
+
throw new Error(`[mathpix] timeout ${maxWaitSec}s (pdf_id=${pdf_id})`);
|
|
82
|
+
}
|
|
83
|
+
/** 단일 이미지 → markdown. v3/text (즉시). */
|
|
84
|
+
export async function mathpixImageToMarkdown(buffer, mimeType) {
|
|
85
|
+
const headers = { ...mathpixHeaders(), 'Content-Type': 'application/json' };
|
|
86
|
+
const b64 = buffer.toString('base64');
|
|
87
|
+
const resp = await mpFetch(`${MATHPIX_API}/text`, {
|
|
88
|
+
method: 'POST',
|
|
89
|
+
headers,
|
|
90
|
+
body: JSON.stringify({
|
|
91
|
+
src: `data:${mimeType};base64,${b64}`,
|
|
92
|
+
formats: ['text', 'data'],
|
|
93
|
+
math_inline_delimiters: ['$', '$'],
|
|
94
|
+
rm_spaces: false,
|
|
95
|
+
}),
|
|
96
|
+
});
|
|
97
|
+
if (!resp.ok)
|
|
98
|
+
throw new Error(`[mathpix] image 실패 HTTP ${resp.status}`);
|
|
99
|
+
const data = (await resp.json());
|
|
100
|
+
return data.text ?? '';
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=mathpix.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mathpix.js","sourceRoot":"","sources":["../../../src/plugins/drive/mathpix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,MAAM,WAAW,GAAG,4BAA4B,CAAC;AAEjD,SAAS,cAAc;IACrB,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;IAC3C,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACzF,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;AAC5C,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpE;;;;GAIG;AACH,KAAK,UAAU,OAAO,CACpB,GAAW,EACX,IAAiB,EACjB,SAAS,GAAG,MAAM,EAClB,OAAO,GAAG,CAAC;IAEX,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YACnF,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBAClE,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,kDAAkD;YAClD,IAAI,OAAO,GAAG,OAAO,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,4DAA4D;AAC5D,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,MAAc,EAAE,UAAU,GAAG,GAAG;IACzE,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IAEjC,YAAY;IACZ,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAChF,IAAI,CAAC,MAAM,CACT,cAAc,EACd,IAAI,CAAC,SAAS,CAAC;QACb,kBAAkB,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE;QAChC,sBAAsB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;QAClC,SAAS,EAAE,KAAK;QAChB,sBAAsB,EAAE,IAAI;QAC5B,oBAAoB,EAAE,IAAI,EAAE,+CAA+C;KAC5E,CAAC,CACH,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,WAAW,MAAM,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;IACpG,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,MAAM,CAAC,MAAM,KAAK,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACrH,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAwB,CAAC;IAChE,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAEpD,kBAAkB;IAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,GAAG,IAAI,CAAC;IAChD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QAClB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,WAAW,QAAQ,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAwB,CAAC;QAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,cAAc;YACd,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,WAAW,QAAQ,MAAM,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7E,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC1E,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,aAAa,MAAM,GAAG,CAAC,CAAC;AACzE,CAAC;AAED,uCAAuC;AACvC,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,MAAc,EAAE,QAAgB;IAC3E,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC;IAC5E,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,GAAG,WAAW,OAAO,EAAE;QAChD,MAAM,EAAE,MAAM;QACd,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,GAAG,EAAE,QAAQ,QAAQ,WAAW,GAAG,EAAE;YACrC,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;YACzB,sBAAsB,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC;YAClC,SAAS,EAAE,KAAK;SACjB,CAAC;KACH,CAAC,CAAC;IACH,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE,CAAsB,CAAC;IACtD,OAAO,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;AACzB,CAAC"}
|