a2a-memory 0.1.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +262 -45
- package/dist/adapters/anthropic.d.ts +69 -0
- package/dist/adapters/anthropic.d.ts.map +1 -0
- package/dist/adapters/anthropic.js +116 -0
- package/dist/adapters/anthropic.js.map +1 -0
- package/dist/claude/sync.d.ts +57 -0
- package/dist/claude/sync.d.ts.map +1 -0
- package/dist/claude/sync.js +201 -0
- package/dist/claude/sync.js.map +1 -0
- package/dist/cli/commands/add.d.ts +8 -0
- package/dist/cli/commands/add.d.ts.map +1 -0
- package/dist/cli/commands/add.js +45 -0
- package/dist/cli/commands/add.js.map +1 -0
- package/dist/cli/commands/claude-sync.d.ts +11 -0
- package/dist/cli/commands/claude-sync.d.ts.map +1 -0
- package/dist/cli/commands/claude-sync.js +69 -0
- package/dist/cli/commands/claude-sync.js.map +1 -0
- package/dist/cli/commands/cleanup.d.ts +8 -0
- package/dist/cli/commands/cleanup.d.ts.map +1 -0
- package/dist/cli/commands/cleanup.js +82 -0
- package/dist/cli/commands/cleanup.js.map +1 -0
- package/dist/cli/commands/edit.d.ts +8 -0
- package/dist/cli/commands/edit.d.ts.map +1 -0
- package/dist/cli/commands/edit.js +66 -0
- package/dist/cli/commands/edit.js.map +1 -0
- package/dist/cli/commands/embed.d.ts +8 -0
- package/dist/cli/commands/embed.d.ts.map +1 -0
- package/dist/cli/commands/embed.js +93 -0
- package/dist/cli/commands/embed.js.map +1 -0
- package/dist/cli/commands/health.d.ts +8 -0
- package/dist/cli/commands/health.d.ts.map +1 -0
- package/dist/cli/commands/health.js +108 -0
- package/dist/cli/commands/health.js.map +1 -0
- package/dist/cli/commands/rm.d.ts +8 -0
- package/dist/cli/commands/rm.d.ts.map +1 -0
- package/dist/cli/commands/rm.js +60 -0
- package/dist/cli/commands/rm.js.map +1 -0
- package/dist/cli/commands/sync.d.ts.map +1 -1
- package/dist/cli/commands/sync.js +52 -0
- package/dist/cli/commands/sync.js.map +1 -1
- package/dist/cli/commands/team.d.ts +7 -0
- package/dist/cli/commands/team.d.ts.map +1 -0
- package/dist/cli/commands/team.js +144 -0
- package/dist/cli/commands/team.js.map +1 -0
- package/dist/cli/index.d.ts +5 -0
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/cli/index.js +44 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/config/manager.d.ts +16 -0
- package/dist/config/manager.d.ts.map +1 -1
- package/dist/config/manager.js +170 -5
- package/dist/config/manager.js.map +1 -1
- package/dist/db/database.d.ts +28 -3
- package/dist/db/database.d.ts.map +1 -1
- package/dist/db/database.js +291 -64
- package/dist/db/database.js.map +1 -1
- package/dist/embedding/index.d.ts +21 -0
- package/dist/embedding/index.d.ts.map +1 -0
- package/dist/embedding/index.js +29 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/local-provider.d.ts +40 -0
- package/dist/embedding/local-provider.d.ts.map +1 -0
- package/dist/embedding/local-provider.js +157 -0
- package/dist/embedding/local-provider.js.map +1 -0
- package/dist/embedding/openai-provider.d.ts +31 -0
- package/dist/embedding/openai-provider.d.ts.map +1 -0
- package/dist/embedding/openai-provider.js +92 -0
- package/dist/embedding/openai-provider.js.map +1 -0
- package/dist/embedding/quantization.d.ts +34 -0
- package/dist/embedding/quantization.d.ts.map +1 -0
- package/dist/embedding/quantization.js +89 -0
- package/dist/embedding/quantization.js.map +1 -0
- package/dist/extraction/extractor.d.ts +11 -1
- package/dist/extraction/extractor.d.ts.map +1 -1
- package/dist/extraction/extractor.js +63 -20
- package/dist/extraction/extractor.js.map +1 -1
- package/dist/extraction/filter.d.ts.map +1 -1
- package/dist/extraction/filter.js +25 -3
- package/dist/extraction/filter.js.map +1 -1
- package/dist/extraction/scorer.d.ts +2 -0
- package/dist/extraction/scorer.d.ts.map +1 -1
- package/dist/extraction/scorer.js +23 -1
- package/dist/extraction/scorer.js.map +1 -1
- package/dist/extraction/similarity.d.ts +25 -0
- package/dist/extraction/similarity.d.ts.map +1 -0
- package/dist/extraction/similarity.js +85 -0
- package/dist/extraction/similarity.js.map +1 -0
- package/dist/hooks/post-tool-use.d.ts.map +1 -1
- package/dist/hooks/post-tool-use.js +88 -53
- package/dist/hooks/post-tool-use.js.map +1 -1
- package/dist/hooks/session-end.d.ts +1 -0
- package/dist/hooks/session-end.d.ts.map +1 -1
- package/dist/hooks/session-end.js +173 -33
- package/dist/hooks/session-end.js.map +1 -1
- package/dist/hooks/session-start.d.ts +2 -2
- package/dist/hooks/session-start.d.ts.map +1 -1
- package/dist/hooks/session-start.js +172 -22
- package/dist/hooks/session-start.js.map +1 -1
- package/dist/hooks/shared.d.ts +18 -0
- package/dist/hooks/shared.d.ts.map +1 -0
- package/dist/hooks/shared.js +46 -0
- package/dist/hooks/shared.js.map +1 -0
- package/dist/i18n/index.d.ts +3 -0
- package/dist/i18n/index.d.ts.map +1 -0
- package/dist/i18n/index.js +2 -0
- package/dist/i18n/index.js.map +1 -0
- package/dist/i18n/messages.d.ts +82 -0
- package/dist/i18n/messages.d.ts.map +1 -0
- package/dist/i18n/messages.js +150 -0
- package/dist/i18n/messages.js.map +1 -0
- package/dist/index.d.ts +25 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +20 -3
- package/dist/index.js.map +1 -1
- package/dist/lifecycle/cleanup.d.ts +25 -0
- package/dist/lifecycle/cleanup.d.ts.map +1 -0
- package/dist/lifecycle/cleanup.js +52 -0
- package/dist/lifecycle/cleanup.js.map +1 -0
- package/dist/lifecycle/index.d.ts +15 -0
- package/dist/lifecycle/index.d.ts.map +1 -0
- package/dist/lifecycle/index.js +12 -0
- package/dist/lifecycle/index.js.map +1 -0
- package/dist/lifecycle/quality-scorer.d.ts +19 -0
- package/dist/lifecycle/quality-scorer.d.ts.map +1 -0
- package/dist/lifecycle/quality-scorer.js +40 -0
- package/dist/lifecycle/quality-scorer.js.map +1 -0
- package/dist/lifecycle/tiering.d.ts +50 -0
- package/dist/lifecycle/tiering.d.ts.map +1 -0
- package/dist/lifecycle/tiering.js +235 -0
- package/dist/lifecycle/tiering.js.map +1 -0
- package/dist/llm/client.d.ts +37 -0
- package/dist/llm/client.d.ts.map +1 -0
- package/dist/llm/client.js +154 -0
- package/dist/llm/client.js.map +1 -0
- package/dist/llm/index.d.ts +6 -0
- package/dist/llm/index.d.ts.map +1 -0
- package/dist/llm/index.js +5 -0
- package/dist/llm/index.js.map +1 -0
- package/dist/search/index.d.ts +8 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +7 -0
- package/dist/search/index.js.map +1 -0
- package/dist/search/ranker.d.ts +30 -0
- package/dist/search/ranker.d.ts.map +1 -0
- package/dist/search/ranker.js +91 -0
- package/dist/search/ranker.js.map +1 -0
- package/dist/sync/client.d.ts +103 -0
- package/dist/sync/client.d.ts.map +1 -1
- package/dist/sync/client.js +186 -6
- package/dist/sync/client.js.map +1 -1
- package/dist/sync/encryption.d.ts +72 -0
- package/dist/sync/encryption.d.ts.map +1 -0
- package/dist/sync/encryption.js +203 -0
- package/dist/sync/encryption.js.map +1 -0
- package/dist/sync/index.d.ts +9 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +6 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/queue.d.ts +49 -0
- package/dist/sync/queue.d.ts.map +1 -0
- package/dist/sync/queue.js +112 -0
- package/dist/sync/queue.js.map +1 -0
- package/dist/sync/scheduler.d.ts +69 -0
- package/dist/sync/scheduler.d.ts.map +1 -0
- package/dist/sync/scheduler.js +140 -0
- package/dist/sync/scheduler.js.map +1 -0
- package/dist/sync/synchronizer.d.ts +19 -5
- package/dist/sync/synchronizer.d.ts.map +1 -1
- package/dist/sync/synchronizer.js +128 -70
- package/dist/sync/synchronizer.js.map +1 -1
- package/dist/sync/team-synchronizer.d.ts +43 -0
- package/dist/sync/team-synchronizer.d.ts.map +1 -0
- package/dist/sync/team-synchronizer.js +126 -0
- package/dist/sync/team-synchronizer.js.map +1 -0
- package/dist/sync/vector-clock.d.ts +23 -0
- package/dist/sync/vector-clock.d.ts.map +1 -0
- package/dist/sync/vector-clock.js +40 -0
- package/dist/sync/vector-clock.js.map +1 -0
- package/dist/types/index.d.ts +74 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +56 -4
- package/dist/types/index.js.map +1 -1
- package/dist/utils/keychain.d.ts +50 -0
- package/dist/utils/keychain.d.ts.map +1 -0
- package/dist/utils/keychain.js +166 -0
- package/dist/utils/keychain.js.map +1 -0
- package/dist/utils/logger.d.ts +16 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +118 -0
- package/dist/utils/logger.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Module
|
|
3
|
+
*
|
|
4
|
+
* 로컬/OpenAI 임베딩 프로바이더 팩토리.
|
|
5
|
+
*/
|
|
6
|
+
import { LocalEmbeddingProvider } from './local-provider.js';
|
|
7
|
+
import { OpenAIEmbeddingProvider } from './openai-provider.js';
|
|
8
|
+
/**
|
|
9
|
+
* 임베딩 프로바이더 생성 팩토리
|
|
10
|
+
*/
|
|
11
|
+
export function createEmbeddingProvider(config, llmConfig) {
|
|
12
|
+
if (config.provider === 'local') {
|
|
13
|
+
return new LocalEmbeddingProvider(config.dimensions);
|
|
14
|
+
}
|
|
15
|
+
if (config.provider === 'openai') {
|
|
16
|
+
// OpenAI는 llmConfig에서 apiKey 가져옴
|
|
17
|
+
if (!llmConfig?.apiKey) {
|
|
18
|
+
throw new Error('OpenAI 임베딩에는 LLM API Key가 필요합니다');
|
|
19
|
+
}
|
|
20
|
+
return new OpenAIEmbeddingProvider({
|
|
21
|
+
apiKey: llmConfig.apiKey,
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
throw new Error(`Unsupported embedding provider: ${config.provider}`);
|
|
25
|
+
}
|
|
26
|
+
export { LocalEmbeddingProvider } from './local-provider.js';
|
|
27
|
+
export { OpenAIEmbeddingProvider } from './openai-provider.js';
|
|
28
|
+
export { quantizeToFloat32, dequantizeFromFloat32, quantizeToInt8, dequantizeFromInt8, } from './quantization.js';
|
|
29
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAW/D;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,MAAuB,EACvB,SAAqB;IAErB,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAChC,OAAO,IAAI,sBAAsB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACjC,iCAAiC;QACjC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,uBAAuB,CAAC;YACjC,MAAM,EAAE,SAAS,CAAC,MAAM;SACzB,CAAC,CAAC;IACL,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,cAAc,EACd,kBAAkB,GAGnB,MAAM,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local Embedding Provider (TF-IDF 기반)
|
|
3
|
+
*
|
|
4
|
+
* 64차원 경량 로컬 임베딩 생성기.
|
|
5
|
+
* 외부 의존성 없이 순수 TypeScript로 구현.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Local Embedding Provider (TF-IDF)
|
|
9
|
+
*/
|
|
10
|
+
export declare class LocalEmbeddingProvider {
|
|
11
|
+
private idfMap;
|
|
12
|
+
private dimensions;
|
|
13
|
+
private corpusSize;
|
|
14
|
+
private tokenCache;
|
|
15
|
+
private tokenCacheOrder;
|
|
16
|
+
private readonly TOKEN_CACHE_MAX;
|
|
17
|
+
constructor(dimensions?: number);
|
|
18
|
+
private cachedTokenize;
|
|
19
|
+
/**
|
|
20
|
+
* Corpus 업데이트 (IDF 계산)
|
|
21
|
+
*/
|
|
22
|
+
updateCorpus(documents: string[]): void;
|
|
23
|
+
/**
|
|
24
|
+
* 단일 텍스트 임베딩 생성
|
|
25
|
+
*/
|
|
26
|
+
embed(text: string): number[];
|
|
27
|
+
/**
|
|
28
|
+
* Corpus와 함께 임베딩 생성
|
|
29
|
+
*/
|
|
30
|
+
embedWithCorpus(text: string, corpusTexts?: string[]): number[];
|
|
31
|
+
/**
|
|
32
|
+
* 배치 임베딩 생성
|
|
33
|
+
*/
|
|
34
|
+
embedBatch(texts: string[]): number[][];
|
|
35
|
+
/**
|
|
36
|
+
* L2 정규화
|
|
37
|
+
*/
|
|
38
|
+
private normalize;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=local-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/local-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAoEH;;GAEG;AACH,qBAAa,sBAAsB;IACjC,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,eAAe,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAO;gBAE3B,UAAU,GAAE,MAAW;IAInC,OAAO,CAAC,cAAc;IAatB;;OAEG;IACH,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI;IAoBvC;;OAEG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAmB7B;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE;IAO/D;;OAEG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE;IAIvC;;OAEG;IACH,OAAO,CAAC,SAAS;CAKlB"}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Local Embedding Provider (TF-IDF 기반)
|
|
3
|
+
*
|
|
4
|
+
* 64차원 경량 로컬 임베딩 생성기.
|
|
5
|
+
* 외부 의존성 없이 순수 TypeScript로 구현.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* 간단한 해시 함수 (murmurhash 스타일)
|
|
9
|
+
*/
|
|
10
|
+
function hashString(str, seed = 0) {
|
|
11
|
+
let h = seed;
|
|
12
|
+
for (let i = 0; i < str.length; i++) {
|
|
13
|
+
h = Math.imul(h ^ str.charCodeAt(i), 2654435761);
|
|
14
|
+
}
|
|
15
|
+
return ((h ^ (h >>> 16)) >>> 0) % 0x7fffffff;
|
|
16
|
+
}
|
|
17
|
+
const _hashCache = new Map();
|
|
18
|
+
function cachedHashString(str, seed = 0) {
|
|
19
|
+
const key = `${str}:${seed}`;
|
|
20
|
+
let val = _hashCache.get(key);
|
|
21
|
+
if (val === undefined) {
|
|
22
|
+
val = hashString(str, seed);
|
|
23
|
+
_hashCache.set(key, val);
|
|
24
|
+
}
|
|
25
|
+
return val;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* 한국어/영어 기본 불용어
|
|
29
|
+
*/
|
|
30
|
+
const STOPWORDS = new Set([
|
|
31
|
+
// 영어
|
|
32
|
+
'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
|
|
33
|
+
'of', 'with', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
|
|
34
|
+
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should',
|
|
35
|
+
'can', 'could', 'may', 'might', 'must', 'shall',
|
|
36
|
+
// 한국어
|
|
37
|
+
'이', '그', '저', '것', '수', '등', '및', '또는', '하다',
|
|
38
|
+
'있다', '없다', '되다', '이다', '아니다', '의', '가', '을', '를',
|
|
39
|
+
'에', '에서', '로', '으로', '와', '과', '도', '만', '까지',
|
|
40
|
+
]);
|
|
41
|
+
/**
|
|
42
|
+
* 텍스트를 토큰으로 분리 (공백/구두점 기준)
|
|
43
|
+
*/
|
|
44
|
+
function tokenize(text) {
|
|
45
|
+
return text
|
|
46
|
+
.toLowerCase()
|
|
47
|
+
.split(/[\s\p{P}]+/u)
|
|
48
|
+
.filter((token) => token.length > 1 && !STOPWORDS.has(token));
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Term Frequency 계산
|
|
52
|
+
*/
|
|
53
|
+
function computeTF(tokens) {
|
|
54
|
+
const tf = new Map();
|
|
55
|
+
for (const token of tokens) {
|
|
56
|
+
tf.set(token, (tf.get(token) ?? 0) + 1);
|
|
57
|
+
}
|
|
58
|
+
// 정규화 (전체 토큰 수로 나눔)
|
|
59
|
+
const total = tokens.length;
|
|
60
|
+
if (total > 0) {
|
|
61
|
+
for (const [token, count] of tf.entries()) {
|
|
62
|
+
tf.set(token, count / total);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return tf;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Local Embedding Provider (TF-IDF)
|
|
69
|
+
*/
|
|
70
|
+
export class LocalEmbeddingProvider {
|
|
71
|
+
idfMap = new Map();
|
|
72
|
+
dimensions;
|
|
73
|
+
corpusSize = 0;
|
|
74
|
+
tokenCache = new Map();
|
|
75
|
+
tokenCacheOrder = [];
|
|
76
|
+
TOKEN_CACHE_MAX = 500;
|
|
77
|
+
constructor(dimensions = 64) {
|
|
78
|
+
this.dimensions = dimensions;
|
|
79
|
+
}
|
|
80
|
+
cachedTokenize(text) {
|
|
81
|
+
const cached = this.tokenCache.get(text);
|
|
82
|
+
if (cached)
|
|
83
|
+
return cached;
|
|
84
|
+
const tokens = tokenize(text);
|
|
85
|
+
this.tokenCache.set(text, tokens);
|
|
86
|
+
this.tokenCacheOrder.push(text);
|
|
87
|
+
if (this.tokenCacheOrder.length > this.TOKEN_CACHE_MAX) {
|
|
88
|
+
const oldest = this.tokenCacheOrder.shift();
|
|
89
|
+
this.tokenCache.delete(oldest);
|
|
90
|
+
}
|
|
91
|
+
return tokens;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Corpus 업데이트 (IDF 계산)
|
|
95
|
+
*/
|
|
96
|
+
updateCorpus(documents) {
|
|
97
|
+
this.corpusSize = documents.length;
|
|
98
|
+
if (documents.length === 0)
|
|
99
|
+
return;
|
|
100
|
+
// 각 토큰이 몇 개 문서에 등장하는지 카운트
|
|
101
|
+
const df = new Map();
|
|
102
|
+
for (const doc of documents) {
|
|
103
|
+
const tokens = new Set(tokenize(doc));
|
|
104
|
+
for (const token of tokens) {
|
|
105
|
+
df.set(token, (df.get(token) ?? 0) + 1);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// IDF 계산: log(N / df)
|
|
109
|
+
this.idfMap.clear();
|
|
110
|
+
for (const [token, docFreq] of df.entries()) {
|
|
111
|
+
this.idfMap.set(token, Math.log(this.corpusSize / docFreq));
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* 단일 텍스트 임베딩 생성
|
|
116
|
+
*/
|
|
117
|
+
embed(text) {
|
|
118
|
+
const tokens = this.cachedTokenize(text);
|
|
119
|
+
if (tokens.length === 0) {
|
|
120
|
+
return new Array(this.dimensions).fill(0);
|
|
121
|
+
}
|
|
122
|
+
const tf = computeTF(tokens);
|
|
123
|
+
const vector = new Array(this.dimensions).fill(0);
|
|
124
|
+
for (const [token, tfValue] of tf.entries()) {
|
|
125
|
+
const idf = this.idfMap.get(token) ?? 1;
|
|
126
|
+
const tfidf = tfValue * idf;
|
|
127
|
+
const index = cachedHashString(token) % this.dimensions;
|
|
128
|
+
vector[index] += tfidf;
|
|
129
|
+
}
|
|
130
|
+
return this.normalize(vector);
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Corpus와 함께 임베딩 생성
|
|
134
|
+
*/
|
|
135
|
+
embedWithCorpus(text, corpusTexts) {
|
|
136
|
+
if (corpusTexts && corpusTexts.length > 0) {
|
|
137
|
+
this.updateCorpus(corpusTexts);
|
|
138
|
+
}
|
|
139
|
+
return this.embed(text);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* 배치 임베딩 생성
|
|
143
|
+
*/
|
|
144
|
+
embedBatch(texts) {
|
|
145
|
+
return texts.map((text) => this.embed(text));
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* L2 정규화
|
|
149
|
+
*/
|
|
150
|
+
normalize(vector) {
|
|
151
|
+
const norm = Math.sqrt(vector.reduce((sum, val) => sum + val * val, 0));
|
|
152
|
+
if (norm === 0)
|
|
153
|
+
return vector;
|
|
154
|
+
return vector.map((val) => val / norm);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
//# sourceMappingURL=local-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-provider.js","sourceRoot":"","sources":["../../src/embedding/local-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,SAAS,UAAU,CAAC,GAAW,EAAE,OAAe,CAAC;IAC/C,IAAI,CAAC,GAAG,IAAI,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE7C,SAAS,gBAAgB,CAAC,GAAW,EAAE,OAAe,CAAC;IACrD,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACtB,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC5B,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,KAAK;IACL,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK;IACnE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IAC/D,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ;IACpE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO;IAC/C,MAAM;IACN,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI;IAC7C,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG;IACjD,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI;CAC/C,CAAC,CAAC;AAEH;;GAEG;AACH,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,KAAK,CAAC,aAAa,CAAC;SACpB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAClE,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,MAAgB;IACjC,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,oBAAoB;IACpB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1C,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,GAAG,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,sBAAsB;IACzB,MAAM,GAAwB,IAAI,GAAG,EAAE,CAAC;IACxC,UAAU,CAAS;IACnB,UAAU,GAAW,CAAC,CAAC;IACvB,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;IACzC,eAAe,GAAa,EAAE,CAAC;IACtB,eAAe,GAAG,GAAG,CAAC;IAEvC,YAAY,aAAqB,EAAE;QACjC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,EAAG,CAAC;YAC7C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,YAAY,CAAC,SAAmB;QAC9B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC;QACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAEnC,0BAA0B;QAC1B,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAC;QACrC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QAED,sBAAsB;QACtB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,IAAY;QAChB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC5C,CAAC;QAED,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAElD,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,OAAO,GAAG,GAAG,CAAC;YAC5B,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;YACxD,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;QACzB,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC;IAED;;OAEG;IACH,eAAe,CAAC,IAAY,EAAE,WAAsB;QAClD,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,UAAU,CAAC,KAAe;QACxB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,MAAgB;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,MAAM,CAAC;QAC9B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;IACzC,CAAC;CACF"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Embedding Provider
|
|
3
|
+
*
|
|
4
|
+
* text-embedding-3-small (1536차원) 사용.
|
|
5
|
+
* Native fetch 사용, 외부 의존성 없음.
|
|
6
|
+
*/
|
|
7
|
+
export interface OpenAIEmbeddingOptions {
|
|
8
|
+
apiKey: string;
|
|
9
|
+
model?: string;
|
|
10
|
+
timeout?: number;
|
|
11
|
+
}
|
|
12
|
+
export declare class OpenAIEmbeddingProvider {
|
|
13
|
+
private apiKey;
|
|
14
|
+
private model;
|
|
15
|
+
private timeout;
|
|
16
|
+
private baseUrl;
|
|
17
|
+
constructor(options: OpenAIEmbeddingOptions);
|
|
18
|
+
/**
|
|
19
|
+
* 단일 텍스트 임베딩 생성
|
|
20
|
+
*/
|
|
21
|
+
embed(text: string): Promise<number[]>;
|
|
22
|
+
/**
|
|
23
|
+
* 배치 임베딩 생성 (최대 2048개)
|
|
24
|
+
*/
|
|
25
|
+
embedBatch(texts: string[]): Promise<number[][]>;
|
|
26
|
+
/**
|
|
27
|
+
* 에러 응답 처리
|
|
28
|
+
*/
|
|
29
|
+
private handleErrorResponse;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=openai-provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,uBAAuB;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAA+B;gBAElC,OAAO,EAAE,sBAAsB;IAM3C;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAK5C;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAkDtD;;OAEG;YACW,mBAAmB;CAuBlC"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Embedding Provider
|
|
3
|
+
*
|
|
4
|
+
* text-embedding-3-small (1536차원) 사용.
|
|
5
|
+
* Native fetch 사용, 외부 의존성 없음.
|
|
6
|
+
*/
|
|
7
|
+
export class OpenAIEmbeddingProvider {
|
|
8
|
+
apiKey;
|
|
9
|
+
model;
|
|
10
|
+
timeout;
|
|
11
|
+
baseUrl = 'https://api.openai.com/v1';
|
|
12
|
+
constructor(options) {
|
|
13
|
+
this.apiKey = options.apiKey;
|
|
14
|
+
this.model = options.model ?? 'text-embedding-3-small';
|
|
15
|
+
this.timeout = options.timeout ?? 30000;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* 단일 텍스트 임베딩 생성
|
|
19
|
+
*/
|
|
20
|
+
async embed(text) {
|
|
21
|
+
const result = await this.embedBatch([text]);
|
|
22
|
+
return result[0];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* 배치 임베딩 생성 (최대 2048개)
|
|
26
|
+
*/
|
|
27
|
+
async embedBatch(texts) {
|
|
28
|
+
if (texts.length === 0) {
|
|
29
|
+
return [];
|
|
30
|
+
}
|
|
31
|
+
const controller = new AbortController();
|
|
32
|
+
const timeoutId = setTimeout(() => controller.abort(), this.timeout);
|
|
33
|
+
try {
|
|
34
|
+
const response = await fetch(`${this.baseUrl}/embeddings`, {
|
|
35
|
+
method: 'POST',
|
|
36
|
+
headers: {
|
|
37
|
+
'Content-Type': 'application/json',
|
|
38
|
+
'Authorization': `Bearer ${this.apiKey}`,
|
|
39
|
+
},
|
|
40
|
+
body: JSON.stringify({
|
|
41
|
+
input: texts,
|
|
42
|
+
model: this.model,
|
|
43
|
+
}),
|
|
44
|
+
signal: controller.signal,
|
|
45
|
+
});
|
|
46
|
+
clearTimeout(timeoutId);
|
|
47
|
+
if (!response.ok) {
|
|
48
|
+
await this.handleErrorResponse(response);
|
|
49
|
+
}
|
|
50
|
+
const data = await response.json();
|
|
51
|
+
// 응답 순서대로 임베딩 추출
|
|
52
|
+
const embeddings = data.data
|
|
53
|
+
.sort((a, b) => a.index - b.index)
|
|
54
|
+
.map((item) => item.embedding);
|
|
55
|
+
return embeddings;
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
clearTimeout(timeoutId);
|
|
59
|
+
if (error instanceof Error) {
|
|
60
|
+
if (error.name === 'AbortError') {
|
|
61
|
+
throw new Error(`OpenAI 요청 시간 초과: ${this.timeout}ms`);
|
|
62
|
+
}
|
|
63
|
+
throw new Error(`OpenAI API 오류: ${error.message}`);
|
|
64
|
+
}
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* 에러 응답 처리
|
|
70
|
+
*/
|
|
71
|
+
async handleErrorResponse(response) {
|
|
72
|
+
const status = response.status;
|
|
73
|
+
if (status === 401) {
|
|
74
|
+
throw new Error('OpenAI 인증 실패: API Key를 확인하세요');
|
|
75
|
+
}
|
|
76
|
+
if (status === 429) {
|
|
77
|
+
throw new Error('OpenAI Rate Limit: 잠시 후 다시 시도하세요');
|
|
78
|
+
}
|
|
79
|
+
let message = `OpenAI HTTP ${status}`;
|
|
80
|
+
try {
|
|
81
|
+
const errorBody = await response.json();
|
|
82
|
+
if (errorBody.error?.message) {
|
|
83
|
+
message += `: ${errorBody.error.message}`;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// JSON 파싱 실패 시 기본 메시지 사용
|
|
88
|
+
}
|
|
89
|
+
throw new Error(message);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=openai-provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-provider.js","sourceRoot":"","sources":["../../src/embedding/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAQH,MAAM,OAAO,uBAAuB;IAC1B,MAAM,CAAS;IACf,KAAK,CAAS;IACd,OAAO,CAAS;IAChB,OAAO,GAAG,2BAA2B,CAAC;IAE9C,YAAY,OAA+B;QACzC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACvD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,aAAa,EAAE;gBACzD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACzC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC;gBACF,MAAM,EAAE,UAAU,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAC3C,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAA6B,CAAC;YAE9D,iBAAiB;YACjB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI;iBACzB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAEjC,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YAExB,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBAChC,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBACxD,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,kBAAkB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,QAAkB;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE/B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;QAED,IAAI,OAAO,GAAG,eAAe,MAAM,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAsC,CAAC;YAC5E,IAAI,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;gBAC7B,OAAO,IAAI,KAAK,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;CACF"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Quantization Module
|
|
3
|
+
*
|
|
4
|
+
* Scalar Quantization으로 임베딩 저장 공간 절감.
|
|
5
|
+
* - Float32: 50% 절감 (float64 → float32)
|
|
6
|
+
* - Int8: 87.5% 절감 (float64 → int8)
|
|
7
|
+
*/
|
|
8
|
+
export type QuantizationLevel = 'none' | 'float32' | 'int8';
|
|
9
|
+
export interface QuantizationStats {
|
|
10
|
+
min: number;
|
|
11
|
+
max: number;
|
|
12
|
+
dimensions: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Float32 양자화 (float64 → float32, 50% 저장 공간 절감)
|
|
16
|
+
*/
|
|
17
|
+
export declare function quantizeToFloat32(embedding: number[]): Buffer;
|
|
18
|
+
/**
|
|
19
|
+
* Float32 역양자화 (buffer → number[])
|
|
20
|
+
*/
|
|
21
|
+
export declare function dequantizeFromFloat32(buffer: Buffer): number[];
|
|
22
|
+
/**
|
|
23
|
+
* Int8 양자화 (float64 → int8, 87.5% 저장 공간 절감)
|
|
24
|
+
* Min-Max Scaling: value → round((value - min) / (max - min) * 255) - 128
|
|
25
|
+
*/
|
|
26
|
+
export declare function quantizeToInt8(embedding: number[], stats?: QuantizationStats): {
|
|
27
|
+
buffer: Buffer;
|
|
28
|
+
stats: QuantizationStats;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Int8 역양자화 (buffer + stats → number[])
|
|
32
|
+
*/
|
|
33
|
+
export declare function dequantizeFromInt8(buffer: Buffer, stats: QuantizationStats): number[];
|
|
34
|
+
//# sourceMappingURL=quantization.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quantization.d.ts","sourceRoot":"","sources":["../../src/embedding/quantization.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,CAAC;AAE5D,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAG7D;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI9D;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,CAAC,EAAE,iBAAiB,GACxB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,iBAAiB,CAAA;CAAE,CAwC9C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,GAAG,MAAM,EAAE,CAsBrF"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Quantization Module
|
|
3
|
+
*
|
|
4
|
+
* Scalar Quantization으로 임베딩 저장 공간 절감.
|
|
5
|
+
* - Float32: 50% 절감 (float64 → float32)
|
|
6
|
+
* - Int8: 87.5% 절감 (float64 → int8)
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Float32 양자화 (float64 → float32, 50% 저장 공간 절감)
|
|
10
|
+
*/
|
|
11
|
+
export function quantizeToFloat32(embedding) {
|
|
12
|
+
const float32Array = new Float32Array(embedding);
|
|
13
|
+
return Buffer.from(float32Array.buffer, float32Array.byteOffset, float32Array.byteLength);
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Float32 역양자화 (buffer → number[])
|
|
17
|
+
*/
|
|
18
|
+
export function dequantizeFromFloat32(buffer) {
|
|
19
|
+
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
20
|
+
const float32Array = new Float32Array(arrayBuffer);
|
|
21
|
+
return Array.from(float32Array);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Int8 양자화 (float64 → int8, 87.5% 저장 공간 절감)
|
|
25
|
+
* Min-Max Scaling: value → round((value - min) / (max - min) * 255) - 128
|
|
26
|
+
*/
|
|
27
|
+
export function quantizeToInt8(embedding, stats) {
|
|
28
|
+
if (embedding.length === 0) {
|
|
29
|
+
return {
|
|
30
|
+
buffer: Buffer.alloc(0),
|
|
31
|
+
stats: { min: 0, max: 0, dimensions: 0 },
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
// Stats 계산 (제공되지 않은 경우)
|
|
35
|
+
let min, max;
|
|
36
|
+
if (stats) {
|
|
37
|
+
min = stats.min;
|
|
38
|
+
max = stats.max;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
min = Math.min(...embedding);
|
|
42
|
+
max = Math.max(...embedding);
|
|
43
|
+
}
|
|
44
|
+
const range = max - min;
|
|
45
|
+
const int8Array = new Int8Array(embedding.length);
|
|
46
|
+
// 모든 값이 같은 경우 (range = 0)
|
|
47
|
+
if (range === 0) {
|
|
48
|
+
int8Array.fill(0);
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
52
|
+
const normalized = (embedding[i] - min) / range; // 0~1 범위로 정규화
|
|
53
|
+
const scaled = Math.round(normalized * 255) - 128; // -128~127 범위로 변환
|
|
54
|
+
int8Array[i] = Math.max(-128, Math.min(127, scaled)); // clamp
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
buffer: Buffer.from(int8Array.buffer, int8Array.byteOffset, int8Array.byteLength),
|
|
59
|
+
stats: {
|
|
60
|
+
min,
|
|
61
|
+
max,
|
|
62
|
+
dimensions: embedding.length,
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Int8 역양자화 (buffer + stats → number[])
|
|
68
|
+
*/
|
|
69
|
+
export function dequantizeFromInt8(buffer, stats) {
|
|
70
|
+
if (buffer.length === 0 || stats.dimensions === 0) {
|
|
71
|
+
return [];
|
|
72
|
+
}
|
|
73
|
+
const arrayBuffer = buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
|
|
74
|
+
const int8Array = new Int8Array(arrayBuffer);
|
|
75
|
+
const range = stats.max - stats.min;
|
|
76
|
+
const result = new Array(stats.dimensions);
|
|
77
|
+
// 모든 값이 같았던 경우 (range = 0)
|
|
78
|
+
if (range === 0) {
|
|
79
|
+
result.fill(stats.min);
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
for (let i = 0; i < stats.dimensions; i++) {
|
|
83
|
+
const scaled = (int8Array[i] + 128) / 255; // 0~1 범위로 복원
|
|
84
|
+
result[i] = scaled * range + stats.min; // 원래 범위로 복원
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=quantization.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quantization.js","sourceRoot":"","sources":["../../src/embedding/quantization.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAmB;IACnD,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IACjD,OAAO,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,UAAU,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;AAC5F,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAClD,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAClG,MAAM,YAAY,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC;IACnD,OAAO,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,SAAmB,EACnB,KAAyB;IAEzB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YACvB,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;SACzC,CAAC;IACJ,CAAC;IAED,wBAAwB;IACxB,IAAI,GAAW,EAAE,GAAW,CAAC;IAC7B,IAAI,KAAK,EAAE,CAAC;QACV,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QAChB,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QAC7B,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,CAAC;IACxB,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAElD,0BAA0B;IAC1B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;SAAM,CAAC;QACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,UAAU,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,cAAc;YAC/D,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,kBAAkB;YACrE,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ;QAChE,CAAC;IACH,CAAC;IAED,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC;QACjF,KAAK,EAAE;YACL,GAAG;YACH,GAAG;YACH,UAAU,EAAE,SAAS,CAAC,MAAM;SAC7B;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAc,EAAE,KAAwB;IACzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;IAClG,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAE3C,2BAA2B;IAC3B,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;SAAM,CAAC;QACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,aAAa;YACxD,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,YAAY;QACtD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -2,11 +2,21 @@
|
|
|
2
2
|
* Memory Extractor
|
|
3
3
|
*
|
|
4
4
|
* 세션 메시지에서 의미 있는 메모리를 추출합니다.
|
|
5
|
-
* 카테고리: error_solution, code_pattern, decision, context,
|
|
5
|
+
* 카테고리: error_solution, code_pattern, decision, context, convention
|
|
6
6
|
*/
|
|
7
7
|
import type { SessionMessage, ExtractionResult } from '../types/index.js';
|
|
8
|
+
import type { LLMClient } from '../llm/client.js';
|
|
8
9
|
/**
|
|
9
10
|
* 세션 메시지에서 메모리 추출
|
|
10
11
|
*/
|
|
11
12
|
export declare function extractMemories(messages: SessionMessage[], projectPath: string): ExtractionResult;
|
|
13
|
+
/**
|
|
14
|
+
* LLM 기반 분류가 포함된 메모리 추출 (async)
|
|
15
|
+
* LLM 없으면 extractMemories와 동일한 결과
|
|
16
|
+
*/
|
|
17
|
+
export declare function extractMemoriesWithLLM(messages: SessionMessage[], projectPath: string, llmClient?: LLMClient): Promise<ExtractionResult>;
|
|
18
|
+
/**
|
|
19
|
+
* LLM 기반 세션 요약 생성
|
|
20
|
+
*/
|
|
21
|
+
export declare function summarizeSession(messages: SessionMessage[], llmClient: LLMClient): Promise<string>;
|
|
12
22
|
//# sourceMappingURL=extractor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/extraction/extractor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,cAAc,
|
|
1
|
+
{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/extraction/extractor.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EACV,cAAc,EAId,gBAAgB,EAEjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AA0BlD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,cAAc,EAAE,EAC1B,WAAW,EAAE,MAAM,GAClB,gBAAgB,CAyDlB;AAwOD;;;GAGG;AACH,wBAAsB,sBAAsB,CAC1C,QAAQ,EAAE,cAAc,EAAE,EAC1B,WAAW,EAAE,MAAM,EACnB,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,gBAAgB,CAAC,CAuB3B;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,cAAc,EAAE,EAC1B,SAAS,EAAE,SAAS,GACnB,OAAO,CAAC,MAAM,CAAC,CAgBjB"}
|