@sparkleideas/embeddings 3.0.0-alpha.12-patch.25 → 3.0.0-alpha.12-patch.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/embedding-service.test.d.ts +2 -0
- package/dist/__tests__/embedding-service.test.d.ts.map +1 -0
- package/dist/__tests__/embedding-service.test.js +98 -0
- package/dist/__tests__/embedding-service.test.js.map +1 -0
- package/dist/chunking.d.ts +68 -0
- package/dist/chunking.d.ts.map +1 -0
- package/dist/chunking.js +251 -0
- package/dist/chunking.js.map +1 -0
- package/dist/embedding-service.d.ts +207 -0
- package/dist/embedding-service.d.ts.map +1 -0
- package/dist/embedding-service.js +965 -0
- package/dist/embedding-service.js.map +1 -0
- package/dist/hyperbolic.d.ts +103 -0
- package/dist/hyperbolic.d.ts.map +1 -0
- package/dist/hyperbolic.js +343 -0
- package/dist/hyperbolic.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +37 -0
- package/dist/index.js.map +1 -0
- package/dist/neural-integration.d.ts +203 -0
- package/dist/neural-integration.d.ts.map +1 -0
- package/dist/neural-integration.js +213 -0
- package/dist/neural-integration.js.map +1 -0
- package/dist/normalization.d.ts +73 -0
- package/dist/normalization.d.ts.map +1 -0
- package/dist/normalization.js +192 -0
- package/dist/normalization.js.map +1 -0
- package/dist/persistent-cache.d.ts +119 -0
- package/dist/persistent-cache.d.ts.map +1 -0
- package/dist/persistent-cache.js +337 -0
- package/dist/persistent-cache.js.map +1 -0
- package/dist/rvf-embedding-cache.d.ts +118 -0
- package/dist/rvf-embedding-cache.d.ts.map +1 -0
- package/dist/rvf-embedding-cache.js +458 -0
- package/dist/rvf-embedding-cache.js.map +1 -0
- package/dist/rvf-embedding-service.d.ts +79 -0
- package/dist/rvf-embedding-service.d.ts.map +1 -0
- package/dist/rvf-embedding-service.js +318 -0
- package/dist/rvf-embedding-service.js.map +1 -0
- package/dist/types.d.ts +237 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +15 -0
- package/dist/types.js.map +1 -0
- package/package.json +3 -3
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Normalization Utilities
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - L2 (Euclidean) normalization
|
|
6
|
+
* - L1 (Manhattan) normalization
|
|
7
|
+
* - Min-max normalization
|
|
8
|
+
* - Z-score standardization
|
|
9
|
+
* - Batch normalization
|
|
10
|
+
*/
|
|
11
|
+
/**
|
|
12
|
+
* L2 (Euclidean) normalize embedding to unit length
|
|
13
|
+
* Most common for cosine similarity
|
|
14
|
+
*
|
|
15
|
+
* @param embedding - Input embedding vector
|
|
16
|
+
* @param epsilon - Small value to prevent division by zero
|
|
17
|
+
* @returns Normalized embedding with ||v|| = 1
|
|
18
|
+
*/
|
|
19
|
+
export function l2Normalize(embedding, epsilon = 1e-12) {
|
|
20
|
+
const result = embedding instanceof Float32Array
|
|
21
|
+
? new Float32Array(embedding.length)
|
|
22
|
+
: new Float32Array(embedding.length);
|
|
23
|
+
// Calculate L2 norm (Euclidean length)
|
|
24
|
+
let sumSquares = 0;
|
|
25
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
26
|
+
sumSquares += embedding[i] * embedding[i];
|
|
27
|
+
}
|
|
28
|
+
const norm = Math.sqrt(sumSquares);
|
|
29
|
+
const scale = norm > epsilon ? 1 / norm : 0;
|
|
30
|
+
// Normalize
|
|
31
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
32
|
+
result[i] = embedding[i] * scale;
|
|
33
|
+
}
|
|
34
|
+
return result;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* L2 normalize embedding in-place (modifies original array)
|
|
38
|
+
*/
|
|
39
|
+
export function l2NormalizeInPlace(embedding, epsilon = 1e-12) {
|
|
40
|
+
let sumSquares = 0;
|
|
41
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
42
|
+
sumSquares += embedding[i] * embedding[i];
|
|
43
|
+
}
|
|
44
|
+
const norm = Math.sqrt(sumSquares);
|
|
45
|
+
const scale = norm > epsilon ? 1 / norm : 0;
|
|
46
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
47
|
+
embedding[i] *= scale;
|
|
48
|
+
}
|
|
49
|
+
return embedding;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* L1 (Manhattan) normalize embedding
|
|
53
|
+
* Sum of absolute values = 1
|
|
54
|
+
*/
|
|
55
|
+
export function l1Normalize(embedding, epsilon = 1e-12) {
|
|
56
|
+
const result = new Float32Array(embedding.length);
|
|
57
|
+
// Calculate L1 norm (sum of absolute values)
|
|
58
|
+
let sumAbs = 0;
|
|
59
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
60
|
+
sumAbs += Math.abs(embedding[i]);
|
|
61
|
+
}
|
|
62
|
+
const scale = sumAbs > epsilon ? 1 / sumAbs : 0;
|
|
63
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
64
|
+
result[i] = embedding[i] * scale;
|
|
65
|
+
}
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Min-max normalize embedding to [0, 1] range
|
|
70
|
+
*/
|
|
71
|
+
export function minMaxNormalize(embedding, epsilon = 1e-12) {
|
|
72
|
+
const result = new Float32Array(embedding.length);
|
|
73
|
+
// Find min and max
|
|
74
|
+
let min = Infinity;
|
|
75
|
+
let max = -Infinity;
|
|
76
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
77
|
+
if (embedding[i] < min)
|
|
78
|
+
min = embedding[i];
|
|
79
|
+
if (embedding[i] > max)
|
|
80
|
+
max = embedding[i];
|
|
81
|
+
}
|
|
82
|
+
const range = max - min;
|
|
83
|
+
const scale = range > epsilon ? 1 / range : 0;
|
|
84
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
85
|
+
result[i] = (embedding[i] - min) * scale;
|
|
86
|
+
}
|
|
87
|
+
return result;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Z-score standardize embedding (mean=0, std=1)
|
|
91
|
+
*/
|
|
92
|
+
export function zScoreNormalize(embedding, epsilon = 1e-12) {
|
|
93
|
+
const result = new Float32Array(embedding.length);
|
|
94
|
+
const n = embedding.length;
|
|
95
|
+
// Calculate mean
|
|
96
|
+
let sum = 0;
|
|
97
|
+
for (let i = 0; i < n; i++) {
|
|
98
|
+
sum += embedding[i];
|
|
99
|
+
}
|
|
100
|
+
const mean = sum / n;
|
|
101
|
+
// Calculate standard deviation
|
|
102
|
+
let sumSquaredDiff = 0;
|
|
103
|
+
for (let i = 0; i < n; i++) {
|
|
104
|
+
const diff = embedding[i] - mean;
|
|
105
|
+
sumSquaredDiff += diff * diff;
|
|
106
|
+
}
|
|
107
|
+
const std = Math.sqrt(sumSquaredDiff / n);
|
|
108
|
+
const scale = std > epsilon ? 1 / std : 0;
|
|
109
|
+
// Standardize
|
|
110
|
+
for (let i = 0; i < n; i++) {
|
|
111
|
+
result[i] = (embedding[i] - mean) * scale;
|
|
112
|
+
}
|
|
113
|
+
return result;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Normalize embedding using specified method
|
|
117
|
+
*/
|
|
118
|
+
export function normalize(embedding, options = {}) {
|
|
119
|
+
const { type = 'l2', epsilon = 1e-12, inPlace = false } = options;
|
|
120
|
+
if (type === 'none') {
|
|
121
|
+
return embedding instanceof Float32Array
|
|
122
|
+
? embedding
|
|
123
|
+
: new Float32Array(embedding);
|
|
124
|
+
}
|
|
125
|
+
if (inPlace && embedding instanceof Float32Array && type === 'l2') {
|
|
126
|
+
return l2NormalizeInPlace(embedding, epsilon);
|
|
127
|
+
}
|
|
128
|
+
switch (type) {
|
|
129
|
+
case 'l2':
|
|
130
|
+
return l2Normalize(embedding, epsilon);
|
|
131
|
+
case 'l1':
|
|
132
|
+
return l1Normalize(embedding, epsilon);
|
|
133
|
+
case 'minmax':
|
|
134
|
+
return minMaxNormalize(embedding, epsilon);
|
|
135
|
+
case 'zscore':
|
|
136
|
+
return zScoreNormalize(embedding, epsilon);
|
|
137
|
+
default:
|
|
138
|
+
return l2Normalize(embedding, epsilon);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Batch normalize multiple embeddings
|
|
143
|
+
*/
|
|
144
|
+
export function normalizeBatch(embeddings, options = {}) {
|
|
145
|
+
return embeddings.map(emb => normalize(emb, options));
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Calculate L2 norm of embedding
|
|
149
|
+
*/
|
|
150
|
+
export function l2Norm(embedding) {
|
|
151
|
+
let sumSquares = 0;
|
|
152
|
+
for (let i = 0; i < embedding.length; i++) {
|
|
153
|
+
sumSquares += embedding[i] * embedding[i];
|
|
154
|
+
}
|
|
155
|
+
return Math.sqrt(sumSquares);
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Check if embedding is already normalized (L2 norm ≈ 1)
|
|
159
|
+
*/
|
|
160
|
+
export function isNormalized(embedding, tolerance = 1e-6) {
|
|
161
|
+
const norm = l2Norm(embedding);
|
|
162
|
+
return Math.abs(norm - 1) < tolerance;
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Center embeddings by subtracting mean across batch
|
|
166
|
+
* Useful for improving similarity metrics
|
|
167
|
+
*/
|
|
168
|
+
export function centerEmbeddings(embeddings) {
|
|
169
|
+
if (embeddings.length === 0)
|
|
170
|
+
return [];
|
|
171
|
+
const dim = embeddings[0].length;
|
|
172
|
+
const n = embeddings.length;
|
|
173
|
+
// Calculate mean for each dimension
|
|
174
|
+
const mean = new Float32Array(dim);
|
|
175
|
+
for (const emb of embeddings) {
|
|
176
|
+
for (let i = 0; i < dim; i++) {
|
|
177
|
+
mean[i] += emb[i];
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
for (let i = 0; i < dim; i++) {
|
|
181
|
+
mean[i] /= n;
|
|
182
|
+
}
|
|
183
|
+
// Subtract mean from each embedding
|
|
184
|
+
return embeddings.map(emb => {
|
|
185
|
+
const centered = new Float32Array(dim);
|
|
186
|
+
for (let i = 0; i < dim; i++) {
|
|
187
|
+
centered[i] = emb[i] - mean[i];
|
|
188
|
+
}
|
|
189
|
+
return centered;
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
//# sourceMappingURL=normalization.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalization.js","sourceRoot":"","sources":["../src/normalization.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAmBH;;;;;;;GAOG;AACH,MAAM,UAAU,WAAW,CACzB,SAAkC,EAClC,OAAO,GAAG,KAAK;IAEf,MAAM,MAAM,GAAG,SAAS,YAAY,YAAY;QAC9C,CAAC,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC;QACpC,CAAC,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAEvC,uCAAuC;IACvC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5C,YAAY;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,SAAuB,EACvB,OAAO,GAAG,KAAK;IAEf,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;IACxB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,SAAkC,EAClC,OAAO,GAAG,KAAK;IAEf,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAElD,6CAA6C;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;IACnC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAkC,EAClC,OAAO,GAAG,KAAK;IAEf,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAElD,mBAAmB;IACnB,IAAI,GAAG,GAAG,QAAQ,CAAC;IACnB,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QAC3C,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG;YAAE,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,KAAK,GAAG,GAAG,GAAG,GAAG,CAAC;IACxB,MAAM,KAAK,GAAG,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAE9C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC;IAC3C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAkC,EAClC,OAAO,GAAG,KAAK;IAEf,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC;IAE3B,iBAAiB;IACjB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;IAErB,+BAA+B;IAC/B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QACjC,cAAc,IAAI,IAAI,GAAG,IAAI,CAAC;IAChC,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;IAC1C,MAAM,KAAK,GAAG,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1C,cAAc;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;IAC5C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,SAAS,CACvB,SAAkC,EAClC,UAAgC,EAAE;IAElC,MAAM,EAAE,IAAI,GAAG,IAAI,EAAE,OAAO,GAAG,KAAK,EAAE,OAAO,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAElE,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,OAAO,SAAS,YAAY,YAAY;YACtC,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,IAAI,SAAS,YAAY,YAAY,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAClE,OAAO,kBAAkB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAChD,CAAC;IAED,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,IAAI;YACP,OAAO,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACzC,KAAK,IAAI;YACP,OAAO,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACzC,KAAK,QAAQ;YACX,OAAO,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7C,KAAK,QAAQ;YACX,OAAO,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7C;YACE,OAAO,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,UAA0C,EAC1C,UAAgC,EAAE;IAElC,OAAO,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,SAAkC;IACvD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;AAC/B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,SAAkC,EAClC,SAAS,GAAG,IAAI;IAEhB,MAAM,IAAI,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,UAA0C;IAE1C,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAE5B,oCAAoC;IACpC,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IACnC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,IAAI,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,oCAAoC;IACpC,OAAO,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;QAC1B,MAAM,QAAQ,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,QAAQ,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite-backed Persistent Cache for Embeddings (sql.js)
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - Cross-platform support (pure JavaScript/WASM, no native compilation)
|
|
6
|
+
* - Disk persistence across sessions
|
|
7
|
+
* - LRU eviction with configurable max size
|
|
8
|
+
* - Automatic schema creation
|
|
9
|
+
* - TTL support for cache entries
|
|
10
|
+
* - Lazy initialization (no startup cost if not used)
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Configuration for persistent cache
|
|
14
|
+
*/
|
|
15
|
+
export interface PersistentCacheConfig {
|
|
16
|
+
/** Path to SQLite database file */
|
|
17
|
+
dbPath: string;
|
|
18
|
+
/** Maximum number of entries (default: 10000) */
|
|
19
|
+
maxSize?: number;
|
|
20
|
+
/** TTL in milliseconds (default: 7 days) */
|
|
21
|
+
ttlMs?: number;
|
|
22
|
+
/** Enable compression for large embeddings */
|
|
23
|
+
compress?: boolean;
|
|
24
|
+
/** Auto-save interval in ms (default: 30000) */
|
|
25
|
+
autoSaveInterval?: number;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Cache statistics
|
|
29
|
+
*/
|
|
30
|
+
export interface PersistentCacheStats {
|
|
31
|
+
size: number;
|
|
32
|
+
maxSize: number;
|
|
33
|
+
hitRate: number;
|
|
34
|
+
hits: number;
|
|
35
|
+
misses: number;
|
|
36
|
+
dbSizeBytes?: number;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* SQLite-backed persistent embedding cache using sql.js (pure JS/WASM)
|
|
40
|
+
*/
|
|
41
|
+
export declare class PersistentEmbeddingCache {
|
|
42
|
+
private db;
|
|
43
|
+
private SQL;
|
|
44
|
+
private initialized;
|
|
45
|
+
private dirty;
|
|
46
|
+
private hits;
|
|
47
|
+
private misses;
|
|
48
|
+
private autoSaveTimer;
|
|
49
|
+
private readonly dbPath;
|
|
50
|
+
private readonly maxSize;
|
|
51
|
+
private readonly ttlMs;
|
|
52
|
+
private readonly autoSaveInterval;
|
|
53
|
+
constructor(config: PersistentCacheConfig);
|
|
54
|
+
/**
|
|
55
|
+
* Lazily initialize database connection
|
|
56
|
+
*/
|
|
57
|
+
private ensureInitialized;
|
|
58
|
+
/**
|
|
59
|
+
* Start auto-save timer
|
|
60
|
+
*/
|
|
61
|
+
private startAutoSave;
|
|
62
|
+
/**
|
|
63
|
+
* Stop auto-save timer
|
|
64
|
+
*/
|
|
65
|
+
private stopAutoSave;
|
|
66
|
+
/**
|
|
67
|
+
* Save database to file
|
|
68
|
+
*/
|
|
69
|
+
private saveToFile;
|
|
70
|
+
/**
|
|
71
|
+
* Generate cache key from text
|
|
72
|
+
*/
|
|
73
|
+
private hashKey;
|
|
74
|
+
/**
|
|
75
|
+
* Serialize Float32Array to Uint8Array for sql.js
|
|
76
|
+
*/
|
|
77
|
+
private serializeEmbedding;
|
|
78
|
+
/**
|
|
79
|
+
* Deserialize Uint8Array to Float32Array
|
|
80
|
+
*/
|
|
81
|
+
private deserializeEmbedding;
|
|
82
|
+
/**
|
|
83
|
+
* Get embedding from cache
|
|
84
|
+
*/
|
|
85
|
+
get(text: string): Promise<Float32Array | null>;
|
|
86
|
+
/**
|
|
87
|
+
* Store embedding in cache
|
|
88
|
+
*/
|
|
89
|
+
set(text: string, embedding: Float32Array): Promise<void>;
|
|
90
|
+
/**
|
|
91
|
+
* Evict oldest entries if cache exceeds max size
|
|
92
|
+
*/
|
|
93
|
+
private evictIfNeeded;
|
|
94
|
+
/**
|
|
95
|
+
* Clean expired entries
|
|
96
|
+
*/
|
|
97
|
+
private cleanExpired;
|
|
98
|
+
/**
|
|
99
|
+
* Get cache statistics
|
|
100
|
+
*/
|
|
101
|
+
getStats(): Promise<PersistentCacheStats>;
|
|
102
|
+
/**
|
|
103
|
+
* Clear all cached entries
|
|
104
|
+
*/
|
|
105
|
+
clear(): Promise<void>;
|
|
106
|
+
/**
|
|
107
|
+
* Force save to disk
|
|
108
|
+
*/
|
|
109
|
+
flush(): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Close database connection
|
|
112
|
+
*/
|
|
113
|
+
close(): Promise<void>;
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Check if persistent cache is available (sql.js installed)
|
|
117
|
+
*/
|
|
118
|
+
export declare function isPersistentCacheAvailable(): Promise<boolean>;
|
|
119
|
+
//# sourceMappingURL=persistent-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"persistent-cache.d.ts","sourceRoot":"","sources":["../src/persistent-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAUH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gDAAgD;IAChD,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,qBAAa,wBAAwB;IACnC,OAAO,CAAC,EAAE,CAA8B;IACxC,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,IAAI,CAAK;IACjB,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,aAAa,CAA+C;IAEpE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,qBAAqB;IAOzC;;OAEG;YACW,iBAAiB;IAwD/B;;OAEG;IACH,OAAO,CAAC,aAAa;IAUrB;;OAEG;IACH,OAAO,CAAC,YAAY;IAOpB;;OAEG;IACH,OAAO,CAAC,UAAU;IAalB;;OAEG;IACH,OAAO,CAAC,OAAO;IAUf;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACG,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAwDrD;;OAEG;IACG,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B/D;;OAEG;YACW,aAAa;IAoB3B;;OAEG;IACH,OAAO,CAAC,YAAY;IAQpB;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,oBAAoB,CAAC;IA8B/C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAc7B;AAED;;GAEG;AACH,wBAAsB,0BAA0B,IAAI,OAAO,CAAC,OAAO,CAAC,CAQnE"}
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQLite-backed Persistent Cache for Embeddings (sql.js)
|
|
3
|
+
*
|
|
4
|
+
* Features:
|
|
5
|
+
* - Cross-platform support (pure JavaScript/WASM, no native compilation)
|
|
6
|
+
* - Disk persistence across sessions
|
|
7
|
+
* - LRU eviction with configurable max size
|
|
8
|
+
* - Automatic schema creation
|
|
9
|
+
* - TTL support for cache entries
|
|
10
|
+
* - Lazy initialization (no startup cost if not used)
|
|
11
|
+
*/
|
|
12
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
|
13
|
+
import { dirname } from 'path';
|
|
14
|
+
/**
|
|
15
|
+
* SQLite-backed persistent embedding cache using sql.js (pure JS/WASM)
|
|
16
|
+
*/
|
|
17
|
+
export class PersistentEmbeddingCache {
|
|
18
|
+
db = null;
|
|
19
|
+
SQL = null;
|
|
20
|
+
initialized = false;
|
|
21
|
+
dirty = false;
|
|
22
|
+
hits = 0;
|
|
23
|
+
misses = 0;
|
|
24
|
+
autoSaveTimer = null;
|
|
25
|
+
dbPath;
|
|
26
|
+
maxSize;
|
|
27
|
+
ttlMs;
|
|
28
|
+
autoSaveInterval;
|
|
29
|
+
constructor(config) {
|
|
30
|
+
this.dbPath = config.dbPath;
|
|
31
|
+
this.maxSize = config.maxSize ?? 10000;
|
|
32
|
+
this.ttlMs = config.ttlMs ?? 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
33
|
+
this.autoSaveInterval = config.autoSaveInterval ?? 30000; // 30 seconds
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Lazily initialize database connection
|
|
37
|
+
*/
|
|
38
|
+
async ensureInitialized() {
|
|
39
|
+
if (this.initialized)
|
|
40
|
+
return;
|
|
41
|
+
try {
|
|
42
|
+
// Dynamically import sql.js
|
|
43
|
+
const initSqlJs = (await import('sql.js')).default;
|
|
44
|
+
// Initialize sql.js (loads WASM)
|
|
45
|
+
this.SQL = await initSqlJs();
|
|
46
|
+
// Ensure directory exists
|
|
47
|
+
const dir = dirname(this.dbPath);
|
|
48
|
+
if (!existsSync(dir)) {
|
|
49
|
+
mkdirSync(dir, { recursive: true });
|
|
50
|
+
}
|
|
51
|
+
// Load existing database or create new
|
|
52
|
+
if (existsSync(this.dbPath)) {
|
|
53
|
+
const fileBuffer = readFileSync(this.dbPath);
|
|
54
|
+
this.db = new this.SQL.Database(fileBuffer);
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
this.db = new this.SQL.Database();
|
|
58
|
+
}
|
|
59
|
+
// Create schema
|
|
60
|
+
this.db.run(`
|
|
61
|
+
CREATE TABLE IF NOT EXISTS embeddings (
|
|
62
|
+
key TEXT PRIMARY KEY,
|
|
63
|
+
embedding BLOB NOT NULL,
|
|
64
|
+
dimensions INTEGER NOT NULL,
|
|
65
|
+
created_at INTEGER NOT NULL,
|
|
66
|
+
accessed_at INTEGER NOT NULL,
|
|
67
|
+
access_count INTEGER DEFAULT 1
|
|
68
|
+
)
|
|
69
|
+
`);
|
|
70
|
+
this.db.run('CREATE INDEX IF NOT EXISTS idx_accessed_at ON embeddings(accessed_at)');
|
|
71
|
+
this.db.run('CREATE INDEX IF NOT EXISTS idx_created_at ON embeddings(created_at)');
|
|
72
|
+
// Clean expired entries on startup
|
|
73
|
+
this.cleanExpired();
|
|
74
|
+
// Save after initialization to persist schema
|
|
75
|
+
this.saveToFile();
|
|
76
|
+
// Start auto-save timer
|
|
77
|
+
this.startAutoSave();
|
|
78
|
+
this.initialized = true;
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
// If sql.js not available, fall back gracefully
|
|
82
|
+
console.warn('[persistent-cache] sql.js not available, cache disabled:', error instanceof Error ? error.message : error);
|
|
83
|
+
this.initialized = true; // Mark as initialized to prevent retry
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Start auto-save timer
|
|
88
|
+
*/
|
|
89
|
+
startAutoSave() {
|
|
90
|
+
if (this.autoSaveTimer)
|
|
91
|
+
return;
|
|
92
|
+
this.autoSaveTimer = setInterval(() => {
|
|
93
|
+
if (this.dirty && this.db) {
|
|
94
|
+
this.saveToFile();
|
|
95
|
+
}
|
|
96
|
+
}, this.autoSaveInterval);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Stop auto-save timer
|
|
100
|
+
*/
|
|
101
|
+
stopAutoSave() {
|
|
102
|
+
if (this.autoSaveTimer) {
|
|
103
|
+
clearInterval(this.autoSaveTimer);
|
|
104
|
+
this.autoSaveTimer = null;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Save database to file
|
|
109
|
+
*/
|
|
110
|
+
saveToFile() {
|
|
111
|
+
if (!this.db)
|
|
112
|
+
return;
|
|
113
|
+
try {
|
|
114
|
+
const data = this.db.export();
|
|
115
|
+
const buffer = Buffer.from(data);
|
|
116
|
+
writeFileSync(this.dbPath, buffer);
|
|
117
|
+
this.dirty = false;
|
|
118
|
+
}
|
|
119
|
+
catch (error) {
|
|
120
|
+
console.error('[persistent-cache] Save error:', error);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Generate cache key from text
|
|
125
|
+
*/
|
|
126
|
+
hashKey(text) {
|
|
127
|
+
// FNV-1a hash for fast, deterministic key generation
|
|
128
|
+
let hash = 0x811c9dc5;
|
|
129
|
+
for (let i = 0; i < text.length; i++) {
|
|
130
|
+
hash ^= text.charCodeAt(i);
|
|
131
|
+
hash = (hash * 0x01000193) >>> 0;
|
|
132
|
+
}
|
|
133
|
+
return `emb_${hash.toString(16)}_${text.length}`;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Serialize Float32Array to Uint8Array for sql.js
|
|
137
|
+
*/
|
|
138
|
+
serializeEmbedding(embedding) {
|
|
139
|
+
return new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Deserialize Uint8Array to Float32Array
|
|
143
|
+
*/
|
|
144
|
+
deserializeEmbedding(data, dimensions) {
|
|
145
|
+
const buffer = new ArrayBuffer(data.length);
|
|
146
|
+
const view = new Uint8Array(buffer);
|
|
147
|
+
view.set(data);
|
|
148
|
+
return new Float32Array(buffer);
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Get embedding from cache
|
|
152
|
+
*/
|
|
153
|
+
async get(text) {
|
|
154
|
+
await this.ensureInitialized();
|
|
155
|
+
if (!this.db) {
|
|
156
|
+
this.misses++;
|
|
157
|
+
return null;
|
|
158
|
+
}
|
|
159
|
+
const key = this.hashKey(text);
|
|
160
|
+
const now = Date.now();
|
|
161
|
+
try {
|
|
162
|
+
const stmt = this.db.prepare(`
|
|
163
|
+
SELECT embedding, dimensions, created_at
|
|
164
|
+
FROM embeddings
|
|
165
|
+
WHERE key = ?
|
|
166
|
+
`);
|
|
167
|
+
stmt.bind([key]);
|
|
168
|
+
if (!stmt.step()) {
|
|
169
|
+
stmt.free();
|
|
170
|
+
this.misses++;
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
const row = stmt.getAsObject();
|
|
174
|
+
stmt.free();
|
|
175
|
+
// Check TTL
|
|
176
|
+
if (now - row.created_at > this.ttlMs) {
|
|
177
|
+
this.db.run('DELETE FROM embeddings WHERE key = ?', [key]);
|
|
178
|
+
this.dirty = true;
|
|
179
|
+
this.misses++;
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
// Update access time and count
|
|
183
|
+
this.db.run(`
|
|
184
|
+
UPDATE embeddings
|
|
185
|
+
SET accessed_at = ?, access_count = access_count + 1
|
|
186
|
+
WHERE key = ?
|
|
187
|
+
`, [now, key]);
|
|
188
|
+
this.dirty = true;
|
|
189
|
+
this.hits++;
|
|
190
|
+
return this.deserializeEmbedding(row.embedding, row.dimensions);
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
console.error('[persistent-cache] Get error:', error);
|
|
194
|
+
this.misses++;
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Store embedding in cache
|
|
200
|
+
*/
|
|
201
|
+
async set(text, embedding) {
|
|
202
|
+
await this.ensureInitialized();
|
|
203
|
+
if (!this.db)
|
|
204
|
+
return;
|
|
205
|
+
const key = this.hashKey(text);
|
|
206
|
+
const now = Date.now();
|
|
207
|
+
const data = this.serializeEmbedding(embedding);
|
|
208
|
+
try {
|
|
209
|
+
// Upsert entry using INSERT OR REPLACE
|
|
210
|
+
this.db.run(`
|
|
211
|
+
INSERT OR REPLACE INTO embeddings
|
|
212
|
+
(key, embedding, dimensions, created_at, accessed_at, access_count)
|
|
213
|
+
VALUES (?, ?, ?, ?, ?,
|
|
214
|
+
COALESCE((SELECT access_count + 1 FROM embeddings WHERE key = ?), 1)
|
|
215
|
+
)
|
|
216
|
+
`, [key, data, embedding.length, now, now, key]);
|
|
217
|
+
this.dirty = true;
|
|
218
|
+
// Check size and evict if needed
|
|
219
|
+
await this.evictIfNeeded();
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
console.error('[persistent-cache] Set error:', error);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Evict oldest entries if cache exceeds max size
|
|
227
|
+
*/
|
|
228
|
+
async evictIfNeeded() {
|
|
229
|
+
if (!this.db)
|
|
230
|
+
return;
|
|
231
|
+
const result = this.db.exec('SELECT COUNT(*) as count FROM embeddings');
|
|
232
|
+
const count = result[0]?.values[0]?.[0] ?? 0;
|
|
233
|
+
if (count > this.maxSize) {
|
|
234
|
+
const toDelete = count - this.maxSize + Math.floor(this.maxSize * 0.1); // Delete 10% extra
|
|
235
|
+
this.db.run(`
|
|
236
|
+
DELETE FROM embeddings
|
|
237
|
+
WHERE key IN (
|
|
238
|
+
SELECT key FROM embeddings
|
|
239
|
+
ORDER BY accessed_at ASC
|
|
240
|
+
LIMIT ?
|
|
241
|
+
)
|
|
242
|
+
`, [toDelete]);
|
|
243
|
+
this.dirty = true;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Clean expired entries
|
|
248
|
+
*/
|
|
249
|
+
cleanExpired() {
|
|
250
|
+
if (!this.db)
|
|
251
|
+
return;
|
|
252
|
+
const cutoff = Date.now() - this.ttlMs;
|
|
253
|
+
this.db.run('DELETE FROM embeddings WHERE created_at < ?', [cutoff]);
|
|
254
|
+
this.dirty = true;
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Get cache statistics
|
|
258
|
+
*/
|
|
259
|
+
async getStats() {
|
|
260
|
+
await this.ensureInitialized();
|
|
261
|
+
const total = this.hits + this.misses;
|
|
262
|
+
const stats = {
|
|
263
|
+
size: 0,
|
|
264
|
+
maxSize: this.maxSize,
|
|
265
|
+
hitRate: total > 0 ? this.hits / total : 0,
|
|
266
|
+
hits: this.hits,
|
|
267
|
+
misses: this.misses,
|
|
268
|
+
};
|
|
269
|
+
if (this.db) {
|
|
270
|
+
const result = this.db.exec('SELECT COUNT(*) as count FROM embeddings');
|
|
271
|
+
stats.size = result[0]?.values[0]?.[0] ?? 0;
|
|
272
|
+
// Get file size if exists
|
|
273
|
+
if (existsSync(this.dbPath)) {
|
|
274
|
+
try {
|
|
275
|
+
const buffer = readFileSync(this.dbPath);
|
|
276
|
+
stats.dbSizeBytes = buffer.length;
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
// Ignore
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return stats;
|
|
284
|
+
}
|
|
285
|
+
/**
|
|
286
|
+
* Clear all cached entries
|
|
287
|
+
*/
|
|
288
|
+
async clear() {
|
|
289
|
+
await this.ensureInitialized();
|
|
290
|
+
if (!this.db)
|
|
291
|
+
return;
|
|
292
|
+
this.db.run('DELETE FROM embeddings');
|
|
293
|
+
this.dirty = true;
|
|
294
|
+
this.hits = 0;
|
|
295
|
+
this.misses = 0;
|
|
296
|
+
this.saveToFile();
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* Force save to disk
|
|
300
|
+
*/
|
|
301
|
+
async flush() {
|
|
302
|
+
await this.ensureInitialized();
|
|
303
|
+
if (this.db && this.dirty) {
|
|
304
|
+
this.saveToFile();
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Close database connection
|
|
309
|
+
*/
|
|
310
|
+
async close() {
|
|
311
|
+
this.stopAutoSave();
|
|
312
|
+
if (this.db) {
|
|
313
|
+
// Save before closing
|
|
314
|
+
if (this.dirty) {
|
|
315
|
+
this.saveToFile();
|
|
316
|
+
}
|
|
317
|
+
this.db.close();
|
|
318
|
+
this.db = null;
|
|
319
|
+
this.SQL = null;
|
|
320
|
+
this.initialized = false;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Check if persistent cache is available (sql.js installed)
|
|
326
|
+
*/
|
|
327
|
+
export async function isPersistentCacheAvailable() {
|
|
328
|
+
try {
|
|
329
|
+
const initSqlJs = (await import('sql.js')).default;
|
|
330
|
+
await initSqlJs();
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
catch {
|
|
334
|
+
return false;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
//# sourceMappingURL=persistent-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"persistent-cache.js","sourceRoot":"","sources":["../src/persistent-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAmC/B;;GAEG;AACH,MAAM,OAAO,wBAAwB;IAC3B,EAAE,GAAyB,IAAI,CAAC;IAChC,GAAG,GAAuB,IAAI,CAAC;IAC/B,WAAW,GAAG,KAAK,CAAC;IACpB,KAAK,GAAG,KAAK,CAAC;IACd,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IACX,aAAa,GAA0C,IAAI,CAAC;IAEnD,MAAM,CAAS;IACf,OAAO,CAAS;IAChB,KAAK,CAAS;IACd,gBAAgB,CAAS;IAE1C,YAAY,MAA6B;QACvC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,KAAK,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,SAAS;QAC/D,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,KAAK,CAAC,CAAC,aAAa;IACzE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,IAAI,CAAC;YACH,4BAA4B;YAC5B,MAAM,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;YAEnD,iCAAiC;YACjC,IAAI,CAAC,GAAG,GAAG,MAAM,SAAS,EAAE,CAAC;YAE7B,0BAA0B;YAC1B,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YAED,uCAAuC;YACvC,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBAC7C,IAAI,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACpC,CAAC;YAED,gBAAgB;YAChB,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;;;;;;;;;OASX,CAAC,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC;YACrF,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,qEAAqE,CAAC,CAAC;YAEnF,mCAAmC;YACnC,IAAI,CAAC,YAAY,EAAE,CAAC;YAEpB,8CAA8C;YAC9C,IAAI,CAAC,UAAU,EAAE,CAAC;YAElB,wBAAwB;YACxB,IAAI,CAAC,aAAa,EAAE,CAAC;YAErB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gDAAgD;YAChD,OAAO,CAAC,IAAI,CAAC,0DAA0D,EACrE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAClD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,uCAAuC;QAClE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,IAAI,CAAC,aAAa;YAAE,OAAO;QAE/B,IAAI,CAAC,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;YACpC,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBAC1B,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAClC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,UAAU;QAChB,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,OAAO,CAAC,IAAY;QAC1B,qDAAqD;QACrD,IAAI,IAAI,GAAG,UAAU,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,GAAG,CAAC,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAuB;QAChD,OAAO,IAAI,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;IACtF,CAAC;IAED;;OAEG;IACK,oBAAoB,CAAC,IAAgB,EAAE,UAAkB;QAC/D,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACf,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,IAAY;QACpB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAI5B,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAEjB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACZ,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAI3B,CAAC;YACF,IAAI,CAAC,IAAI,EAAE,CAAC;YAEZ,YAAY;YACZ,IAAI,GAAG,GAAG,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;gBACtC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,sCAAsC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC3D,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;gBAClB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC;YACd,CAAC;YAED,+BAA+B;YAC/B,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;;;;OAIX,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAElB,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;YACtD,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,SAAuB;QAC7C,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEhD,IAAI,CAAC;YACH,uCAAuC;YACvC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;;;;;;OAMX,EAAE,CAAC,GAAG,EAAE,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YACjD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAElB,iCAAiC;YACjC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,aAAa;QACzB,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;QACxE,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAW,IAAI,CAAC,CAAC;QAEvD,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,KAAK,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,mBAAmB;YAC3F,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC;;;;;;;OAOX,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;YACf,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,6CAA6C,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,MAAM,KAAK,GAAyB;YAClC,IAAI,EAAE,CAAC;YACP,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;QAEF,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;YACxE,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAW,IAAI,CAAC,CAAC;YAEtD,0BAA0B;YAC1B,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACzC,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;gBACpC,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,EAAE;YAAE,OAAO;QAErB,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChB,IAAI,CAAC,UAAU,EAAE,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,YAAY,EAAE,CAAC;QAEpB,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,sBAAsB;YACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACf,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;YAChB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B;IAC9C,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;QACnD,MAAM,SAAS,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|