@sparkleideas/embeddings 3.0.0-alpha.12-patch.26 → 3.0.0-alpha.12-patch.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/embedding-service.test.d.ts +2 -0
- package/dist/__tests__/embedding-service.test.d.ts.map +1 -0
- package/dist/__tests__/embedding-service.test.js +98 -0
- package/dist/__tests__/embedding-service.test.js.map +1 -0
- package/dist/chunking.d.ts +68 -0
- package/dist/chunking.d.ts.map +1 -0
- package/dist/chunking.js +251 -0
- package/dist/chunking.js.map +1 -0
- package/dist/embedding-service.d.ts +207 -0
- package/dist/embedding-service.d.ts.map +1 -0
- package/dist/embedding-service.js +965 -0
- package/dist/embedding-service.js.map +1 -0
- package/dist/hyperbolic.d.ts +103 -0
- package/dist/hyperbolic.d.ts.map +1 -0
- package/dist/hyperbolic.js +343 -0
- package/dist/hyperbolic.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +37 -0
- package/dist/index.js.map +1 -0
- package/dist/neural-integration.d.ts +203 -0
- package/dist/neural-integration.d.ts.map +1 -0
- package/dist/neural-integration.js +213 -0
- package/dist/neural-integration.js.map +1 -0
- package/dist/normalization.d.ts +73 -0
- package/dist/normalization.d.ts.map +1 -0
- package/dist/normalization.js +192 -0
- package/dist/normalization.js.map +1 -0
- package/dist/persistent-cache.d.ts +119 -0
- package/dist/persistent-cache.d.ts.map +1 -0
- package/dist/persistent-cache.js +337 -0
- package/dist/persistent-cache.js.map +1 -0
- package/dist/rvf-embedding-cache.d.ts +118 -0
- package/dist/rvf-embedding-cache.d.ts.map +1 -0
- package/dist/rvf-embedding-cache.js +458 -0
- package/dist/rvf-embedding-cache.js.map +1 -0
- package/dist/rvf-embedding-service.d.ts +79 -0
- package/dist/rvf-embedding-service.d.ts.map +1 -0
- package/dist/rvf-embedding-service.js +318 -0
- package/dist/rvf-embedding-service.js.map +1 -0
- package/dist/types.d.ts +237 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +15 -0
- package/dist/types.js.map +1 -0
- package/package.json +3 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rvf-embedding-service.d.ts","sourceRoot":"","sources":["../src/rvf-embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,KAAK,EACV,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,iBAAiB,EAEjB,sBAAsB,EAEtB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAsFpB;;;;;;;;;GASG;AACH,qBAAa,mBAAoB,SAAQ,YAAa,YAAW,iBAAiB;IAChF,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IACvD,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoB;IACtD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA0C;IAC7E,OAAO,CAAC,eAAe,CAAkC;gBAE7C,MAAM,EAAE,kBAAkB;IAuBtC;;OAEG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA4CnD;;OAEG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAyDhE,UAAU,IAAI,IAAI;IAMlB,aAAa;;;;;IASP,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IAY/B,gBAAgB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,IAAI;IAIxD,mBAAmB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,IAAI;IAI3D,OAAO,CAAC,SAAS;IAejB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAM3B"}
|
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RVF Embedding Service - Lightweight Hash-Based Embeddings
|
|
3
|
+
*
|
|
4
|
+
* Provides deterministic, sub-millisecond embedding generation using
|
|
5
|
+
* FNV-1a hash-based vectors. No neural model or external API required.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Deterministic: same input always produces the same embedding
|
|
9
|
+
* - FNV-1a hash seeding with multi-round mixing
|
|
10
|
+
* - L2-normalized output vectors
|
|
11
|
+
* - Sub-millisecond generation (<0.1ms typical)
|
|
12
|
+
* - RvfEmbeddingCache for binary file persistence
|
|
13
|
+
* - Zero external dependencies
|
|
14
|
+
*
|
|
15
|
+
* Use cases:
|
|
16
|
+
* - Fast similarity search where relative distances matter more than semantics
|
|
17
|
+
* - Development and testing without API keys
|
|
18
|
+
* - Offline environments without neural model access
|
|
19
|
+
* - Bootstrapping before heavier providers are available
|
|
20
|
+
*
|
|
21
|
+
* @module @sparkleideas/embeddings
|
|
22
|
+
*/
|
|
23
|
+
import { EventEmitter } from 'events';
|
|
24
|
+
import { normalize } from './normalization.js';
|
|
25
|
+
import { RvfEmbeddingCache } from './rvf-embedding-cache.js';
|
|
26
|
+
// ============================================================================
|
|
27
|
+
// Constants
|
|
28
|
+
// ============================================================================
|
|
29
|
+
/** FNV-1a offset basis (32-bit) */
|
|
30
|
+
const FNV_OFFSET_BASIS = 0x811c9dc5;
|
|
31
|
+
/** FNV-1a prime (32-bit) */
|
|
32
|
+
const FNV_PRIME = 0x01000193;
|
|
33
|
+
/** Default embedding dimensions */
|
|
34
|
+
const DEFAULT_DIMENSIONS = 384;
|
|
35
|
+
/** Default in-memory LRU cache size */
|
|
36
|
+
const DEFAULT_CACHE_SIZE = 1000;
|
|
37
|
+
// ============================================================================
|
|
38
|
+
// LRU Cache (lightweight in-memory)
|
|
39
|
+
// ============================================================================
|
|
40
|
+
class LRUCache {
|
|
41
|
+
maxSize;
|
|
42
|
+
cache = new Map();
|
|
43
|
+
hits = 0;
|
|
44
|
+
misses = 0;
|
|
45
|
+
constructor(maxSize) {
|
|
46
|
+
this.maxSize = maxSize;
|
|
47
|
+
}
|
|
48
|
+
get(key) {
|
|
49
|
+
const value = this.cache.get(key);
|
|
50
|
+
if (value !== undefined) {
|
|
51
|
+
// Move to end (most recently used)
|
|
52
|
+
this.cache.delete(key);
|
|
53
|
+
this.cache.set(key, value);
|
|
54
|
+
this.hits++;
|
|
55
|
+
return value;
|
|
56
|
+
}
|
|
57
|
+
this.misses++;
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
set(key, value) {
|
|
61
|
+
if (this.cache.has(key)) {
|
|
62
|
+
this.cache.delete(key);
|
|
63
|
+
}
|
|
64
|
+
else if (this.cache.size >= this.maxSize) {
|
|
65
|
+
const firstKey = this.cache.keys().next().value;
|
|
66
|
+
if (firstKey !== undefined) {
|
|
67
|
+
this.cache.delete(firstKey);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
this.cache.set(key, value);
|
|
71
|
+
}
|
|
72
|
+
clear() {
|
|
73
|
+
this.cache.clear();
|
|
74
|
+
this.hits = 0;
|
|
75
|
+
this.misses = 0;
|
|
76
|
+
}
|
|
77
|
+
get size() {
|
|
78
|
+
return this.cache.size;
|
|
79
|
+
}
|
|
80
|
+
get hitRate() {
|
|
81
|
+
const total = this.hits + this.misses;
|
|
82
|
+
return total > 0 ? this.hits / total : 0;
|
|
83
|
+
}
|
|
84
|
+
getStats() {
|
|
85
|
+
return {
|
|
86
|
+
size: this.cache.size,
|
|
87
|
+
maxSize: this.maxSize,
|
|
88
|
+
hits: this.hits,
|
|
89
|
+
misses: this.misses,
|
|
90
|
+
hitRate: this.hitRate,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
// ============================================================================
|
|
95
|
+
// RVF Embedding Service
|
|
96
|
+
// ============================================================================
|
|
97
|
+
/**
|
|
98
|
+
* Lightweight hash-based embedding service.
|
|
99
|
+
*
|
|
100
|
+
* Generates deterministic embeddings from text using FNV-1a hashing
|
|
101
|
+
* with multi-round mixing and L2 normalization. The output is a unit
|
|
102
|
+
* vector in R^n where n = configured dimensions (default 384).
|
|
103
|
+
*
|
|
104
|
+
* Extends EventEmitter and implements IEmbeddingService for drop-in
|
|
105
|
+
* compatibility with other providers.
|
|
106
|
+
*/
|
|
107
|
+
export class RvfEmbeddingService extends EventEmitter {
|
|
108
|
+
provider = 'rvf';
|
|
109
|
+
dimensions;
|
|
110
|
+
cache;
|
|
111
|
+
normalizationType;
|
|
112
|
+
embeddingListeners = new Set();
|
|
113
|
+
persistentCache = null;
|
|
114
|
+
constructor(config) {
|
|
115
|
+
super();
|
|
116
|
+
this.dimensions = config.dimensions ?? DEFAULT_DIMENSIONS;
|
|
117
|
+
if (this.dimensions <= 0 || !Number.isInteger(this.dimensions)) {
|
|
118
|
+
throw new Error(`Invalid dimensions: ${this.dimensions}. Must be a positive integer.`);
|
|
119
|
+
}
|
|
120
|
+
this.cache = new LRUCache(config.cacheSize ?? DEFAULT_CACHE_SIZE);
|
|
121
|
+
this.normalizationType = config.normalization ?? 'none';
|
|
122
|
+
// Initialize persistent RVF cache if a path is provided
|
|
123
|
+
if (config.cachePath) {
|
|
124
|
+
this.persistentCache = new RvfEmbeddingCache({
|
|
125
|
+
cachePath: config.cachePath,
|
|
126
|
+
maxSize: config.cacheSize ?? 10000,
|
|
127
|
+
dimensions: this.dimensions,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
// --------------------------------------------------------------------------
|
|
132
|
+
// IEmbeddingService Implementation
|
|
133
|
+
// --------------------------------------------------------------------------
|
|
134
|
+
/**
|
|
135
|
+
* Generate an embedding for a single text string.
|
|
136
|
+
*/
|
|
137
|
+
async embed(text) {
|
|
138
|
+
if (typeof text !== 'string') {
|
|
139
|
+
throw new Error('embed() expects a string argument');
|
|
140
|
+
}
|
|
141
|
+
// Check in-memory cache
|
|
142
|
+
const cached = this.cache.get(text);
|
|
143
|
+
if (cached) {
|
|
144
|
+
this.emitEvent({ type: 'cache_hit', text });
|
|
145
|
+
return { embedding: cached, latencyMs: 0, cached: true };
|
|
146
|
+
}
|
|
147
|
+
// Check persistent cache
|
|
148
|
+
if (this.persistentCache) {
|
|
149
|
+
const persisted = await this.persistentCache.get(text);
|
|
150
|
+
if (persisted) {
|
|
151
|
+
this.cache.set(text, persisted);
|
|
152
|
+
this.emitEvent({ type: 'cache_hit', text });
|
|
153
|
+
return { embedding: persisted, latencyMs: 0, cached: true, persistentCached: true };
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
this.emitEvent({ type: 'embed_start', text });
|
|
157
|
+
const startTime = performance.now();
|
|
158
|
+
// Generate deterministic embedding
|
|
159
|
+
const embedding = this.generateHashEmbedding(text);
|
|
160
|
+
// Apply optional normalization (hash embeddings are already L2-normalized,
|
|
161
|
+
// but the user may want a different normalization)
|
|
162
|
+
const normalized = this.applyNormalization(embedding);
|
|
163
|
+
// Store in caches
|
|
164
|
+
this.cache.set(text, normalized);
|
|
165
|
+
if (this.persistentCache) {
|
|
166
|
+
await this.persistentCache.set(text, normalized);
|
|
167
|
+
}
|
|
168
|
+
const latencyMs = performance.now() - startTime;
|
|
169
|
+
this.emitEvent({ type: 'embed_complete', text, latencyMs });
|
|
170
|
+
return { embedding: normalized, latencyMs };
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* Generate embeddings for multiple text strings.
|
|
174
|
+
*/
|
|
175
|
+
async embedBatch(texts) {
|
|
176
|
+
if (!Array.isArray(texts)) {
|
|
177
|
+
throw new Error('embedBatch() expects an array of strings');
|
|
178
|
+
}
|
|
179
|
+
this.emitEvent({ type: 'batch_start', count: texts.length });
|
|
180
|
+
const startTime = performance.now();
|
|
181
|
+
const embeddings = [];
|
|
182
|
+
let cacheHits = 0;
|
|
183
|
+
for (const text of texts) {
|
|
184
|
+
const cached = this.cache.get(text);
|
|
185
|
+
if (cached) {
|
|
186
|
+
embeddings.push(cached);
|
|
187
|
+
cacheHits++;
|
|
188
|
+
this.emitEvent({ type: 'cache_hit', text });
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
// Check persistent cache
|
|
192
|
+
if (this.persistentCache) {
|
|
193
|
+
const persisted = await this.persistentCache.get(text);
|
|
194
|
+
if (persisted) {
|
|
195
|
+
this.cache.set(text, persisted);
|
|
196
|
+
embeddings.push(persisted);
|
|
197
|
+
cacheHits++;
|
|
198
|
+
this.emitEvent({ type: 'cache_hit', text });
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const embedding = this.generateHashEmbedding(text);
|
|
203
|
+
const normalized = this.applyNormalization(embedding);
|
|
204
|
+
this.cache.set(text, normalized);
|
|
205
|
+
if (this.persistentCache) {
|
|
206
|
+
await this.persistentCache.set(text, normalized);
|
|
207
|
+
}
|
|
208
|
+
embeddings.push(normalized);
|
|
209
|
+
}
|
|
210
|
+
const totalLatencyMs = performance.now() - startTime;
|
|
211
|
+
this.emitEvent({ type: 'batch_complete', count: texts.length, latencyMs: totalLatencyMs });
|
|
212
|
+
return {
|
|
213
|
+
embeddings,
|
|
214
|
+
totalLatencyMs,
|
|
215
|
+
avgLatencyMs: totalLatencyMs / texts.length,
|
|
216
|
+
cacheStats: {
|
|
217
|
+
hits: cacheHits,
|
|
218
|
+
misses: texts.length - cacheHits,
|
|
219
|
+
},
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
clearCache() {
|
|
223
|
+
const size = this.cache.size;
|
|
224
|
+
this.cache.clear();
|
|
225
|
+
this.emitEvent({ type: 'cache_eviction', size });
|
|
226
|
+
}
|
|
227
|
+
getCacheStats() {
|
|
228
|
+
const stats = this.cache.getStats();
|
|
229
|
+
return {
|
|
230
|
+
size: stats.size,
|
|
231
|
+
maxSize: stats.maxSize,
|
|
232
|
+
hitRate: stats.hitRate,
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
async shutdown() {
|
|
236
|
+
this.clearCache();
|
|
237
|
+
this.embeddingListeners.clear();
|
|
238
|
+
if (this.persistentCache) {
|
|
239
|
+
await this.persistentCache.close();
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
// --------------------------------------------------------------------------
|
|
243
|
+
// Event System
|
|
244
|
+
// --------------------------------------------------------------------------
|
|
245
|
+
addEventListener(listener) {
|
|
246
|
+
this.embeddingListeners.add(listener);
|
|
247
|
+
}
|
|
248
|
+
removeEventListener(listener) {
|
|
249
|
+
this.embeddingListeners.delete(listener);
|
|
250
|
+
}
|
|
251
|
+
emitEvent(event) {
|
|
252
|
+
for (const listener of this.embeddingListeners) {
|
|
253
|
+
try {
|
|
254
|
+
listener(event);
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
console.error('Error in embedding event listener:', error);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
this.emit(event.type, event);
|
|
261
|
+
}
|
|
262
|
+
// --------------------------------------------------------------------------
|
|
263
|
+
// Hash Embedding Generation
|
|
264
|
+
// --------------------------------------------------------------------------
|
|
265
|
+
/**
|
|
266
|
+
* Generate a deterministic embedding from text using FNV-1a hashing.
|
|
267
|
+
*
|
|
268
|
+
* Algorithm:
|
|
269
|
+
* 1. Compute a base FNV-1a hash of the full text.
|
|
270
|
+
* 2. For each dimension, derive a unique seed by mixing the base hash
|
|
271
|
+
* with the dimension index using the golden ratio constant.
|
|
272
|
+
* 3. Apply a sine-based pseudo-random transform to spread values.
|
|
273
|
+
* 4. L2-normalize the result to produce a unit vector.
|
|
274
|
+
*
|
|
275
|
+
* This is deterministic: the same text always yields the same vector.
|
|
276
|
+
*/
|
|
277
|
+
generateHashEmbedding(text) {
|
|
278
|
+
const embedding = new Float32Array(this.dimensions);
|
|
279
|
+
// Compute base FNV-1a hash of the full text
|
|
280
|
+
let baseHash = FNV_OFFSET_BASIS;
|
|
281
|
+
for (let i = 0; i < text.length; i++) {
|
|
282
|
+
baseHash ^= text.charCodeAt(i);
|
|
283
|
+
baseHash = Math.imul(baseHash, FNV_PRIME) >>> 0;
|
|
284
|
+
}
|
|
285
|
+
// Generate each dimension from a mixed seed
|
|
286
|
+
for (let i = 0; i < this.dimensions; i++) {
|
|
287
|
+
// Mix dimension index with the base hash using golden ratio constant
|
|
288
|
+
const seed = (baseHash + Math.imul(i, 0x9E3779B9)) >>> 0;
|
|
289
|
+
// Use sine for pseudo-random distribution in [-1, 1] range
|
|
290
|
+
const x = Math.sin(seed) * 43758.5453;
|
|
291
|
+
embedding[i] = x - Math.floor(x); // fractional part in [0, 1)
|
|
292
|
+
// Shift to [-0.5, 0.5) for zero-centered distribution
|
|
293
|
+
embedding[i] -= 0.5;
|
|
294
|
+
}
|
|
295
|
+
// L2 normalize to unit vector
|
|
296
|
+
let norm = 0;
|
|
297
|
+
for (let i = 0; i < this.dimensions; i++) {
|
|
298
|
+
norm += embedding[i] * embedding[i];
|
|
299
|
+
}
|
|
300
|
+
norm = Math.sqrt(norm);
|
|
301
|
+
if (norm > 0) {
|
|
302
|
+
for (let i = 0; i < this.dimensions; i++) {
|
|
303
|
+
embedding[i] /= norm;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return embedding;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Apply user-configured normalization if set.
|
|
310
|
+
*/
|
|
311
|
+
applyNormalization(embedding) {
|
|
312
|
+
if (this.normalizationType === 'none') {
|
|
313
|
+
return embedding;
|
|
314
|
+
}
|
|
315
|
+
return normalize(embedding, { type: this.normalizationType });
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
//# sourceMappingURL=rvf-embedding-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rvf-embedding-service.js","sourceRoot":"","sources":["../src/rvf-embedding-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAWtC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,+EAA+E;AAC/E,YAAY;AACZ,+EAA+E;AAE/E,mCAAmC;AACnC,MAAM,gBAAgB,GAAG,UAAU,CAAC;AAEpC,4BAA4B;AAC5B,MAAM,SAAS,GAAG,UAAU,CAAC;AAE7B,mCAAmC;AACnC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,uCAAuC;AACvC,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC,+EAA+E;AAC/E,oCAAoC;AACpC,+EAA+E;AAE/E,MAAM,QAAQ;IAKiB;IAJrB,KAAK,GAAc,IAAI,GAAG,EAAE,CAAC;IAC7B,IAAI,GAAG,CAAC,CAAC;IACT,MAAM,GAAG,CAAC,CAAC;IAEnB,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,GAAG,CAAC,GAAM;QACR,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,mCAAmC;YACnC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,IAAI,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,GAAG,CAAC,GAAM,EAAE,KAAQ;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;aAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;YAChD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,IAAI,OAAO;QACT,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC;QACtC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,QAAQ;QACN,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;CACF;AAED,+EAA+E;AAC/E,wBAAwB;AACxB,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,OAAO,mBAAoB,SAAQ,YAAY;IAC1C,QAAQ,GAAsB,KAAK,CAAC;IAE5B,UAAU,CAAS;IACnB,KAAK,CAAiC;IACtC,iBAAiB,CAAoB;IACrC,kBAAkB,GAAgC,IAAI,GAAG,EAAE,CAAC;IACrE,eAAe,GAA6B,IAAI,CAAC;IAEzD,YAAY,MAA0B;QACpC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAC1D,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,UAAU,+BAA+B,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;QAClE,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC;QAExD,wDAAwD;QACxD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,eAAe,GAAG,IAAI,iBAAiB,CAAC;gBAC3C,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,OAAO,EAAE,MAAM,CAAC,SAAS,IAAI,KAAK;gBAClC,UAAU,EAAE,IAAI,CAAC,UAAU;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,mCAAmC;IACnC,6EAA6E;IAE7E;;OAEG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QAED,wBAAwB;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;YAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC3D,CAAC;QAED,yBAAyB;QACzB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvD,IAAI,SAAS,EAAE,CAAC;gBACd,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;gBAChC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5C,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;YACtF,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,mCAAmC;QACnC,MAAM,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAEnD,2EAA2E;QAC3E,mDAAmD;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QAEtD,kBAAkB;QAClB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAChD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5D,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAEpC,MAAM,UAAU,GAAmB,EAAE,CAAC;QACtC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACX,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACxB,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5C,SAAS;YACX,CAAC;YAED,yBAAyB;YACzB,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvD,IAAI,SAAS,EAAE,CAAC;oBACd,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;oBAChC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC3B,SAAS,EAAE,CAAC;oBACZ,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC5C,SAAS;gBACX,CAAC;YACH,CAAC;YAED,MAAM,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;YACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YAEjC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACnD,CAAC;YAED,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACrD,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;QAE3F,OAAO;YACL,UAAU;YACV,cAAc;YACd,YAAY,EAAE,cAAc,GAAG,KAAK,CAAC,MAAM;YAC3C,UAAU,EAAE;gBACV,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,SAAS;aACjC;SACF,CAAC;IACJ,CAAC;IAED,UAAU;QACR,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACnB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,aAAa;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACpC,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,OAAO;SACvB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,CAAC;QAChC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QACrC,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,eAAe;IACf,6EAA6E;IAE7E,gBAAgB,CAAC,QAAgC;QAC/C,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,mBAAmB,CAAC,QAAgC;QAClD,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC3C,CAAC;IAEO,SAAS,CAAC,KAAqB;QACrC,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC/C,IAAI,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IAC7E,4BAA4B;IAC5B,6EAA6E;IAE7E;;;;;;;;;;;OAWG;IACK,qBAAqB,CAAC,IAAY;QACxC,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEpD,4CAA4C;QAC5C,IAAI,QAAQ,GAAG,gBAAgB,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,QAAQ,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YAC/B,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAClD,CAAC;QAED,4CAA4C;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,qEAAqE;YACrE,MAAM,IAAI,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC;YACzD,2DAA2D;YAC3D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC;YACtC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;YAC9D,sDAAsD;YACtD,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QACtB,CAAC;QAED,8BAA8B;QAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,SAAuB;QAChD,IAAI,IAAI,CAAC,iBAAiB,KAAK,MAAM,EAAE,CAAC;YACtC,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,SAAS,CAAC,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAChE,CAAC;CACF"}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V3 Embedding Service Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for embedding service aligned with agentic-flow@alpha:
|
|
5
|
+
* - OpenAI provider
|
|
6
|
+
* - Transformers.js provider
|
|
7
|
+
* - Mock provider
|
|
8
|
+
*
|
|
9
|
+
* Performance Targets:
|
|
10
|
+
* - Single embedding: <100ms (API), <50ms (local)
|
|
11
|
+
* - Batch embedding: <500ms for 10 items
|
|
12
|
+
* - Cache hit: <1ms
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Supported embedding providers
|
|
16
|
+
*/
|
|
17
|
+
export type EmbeddingProvider = 'openai' | 'transformers' | 'mock' | 'agentic-flow' | 'rvf';
|
|
18
|
+
/**
|
|
19
|
+
* Normalization type for embeddings
|
|
20
|
+
*/
|
|
21
|
+
export type NormalizationType = 'l2' | 'l1' | 'minmax' | 'zscore' | 'none';
|
|
22
|
+
/**
|
|
23
|
+
* Persistent cache configuration
|
|
24
|
+
*/
|
|
25
|
+
export interface PersistentCacheConfig {
|
|
26
|
+
/** Enable persistent disk cache (requires better-sqlite3) */
|
|
27
|
+
enabled: boolean;
|
|
28
|
+
/** Path to SQLite database file (default: .cache/embeddings.db) */
|
|
29
|
+
dbPath?: string;
|
|
30
|
+
/** Maximum entries in persistent cache (default: 10000) */
|
|
31
|
+
maxSize?: number;
|
|
32
|
+
/** TTL in milliseconds (default: 7 days) */
|
|
33
|
+
ttlMs?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Base configuration for all providers
|
|
37
|
+
*/
|
|
38
|
+
export interface EmbeddingBaseConfig {
|
|
39
|
+
/** Provider identifier */
|
|
40
|
+
provider: EmbeddingProvider;
|
|
41
|
+
/** Embedding dimensions */
|
|
42
|
+
dimensions?: number;
|
|
43
|
+
/** Cache size (number of embeddings) */
|
|
44
|
+
cacheSize?: number;
|
|
45
|
+
/** Enable caching */
|
|
46
|
+
enableCache?: boolean;
|
|
47
|
+
/** Normalization type (default: 'none' - most providers pre-normalize) */
|
|
48
|
+
normalization?: NormalizationType;
|
|
49
|
+
/** Persistent disk cache configuration */
|
|
50
|
+
persistentCache?: PersistentCacheConfig;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* OpenAI provider configuration
|
|
54
|
+
*/
|
|
55
|
+
export interface OpenAIEmbeddingConfig extends EmbeddingBaseConfig {
|
|
56
|
+
provider: 'openai';
|
|
57
|
+
/** OpenAI API key */
|
|
58
|
+
apiKey: string;
|
|
59
|
+
/** Model to use */
|
|
60
|
+
model?: 'text-embedding-3-small' | 'text-embedding-3-large' | 'text-embedding-ada-002';
|
|
61
|
+
/** Target dimensions (for text-embedding-3-* models) */
|
|
62
|
+
dimensions?: number;
|
|
63
|
+
/** Base URL override */
|
|
64
|
+
baseURL?: string;
|
|
65
|
+
/** Request timeout in ms */
|
|
66
|
+
timeout?: number;
|
|
67
|
+
/** Max retries */
|
|
68
|
+
maxRetries?: number;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Transformers.js provider configuration
|
|
72
|
+
*/
|
|
73
|
+
export interface TransformersEmbeddingConfig extends EmbeddingBaseConfig {
|
|
74
|
+
provider: 'transformers';
|
|
75
|
+
/** Model name from Hugging Face */
|
|
76
|
+
model?: string;
|
|
77
|
+
/** Quantization level */
|
|
78
|
+
quantized?: boolean;
|
|
79
|
+
/** Use web worker */
|
|
80
|
+
useWorker?: boolean;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Mock provider configuration
|
|
84
|
+
*/
|
|
85
|
+
export interface MockEmbeddingConfig extends EmbeddingBaseConfig {
|
|
86
|
+
provider: 'mock';
|
|
87
|
+
/** Output dimensions */
|
|
88
|
+
dimensions?: number;
|
|
89
|
+
/** Simulated latency in ms */
|
|
90
|
+
simulatedLatency?: number;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Agentic-flow provider configuration
|
|
94
|
+
* Uses optimized ONNX embeddings with:
|
|
95
|
+
* - Float32Array with flattened matrices
|
|
96
|
+
* - 256-entry LRU cache with FNV-1a hash
|
|
97
|
+
* - SIMD-friendly loop unrolling (4x)
|
|
98
|
+
* - Pre-allocated buffers (no GC pressure)
|
|
99
|
+
*/
|
|
100
|
+
export interface AgenticFlowEmbeddingConfig extends EmbeddingBaseConfig {
|
|
101
|
+
provider: 'agentic-flow';
|
|
102
|
+
/** Model ID (default: all-MiniLM-L6-v2) */
|
|
103
|
+
modelId?: string;
|
|
104
|
+
/** Embedding dimensions (default: 384) */
|
|
105
|
+
dimensions?: number;
|
|
106
|
+
/** Internal cache size for embedder (default: 256) */
|
|
107
|
+
embedderCacheSize?: number;
|
|
108
|
+
/** Model directory path */
|
|
109
|
+
modelDir?: string;
|
|
110
|
+
/** Auto-download model if not present */
|
|
111
|
+
autoDownload?: boolean;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* RVF provider configuration
|
|
115
|
+
* Lightweight hash-based embeddings (no neural model, sub-ms latency)
|
|
116
|
+
*/
|
|
117
|
+
export interface RvfEmbeddingConfig extends EmbeddingBaseConfig {
|
|
118
|
+
provider: 'rvf';
|
|
119
|
+
/** Embedding dimensions (default: 384) */
|
|
120
|
+
dimensions?: number;
|
|
121
|
+
/** Path to binary cache file for persistent storage */
|
|
122
|
+
cachePath?: string;
|
|
123
|
+
/** Similarity metric preference (default: 'cosine') */
|
|
124
|
+
metric?: 'cosine' | 'l2' | 'dotproduct';
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Union of all provider configs
|
|
128
|
+
*/
|
|
129
|
+
export type EmbeddingConfig = OpenAIEmbeddingConfig | TransformersEmbeddingConfig | MockEmbeddingConfig | AgenticFlowEmbeddingConfig | RvfEmbeddingConfig;
|
|
130
|
+
/**
|
|
131
|
+
* Single embedding result
|
|
132
|
+
*/
|
|
133
|
+
export interface EmbeddingResult {
|
|
134
|
+
/** The embedding vector */
|
|
135
|
+
embedding: Float32Array | number[];
|
|
136
|
+
/** Latency in milliseconds */
|
|
137
|
+
latencyMs: number;
|
|
138
|
+
/** Token usage (for API providers) */
|
|
139
|
+
usage?: {
|
|
140
|
+
promptTokens: number;
|
|
141
|
+
totalTokens: number;
|
|
142
|
+
};
|
|
143
|
+
/** Whether result was from cache */
|
|
144
|
+
cached?: boolean;
|
|
145
|
+
/** Whether result was from persistent cache */
|
|
146
|
+
persistentCached?: boolean;
|
|
147
|
+
/** Whether embedding was normalized */
|
|
148
|
+
normalized?: boolean;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Batch embedding result
|
|
152
|
+
*/
|
|
153
|
+
export interface BatchEmbeddingResult {
|
|
154
|
+
/** Array of embeddings */
|
|
155
|
+
embeddings: Array<Float32Array | number[]>;
|
|
156
|
+
/** Total latency in milliseconds */
|
|
157
|
+
totalLatencyMs: number;
|
|
158
|
+
/** Average latency per embedding */
|
|
159
|
+
avgLatencyMs: number;
|
|
160
|
+
/** Token usage (for API providers) */
|
|
161
|
+
usage?: {
|
|
162
|
+
promptTokens: number;
|
|
163
|
+
totalTokens: number;
|
|
164
|
+
};
|
|
165
|
+
/** Cache statistics */
|
|
166
|
+
cacheStats?: {
|
|
167
|
+
hits: number;
|
|
168
|
+
misses: number;
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Embedding service interface
|
|
173
|
+
*/
|
|
174
|
+
export interface IEmbeddingService {
|
|
175
|
+
/** Provider identifier */
|
|
176
|
+
readonly provider: EmbeddingProvider;
|
|
177
|
+
/** Get embedding for single text */
|
|
178
|
+
embed(text: string): Promise<EmbeddingResult>;
|
|
179
|
+
/** Get embeddings for multiple texts */
|
|
180
|
+
embedBatch(texts: string[]): Promise<BatchEmbeddingResult>;
|
|
181
|
+
/** Clear cache */
|
|
182
|
+
clearCache(): void;
|
|
183
|
+
/** Get cache statistics */
|
|
184
|
+
getCacheStats(): {
|
|
185
|
+
size: number;
|
|
186
|
+
maxSize: number;
|
|
187
|
+
hitRate: number;
|
|
188
|
+
};
|
|
189
|
+
/** Shutdown service */
|
|
190
|
+
shutdown(): Promise<void>;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Embedding service events
|
|
194
|
+
*/
|
|
195
|
+
export type EmbeddingEvent = {
|
|
196
|
+
type: 'embed_start';
|
|
197
|
+
text: string;
|
|
198
|
+
} | {
|
|
199
|
+
type: 'embed_complete';
|
|
200
|
+
text: string;
|
|
201
|
+
latencyMs: number;
|
|
202
|
+
} | {
|
|
203
|
+
type: 'embed_error';
|
|
204
|
+
text: string;
|
|
205
|
+
error: string;
|
|
206
|
+
} | {
|
|
207
|
+
type: 'batch_start';
|
|
208
|
+
count: number;
|
|
209
|
+
} | {
|
|
210
|
+
type: 'batch_complete';
|
|
211
|
+
count: number;
|
|
212
|
+
latencyMs: number;
|
|
213
|
+
} | {
|
|
214
|
+
type: 'cache_hit';
|
|
215
|
+
text: string;
|
|
216
|
+
} | {
|
|
217
|
+
type: 'cache_eviction';
|
|
218
|
+
size: number;
|
|
219
|
+
};
|
|
220
|
+
/**
|
|
221
|
+
* Event listener type
|
|
222
|
+
*/
|
|
223
|
+
export type EmbeddingEventListener = (event: EmbeddingEvent) => void | Promise<void>;
|
|
224
|
+
/**
|
|
225
|
+
* Similarity metric type
|
|
226
|
+
*/
|
|
227
|
+
export type SimilarityMetric = 'cosine' | 'euclidean' | 'dot';
|
|
228
|
+
/**
|
|
229
|
+
* Similarity result
|
|
230
|
+
*/
|
|
231
|
+
export interface SimilarityResult {
|
|
232
|
+
/** Similarity score (0-1 for cosine, unbounded for others) */
|
|
233
|
+
score: number;
|
|
234
|
+
/** Metric used */
|
|
235
|
+
metric: SimilarityMetric;
|
|
236
|
+
}
|
|
237
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,cAAc,GAAG,MAAM,GAAG,cAAc,GAAG,KAAK,CAAC;AAE5F;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,IAAI,GAAG,IAAI,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE3E;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,6DAA6D;IAC7D,OAAO,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0BAA0B;IAC1B,QAAQ,EAAE,iBAAiB,CAAC;IAE5B,2BAA2B;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,wCAAwC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,qBAAqB;IACrB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,iBAAiB,CAAC;IAElC,0CAA0C;IAC1C,eAAe,CAAC,EAAE,qBAAqB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,qBAAsB,SAAQ,mBAAmB;IAChE,QAAQ,EAAE,QAAQ,CAAC;IAEnB,qBAAqB;IACrB,MAAM,EAAE,MAAM,CAAC;IAEf,mBAAmB;IACnB,KAAK,CAAC,EAAE,wBAAwB,GAAG,wBAAwB,GAAG,wBAAwB,CAAC;IAEvF,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,wBAAwB;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,4BAA4B;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,kBAAkB;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,2BAA4B,SAAQ,mBAAmB;IACtE,QAAQ,EAAE,cAAc,CAAC;IAEzB,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,yBAAyB;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB,qBAAqB;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAoB,SAAQ,mBAAmB;IAC9D,QAAQ,EAAE,MAAM,CAAC;IAEjB,wBAAwB;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,8BAA8B;IAC9B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,0BAA2B,SAAQ,mBAAmB;IACrE,QAAQ,EAAE,cAAc,CAAC;IAEzB,2CAA2C;IAC3C,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,sDAAsD;IACtD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,yCAAyC;IACzC,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAmB,SAAQ,mBAAmB;IAC7D,QAAQ,EAAE,KAAK,CAAC;IAEhB,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,MAAM,CAAC,EAAE,QAAQ,GAAG,IAAI,GAAG,YAAY,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GACvB,qBAAqB,GACrB,2BAA2B,GAC3B,mBAAmB,GACnB,0BAA0B,GAC1B,kBAAkB,CAAC;AAMvB;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,2BAA2B;IAC3B,SAAS,EAAE,YAAY,GAAG,MAAM,EAAE,CAAC;IAEnC,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAElB,sCAAsC;IACtC,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IAEF,oCAAoC;IACpC,MAAM,CAAC,EAAE,OAAO,CAAC;IAEjB,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAE3B,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,0BAA0B;IAC1B,UAAU,EAAE,KAAK,CAAC,YAAY,GAAG,MAAM,EAAE,CAAC,CAAC;IAE3C,oCAAoC;IACpC,cAAc,EAAE,MAAM,CAAC;IAEvB,oCAAoC;IACpC,YAAY,EAAE,MAAM,CAAC;IAErB,sCAAsC;IACtC,KAAK,CAAC,EAAE;QACN,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IAEF,uBAAuB;IACvB,UAAU,CAAC,EAAE;QACX,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAMD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,0BAA0B;IAC1B,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IAErC,oCAAoC;IACpC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAE9C,wCAAwC;IACxC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE3D,kBAAkB;IAClB,UAAU,IAAI,IAAI,CAAC;IAEnB,2BAA2B;IAC3B,aAAa,IAAI;QACf,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IAEF,uBAAuB;IACvB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3B;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,GACtB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACrC;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC3D;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC;AAE7C;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAMrF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,WAAW,GAAG,KAAK,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,KAAK,EAAE,MAAM,CAAC;IAEd,kBAAkB;IAClB,MAAM,EAAE,gBAAgB,CAAC;CAC1B"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* V3 Embedding Service Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for embedding service aligned with agentic-flow@alpha:
|
|
5
|
+
* - OpenAI provider
|
|
6
|
+
* - Transformers.js provider
|
|
7
|
+
* - Mock provider
|
|
8
|
+
*
|
|
9
|
+
* Performance Targets:
|
|
10
|
+
* - Single embedding: <100ms (API), <50ms (local)
|
|
11
|
+
* - Batch embedding: <500ms for 10 items
|
|
12
|
+
* - Cache hit: <1ms
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sparkleideas/embeddings",
|
|
3
|
-
"version": "3.0.0-alpha.12-patch.
|
|
3
|
+
"version": "3.0.0-alpha.12-patch.28",
|
|
4
4
|
"description": "V3 Embedding Service - OpenAI, Transformers.js, Agentic-Flow (ONNX), Mock providers with hyperbolic embeddings, normalization, and chunking",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -48,8 +48,8 @@
|
|
|
48
48
|
"vitest": "^4.0.16"
|
|
49
49
|
},
|
|
50
50
|
"peerDependencies": {
|
|
51
|
-
"@sparkleideas/shared": "3.0.0-alpha.6-patch.
|
|
52
|
-
"@sparkleideas/agentic-flow": "2.0.2-alpha-patch.
|
|
51
|
+
"@sparkleideas/shared": "3.0.0-alpha.6-patch.28",
|
|
52
|
+
"@sparkleideas/agentic-flow": "2.0.2-alpha-patch.20"
|
|
53
53
|
},
|
|
54
54
|
"peerDependenciesMeta": {
|
|
55
55
|
"agentic-flow": {
|