@yamo/memory-mesh 3.0.0 → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -3
- package/bin/memory_mesh.js +95 -8
- package/lib/llm/client.d.ts +23 -48
- package/lib/llm/client.js +1 -0
- package/lib/llm/client.ts +298 -377
- package/lib/llm/index.js +1 -0
- package/lib/llm/index.ts +1 -2
- package/lib/memory/adapters/client.d.ts +22 -85
- package/lib/memory/adapters/client.js +1 -0
- package/lib/memory/adapters/client.ts +474 -633
- package/lib/memory/adapters/config.d.ts +82 -89
- package/lib/memory/adapters/config.js +1 -0
- package/lib/memory/adapters/config.ts +156 -225
- package/lib/memory/adapters/errors.d.ts +28 -20
- package/lib/memory/adapters/errors.js +1 -0
- package/lib/memory/adapters/errors.ts +83 -120
- package/lib/memory/context-manager.d.ts +15 -18
- package/lib/memory/context-manager.js +1 -0
- package/lib/memory/context-manager.ts +314 -401
- package/lib/memory/embeddings/factory.d.ts +18 -20
- package/lib/memory/embeddings/factory.js +1 -0
- package/lib/memory/embeddings/factory.ts +130 -173
- package/lib/memory/embeddings/index.js +1 -0
- package/lib/memory/embeddings/index.ts +1 -0
- package/lib/memory/embeddings/service.d.ts +36 -66
- package/lib/memory/embeddings/service.js +1 -0
- package/lib/memory/embeddings/service.ts +479 -616
- package/lib/memory/index.d.ts +2 -2
- package/lib/memory/index.js +1 -0
- package/lib/memory/index.ts +3 -13
- package/lib/memory/memory-mesh.d.ts +151 -93
- package/lib/memory/memory-mesh.js +1 -0
- package/lib/memory/memory-mesh.ts +1406 -1692
- package/lib/memory/memory-translator.d.ts +1 -6
- package/lib/memory/memory-translator.js +1 -0
- package/lib/memory/memory-translator.ts +96 -128
- package/lib/memory/schema.d.ts +29 -10
- package/lib/memory/schema.js +1 -0
- package/lib/memory/schema.ts +102 -185
- package/lib/memory/scorer.d.ts +3 -4
- package/lib/memory/scorer.js +1 -0
- package/lib/memory/scorer.ts +69 -86
- package/lib/memory/search/index.js +1 -0
- package/lib/memory/search/index.ts +1 -0
- package/lib/memory/search/keyword-search.d.ts +10 -26
- package/lib/memory/search/keyword-search.js +1 -0
- package/lib/memory/search/keyword-search.ts +123 -161
- package/lib/scrubber/config/defaults.d.ts +39 -46
- package/lib/scrubber/config/defaults.js +1 -0
- package/lib/scrubber/config/defaults.ts +50 -112
- package/lib/scrubber/errors/scrubber-error.d.ts +22 -0
- package/lib/scrubber/errors/scrubber-error.js +39 -0
- package/lib/scrubber/errors/scrubber-error.ts +44 -0
- package/lib/scrubber/index.d.ts +0 -1
- package/lib/scrubber/index.js +1 -0
- package/lib/scrubber/index.ts +1 -2
- package/lib/scrubber/scrubber.d.ts +14 -31
- package/lib/scrubber/scrubber.js +1 -0
- package/lib/scrubber/scrubber.ts +93 -152
- package/lib/scrubber/stages/chunker.d.ts +22 -10
- package/lib/scrubber/stages/chunker.js +86 -0
- package/lib/scrubber/stages/chunker.ts +104 -0
- package/lib/scrubber/stages/metadata-annotator.d.ts +14 -15
- package/lib/scrubber/stages/metadata-annotator.js +64 -0
- package/lib/scrubber/stages/metadata-annotator.ts +75 -0
- package/lib/scrubber/stages/normalizer.d.ts +13 -10
- package/lib/scrubber/stages/normalizer.js +51 -0
- package/lib/scrubber/stages/normalizer.ts +60 -0
- package/lib/scrubber/stages/semantic-filter.d.ts +13 -10
- package/lib/scrubber/stages/semantic-filter.js +51 -0
- package/lib/scrubber/stages/semantic-filter.ts +62 -0
- package/lib/scrubber/stages/structural-cleaner.d.ts +15 -10
- package/lib/scrubber/stages/structural-cleaner.js +73 -0
- package/lib/scrubber/stages/structural-cleaner.ts +83 -0
- package/lib/scrubber/stages/validator.d.ts +14 -15
- package/lib/scrubber/stages/validator.js +56 -0
- package/lib/scrubber/stages/validator.ts +67 -0
- package/lib/scrubber/telemetry.d.ts +20 -27
- package/lib/scrubber/telemetry.js +1 -0
- package/lib/scrubber/telemetry.ts +53 -90
- package/lib/scrubber/utils/hash.d.ts +14 -0
- package/lib/scrubber/utils/hash.js +37 -0
- package/lib/scrubber/utils/hash.ts +40 -0
- package/lib/scrubber/utils/html-parser.d.ts +14 -0
- package/lib/scrubber/utils/html-parser.js +38 -0
- package/lib/scrubber/utils/html-parser.ts +46 -0
- package/lib/scrubber/utils/pattern-matcher.d.ts +12 -0
- package/lib/scrubber/utils/pattern-matcher.js +54 -0
- package/lib/scrubber/utils/pattern-matcher.ts +64 -0
- package/lib/scrubber/utils/token-counter.d.ts +18 -0
- package/lib/scrubber/utils/token-counter.js +30 -0
- package/lib/scrubber/utils/token-counter.ts +32 -0
- package/lib/utils/logger.d.ts +1 -11
- package/lib/utils/logger.js +1 -0
- package/lib/utils/logger.ts +43 -63
- package/lib/utils/skill-metadata.d.ts +6 -14
- package/lib/utils/skill-metadata.js +1 -0
- package/lib/utils/skill-metadata.ts +89 -103
- package/lib/yamo/emitter.d.ts +8 -35
- package/lib/yamo/emitter.js +1 -0
- package/lib/yamo/emitter.ts +77 -155
- package/lib/yamo/index.d.ts +14 -0
- package/lib/yamo/index.js +14 -0
- package/lib/yamo/index.ts +16 -0
- package/lib/yamo/schema.d.ts +8 -10
- package/lib/yamo/schema.js +1 -0
- package/lib/yamo/schema.ts +82 -114
- package/package.json +5 -2
|
@@ -2,55 +2,53 @@
|
|
|
2
2
|
* EmbeddingFactory - Multi-provider embedding with automatic fallback
|
|
3
3
|
* Manages primary and fallback embedding services
|
|
4
4
|
*/
|
|
5
|
-
import EmbeddingService
|
|
6
|
-
export interface FactoryStats {
|
|
7
|
-
configured: boolean;
|
|
8
|
-
primary: ServiceStats | null;
|
|
9
|
-
fallbacks: ServiceStats[];
|
|
10
|
-
}
|
|
11
|
-
export interface InitResult {
|
|
12
|
-
success: boolean;
|
|
13
|
-
primary: string | null;
|
|
14
|
-
fallbacks: string[];
|
|
15
|
-
}
|
|
5
|
+
import EmbeddingService from "./service.js";
|
|
16
6
|
declare class EmbeddingFactory {
|
|
17
|
-
primaryService:
|
|
18
|
-
fallbackServices:
|
|
19
|
-
configured:
|
|
20
|
-
ServiceClass:
|
|
7
|
+
primaryService: any;
|
|
8
|
+
fallbackServices: any;
|
|
9
|
+
configured: any;
|
|
10
|
+
ServiceClass: any;
|
|
21
11
|
constructor(ServiceClass?: typeof EmbeddingService);
|
|
22
12
|
/**
|
|
23
13
|
* Configure embedding services with fallback chain
|
|
24
14
|
* @param {Array} configs - Array of { modelType, modelName, priority, apiKey }
|
|
25
15
|
* @returns {Object} Success status
|
|
26
16
|
*/
|
|
27
|
-
configure(configs:
|
|
17
|
+
configure(configs: any): {
|
|
28
18
|
success: boolean;
|
|
29
19
|
};
|
|
30
20
|
/**
|
|
31
21
|
* Initialize all configured services
|
|
32
22
|
* @returns {Promise<Object>} Initialization status
|
|
33
23
|
*/
|
|
34
|
-
init(): Promise<
|
|
24
|
+
init(): Promise<{
|
|
25
|
+
success: boolean;
|
|
26
|
+
primary: any;
|
|
27
|
+
fallbacks: any;
|
|
28
|
+
}>;
|
|
35
29
|
/**
|
|
36
30
|
* Generate embedding with automatic fallback
|
|
37
31
|
* @param {string} text - Text to embed
|
|
38
32
|
* @param {Object} options - Options
|
|
39
33
|
* @returns {Promise<number[]>} Embedding vector
|
|
40
34
|
*/
|
|
41
|
-
embed(text:
|
|
35
|
+
embed(text: any, options?: {}): Promise<any>;
|
|
42
36
|
/**
|
|
43
37
|
* Generate embeddings for batch of texts
|
|
44
38
|
* @param {string[]} texts - Texts to embed
|
|
45
39
|
* @param {Object} options - Options
|
|
46
40
|
* @returns {Promise<number[][]>} Array of embedding vectors
|
|
47
41
|
*/
|
|
48
|
-
embedBatch(texts:
|
|
42
|
+
embedBatch(texts: any, options?: {}): Promise<any>;
|
|
49
43
|
/**
|
|
50
44
|
* Get factory statistics
|
|
51
45
|
* @returns {Object} Statistics
|
|
52
46
|
*/
|
|
53
|
-
getStats():
|
|
47
|
+
getStats(): {
|
|
48
|
+
configured: any;
|
|
49
|
+
primary: any;
|
|
50
|
+
fallbacks: any;
|
|
51
|
+
};
|
|
54
52
|
/**
|
|
55
53
|
* Clear all caches
|
|
56
54
|
*/
|
|
@@ -1,192 +1,149 @@
|
|
|
1
|
+
// @ts-nocheck
|
|
1
2
|
/**
|
|
2
3
|
* EmbeddingFactory - Multi-provider embedding with automatic fallback
|
|
3
4
|
* Manages primary and fallback embedding services
|
|
4
5
|
*/
|
|
5
|
-
|
|
6
|
-
import EmbeddingService, { ServiceConfig, ServiceStats } from "./service.js";
|
|
6
|
+
import EmbeddingService from "./service.js";
|
|
7
7
|
import { ConfigurationError, EmbeddingError } from "../adapters/errors.js";
|
|
8
8
|
import { createLogger } from "../../utils/logger.js";
|
|
9
|
-
|
|
10
9
|
const logger = createLogger("embedding-factory");
|
|
11
|
-
|
|
12
|
-
export interface FactoryStats {
|
|
13
|
-
configured: boolean;
|
|
14
|
-
primary: ServiceStats | null;
|
|
15
|
-
fallbacks: ServiceStats[];
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
export interface InitResult {
|
|
19
|
-
success: boolean;
|
|
20
|
-
primary: string | null;
|
|
21
|
-
fallbacks: string[];
|
|
22
|
-
}
|
|
23
|
-
|
|
24
10
|
class EmbeddingFactory {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
this.ServiceClass = ServiceClass;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Configure embedding services with fallback chain
|
|
39
|
-
* @param {Array} configs - Array of { modelType, modelName, priority, apiKey }
|
|
40
|
-
* @returns {Object} Success status
|
|
41
|
-
*/
|
|
42
|
-
configure(configs: ServiceConfig[]): { success: boolean } {
|
|
43
|
-
// Sort by priority (lower = higher priority)
|
|
44
|
-
configs.sort((a, b) => (a.priority || 0) - (b.priority || 0));
|
|
45
|
-
|
|
46
|
-
if (configs.length > 0) {
|
|
47
|
-
this.primaryService = new this.ServiceClass(configs[0]);
|
|
11
|
+
primaryService;
|
|
12
|
+
fallbackServices;
|
|
13
|
+
configured;
|
|
14
|
+
ServiceClass;
|
|
15
|
+
constructor(ServiceClass = EmbeddingService) {
|
|
16
|
+
this.primaryService = null;
|
|
17
|
+
this.fallbackServices = [];
|
|
18
|
+
this.configured = false;
|
|
19
|
+
this.ServiceClass = ServiceClass;
|
|
48
20
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
"EmbeddingFactory not configured. Call configure() first.",
|
|
68
|
-
);
|
|
21
|
+
/**
|
|
22
|
+
* Configure embedding services with fallback chain
|
|
23
|
+
* @param {Array} configs - Array of { modelType, modelName, priority, apiKey }
|
|
24
|
+
* @returns {Object} Success status
|
|
25
|
+
*/
|
|
26
|
+
configure(configs) {
|
|
27
|
+
// Sort by priority (lower = higher priority)
|
|
28
|
+
configs.sort((a, b) => (a.priority || 0) - (b.priority || 0));
|
|
29
|
+
if (configs.length > 0) {
|
|
30
|
+
this.primaryService = new this.ServiceClass(configs[0]);
|
|
31
|
+
}
|
|
32
|
+
if (configs.length > 1) {
|
|
33
|
+
this.fallbackServices = configs
|
|
34
|
+
.slice(1)
|
|
35
|
+
.map((c) => new this.ServiceClass(c));
|
|
36
|
+
}
|
|
37
|
+
this.configured = true;
|
|
38
|
+
return { success: true };
|
|
69
39
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
40
|
+
/**
|
|
41
|
+
* Initialize all configured services
|
|
42
|
+
* @returns {Promise<Object>} Initialization status
|
|
43
|
+
*/
|
|
44
|
+
async init() {
|
|
45
|
+
if (!this.configured) {
|
|
46
|
+
throw new ConfigurationError("EmbeddingFactory not configured. Call configure() first.");
|
|
47
|
+
}
|
|
48
|
+
// Initialize primary service
|
|
49
|
+
if (this.primaryService && !this.primaryService.initialized) {
|
|
50
|
+
await this.primaryService.init();
|
|
51
|
+
}
|
|
52
|
+
// Initialize fallback services lazily (on first use)
|
|
53
|
+
return {
|
|
54
|
+
success: true,
|
|
55
|
+
primary: this.primaryService ? this.primaryService.modelName : null,
|
|
56
|
+
fallbacks: this.fallbackServices.map((s) => s.modelName),
|
|
57
|
+
};
|
|
74
58
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
59
|
+
/**
|
|
60
|
+
* Generate embedding with automatic fallback
|
|
61
|
+
* @param {string} text - Text to embed
|
|
62
|
+
* @param {Object} options - Options
|
|
63
|
+
* @returns {Promise<number[]>} Embedding vector
|
|
64
|
+
*/
|
|
65
|
+
async embed(text, options = {}) {
|
|
66
|
+
if (!this.configured || !this.primaryService) {
|
|
67
|
+
throw new ConfigurationError("EmbeddingFactory not configured");
|
|
68
|
+
}
|
|
69
|
+
// Try primary service
|
|
70
|
+
try {
|
|
71
|
+
if (!this.primaryService.initialized) {
|
|
72
|
+
await this.primaryService.init();
|
|
73
|
+
}
|
|
74
|
+
return await this.primaryService.embed(text, options);
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
78
|
+
logger.warn({ err: error, primaryService: this.primaryService?.modelName }, "Primary service failed");
|
|
79
|
+
// Try fallback services in order
|
|
80
|
+
for (const fallback of this.fallbackServices) {
|
|
81
|
+
try {
|
|
82
|
+
if (!fallback.initialized) {
|
|
83
|
+
await fallback.init();
|
|
84
|
+
}
|
|
85
|
+
logger.info({ fallbackModel: fallback.modelName }, "Using fallback service");
|
|
86
|
+
return await fallback.embed(text, options);
|
|
87
|
+
}
|
|
88
|
+
catch (fallbackError) {
|
|
89
|
+
logger.warn({ err: fallbackError, fallbackModel: fallback.modelName }, "Fallback service failed");
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
throw new EmbeddingError("All embedding services failed", {
|
|
93
|
+
primaryError: errorMessage,
|
|
94
|
+
fallbackCount: this.fallbackServices.length,
|
|
95
|
+
});
|
|
96
|
+
}
|
|
93
97
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
{ err: error, primaryService: this.primaryService?.modelName },
|
|
106
|
-
"Primary service failed",
|
|
107
|
-
);
|
|
108
|
-
|
|
109
|
-
// Try fallback services in order
|
|
110
|
-
for (const fallback of this.fallbackServices) {
|
|
98
|
+
/**
|
|
99
|
+
* Generate embeddings for batch of texts
|
|
100
|
+
* @param {string[]} texts - Texts to embed
|
|
101
|
+
* @param {Object} options - Options
|
|
102
|
+
* @returns {Promise<number[][]>} Array of embedding vectors
|
|
103
|
+
*/
|
|
104
|
+
async embedBatch(texts, options = {}) {
|
|
105
|
+
if (!this.configured || !this.primaryService) {
|
|
106
|
+
throw new ConfigurationError("EmbeddingFactory not configured");
|
|
107
|
+
}
|
|
108
|
+
// Try primary service
|
|
111
109
|
try {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
110
|
+
if (!this.primaryService.initialized) {
|
|
111
|
+
await this.primaryService.init();
|
|
112
|
+
}
|
|
113
|
+
return await this.primaryService.embedBatch(texts, options);
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
logger.warn({
|
|
117
|
+
err: error,
|
|
118
|
+
primaryService: this.primaryService?.modelName,
|
|
119
|
+
batchSize: texts.length,
|
|
120
|
+
}, "Primary batch embedding failed, falling back to individual embeddings");
|
|
121
|
+
// Fallback to individual embedding with fallback services
|
|
122
|
+
const results = [];
|
|
123
|
+
for (const text of texts) {
|
|
124
|
+
results.push(await this.embed(text, options));
|
|
125
|
+
}
|
|
126
|
+
return results;
|
|
125
127
|
}
|
|
126
|
-
}
|
|
127
|
-
|
|
128
|
-
throw new EmbeddingError("All embedding services failed", {
|
|
129
|
-
primaryError: errorMessage,
|
|
130
|
-
fallbackCount: this.fallbackServices.length,
|
|
131
|
-
});
|
|
132
128
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
129
|
+
/**
|
|
130
|
+
* Get factory statistics
|
|
131
|
+
* @returns {Object} Statistics
|
|
132
|
+
*/
|
|
133
|
+
getStats() {
|
|
134
|
+
const stats = {
|
|
135
|
+
configured: this.configured,
|
|
136
|
+
primary: this.primaryService?.getStats() || null,
|
|
137
|
+
fallbacks: this.fallbackServices.map((s) => s.getStats()),
|
|
138
|
+
};
|
|
139
|
+
return stats;
|
|
144
140
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
return await this.primaryService.embedBatch(texts, options);
|
|
152
|
-
} catch (error) {
|
|
153
|
-
logger.warn(
|
|
154
|
-
{
|
|
155
|
-
err: error,
|
|
156
|
-
primaryService: this.primaryService?.modelName,
|
|
157
|
-
batchSize: texts.length,
|
|
158
|
-
},
|
|
159
|
-
"Primary batch embedding failed, falling back to individual embeddings",
|
|
160
|
-
);
|
|
161
|
-
// Fallback to individual embedding with fallback services
|
|
162
|
-
const results: number[][] = [];
|
|
163
|
-
for (const text of texts) {
|
|
164
|
-
results.push(await this.embed(text, options));
|
|
165
|
-
}
|
|
166
|
-
return results;
|
|
141
|
+
/**
|
|
142
|
+
* Clear all caches
|
|
143
|
+
*/
|
|
144
|
+
clearCache() {
|
|
145
|
+
this.primaryService?.clearCache();
|
|
146
|
+
this.fallbackServices.forEach((s) => s.clearCache());
|
|
167
147
|
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Get factory statistics
|
|
172
|
-
* @returns {Object} Statistics
|
|
173
|
-
*/
|
|
174
|
-
getStats(): FactoryStats {
|
|
175
|
-
const stats = {
|
|
176
|
-
configured: this.configured,
|
|
177
|
-
primary: this.primaryService?.getStats() || null,
|
|
178
|
-
fallbacks: this.fallbackServices.map((s) => s.getStats()),
|
|
179
|
-
};
|
|
180
|
-
return stats;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Clear all caches
|
|
185
|
-
*/
|
|
186
|
-
clearCache(): void {
|
|
187
|
-
this.primaryService?.clearCache();
|
|
188
|
-
this.fallbackServices.forEach((s) => s.clearCache());
|
|
189
|
-
}
|
|
190
148
|
}
|
|
191
|
-
|
|
192
149
|
export default EmbeddingFactory;
|
|
@@ -1,69 +1,25 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* EmbeddingService - Multi-provider embedding generation service
|
|
3
|
-
*
|
|
4
|
-
* Supports:
|
|
5
|
-
* - Local models: Xenova/Transformers.js (ONNX runtime)
|
|
6
|
-
* - Ollama: Local Ollama embeddings API
|
|
7
|
-
* - API models: OpenAI, Cohere
|
|
8
|
-
*
|
|
9
|
-
* Implements TDD for Phase 3, Task 3.1 - Embedding Service Architecture
|
|
10
|
-
*/
|
|
11
|
-
/**
|
|
12
|
-
* Service configuration interface
|
|
13
|
-
*/
|
|
14
|
-
export interface ServiceConfig {
|
|
15
|
-
modelType?: "local" | "ollama" | "openai" | "cohere";
|
|
16
|
-
modelName?: string;
|
|
17
|
-
baseUrl?: string;
|
|
18
|
-
dimension?: number;
|
|
19
|
-
batchSize?: number;
|
|
20
|
-
normalize?: boolean;
|
|
21
|
-
cacheMaxSize?: number;
|
|
22
|
-
apiKey?: string;
|
|
23
|
-
priority?: number;
|
|
24
|
-
}
|
|
25
|
-
export interface ServiceStats {
|
|
26
|
-
modelType: string;
|
|
27
|
-
modelName: string;
|
|
28
|
-
dimension: number;
|
|
29
|
-
initialized: boolean;
|
|
30
|
-
totalEmbeddings: number;
|
|
31
|
-
cacheHits: number;
|
|
32
|
-
cacheMisses: number;
|
|
33
|
-
cacheSize: number;
|
|
34
|
-
cacheMaxSize: number;
|
|
35
|
-
cacheHitRate: number;
|
|
36
|
-
batchCount: number;
|
|
37
|
-
batchSize: number;
|
|
38
|
-
normalize: boolean;
|
|
39
|
-
}
|
|
40
1
|
/**
|
|
41
2
|
* EmbeddingService provides a unified interface for generating text embeddings
|
|
42
3
|
* using multiple backend providers (local ONNX models or cloud APIs).
|
|
43
4
|
*/
|
|
44
5
|
export declare class EmbeddingService {
|
|
45
|
-
modelType:
|
|
46
|
-
modelName:
|
|
47
|
-
baseUrl:
|
|
48
|
-
dimension:
|
|
49
|
-
batchSize:
|
|
50
|
-
normalize:
|
|
51
|
-
apiKey
|
|
6
|
+
modelType: any;
|
|
7
|
+
modelName: any;
|
|
8
|
+
baseUrl: any;
|
|
9
|
+
dimension: any;
|
|
10
|
+
batchSize: any;
|
|
11
|
+
normalize: any;
|
|
12
|
+
apiKey: any;
|
|
52
13
|
model: any;
|
|
53
|
-
cache:
|
|
54
|
-
cacheMaxSize:
|
|
55
|
-
initialized:
|
|
56
|
-
stats:
|
|
57
|
-
totalEmbeddings: number;
|
|
58
|
-
cacheHits: number;
|
|
59
|
-
cacheMisses: number;
|
|
60
|
-
batchCount: number;
|
|
61
|
-
};
|
|
14
|
+
cache: any;
|
|
15
|
+
cacheMaxSize: any;
|
|
16
|
+
initialized: any;
|
|
17
|
+
stats: any;
|
|
62
18
|
/**
|
|
63
19
|
* Create a new EmbeddingService instance
|
|
64
20
|
* @param {Object} [config={}] - Configuration options
|
|
65
21
|
*/
|
|
66
|
-
constructor(config?:
|
|
22
|
+
constructor(config?: {});
|
|
67
23
|
/**
|
|
68
24
|
* Initialize the embedding model
|
|
69
25
|
* Loads the model based on modelType (local, ollama, openai, cohere)
|
|
@@ -75,14 +31,14 @@ export declare class EmbeddingService {
|
|
|
75
31
|
* @param {Object} options - Options for embedding generation
|
|
76
32
|
* @returns {Promise<number[]>} Embedding vector
|
|
77
33
|
*/
|
|
78
|
-
embed(text:
|
|
34
|
+
embed(text: any, _options?: {}): Promise<any>;
|
|
79
35
|
/**
|
|
80
36
|
* Generate embeddings for a batch of texts
|
|
81
37
|
* @param {string[]} texts - Array of texts to embed
|
|
82
38
|
* @param {Object} options - Options for embedding generation
|
|
83
39
|
* @returns {Promise<number[][]>} Array of embedding vectors
|
|
84
40
|
*/
|
|
85
|
-
embedBatch(texts:
|
|
41
|
+
embedBatch(texts: any, _options?: {}): Promise<any[]>;
|
|
86
42
|
/**
|
|
87
43
|
* Initialize local ONNX model using Xenova/Transformers.js
|
|
88
44
|
* @private
|
|
@@ -110,48 +66,62 @@ export declare class EmbeddingService {
|
|
|
110
66
|
* @returns {Promise<number[]>} Embedding vector
|
|
111
67
|
* @private
|
|
112
68
|
*/
|
|
113
|
-
_embedLocal(text:
|
|
69
|
+
_embedLocal(text: any): Promise<unknown[]>;
|
|
114
70
|
/**
|
|
115
71
|
* Generate embedding using Ollama API
|
|
116
72
|
* @param {string} text - Text to embed
|
|
117
73
|
* @returns {Promise<number[]>} Embedding vector
|
|
118
74
|
* @private
|
|
119
75
|
*/
|
|
120
|
-
_embedOllama(text:
|
|
76
|
+
_embedOllama(text: any): Promise<any>;
|
|
121
77
|
/**
|
|
122
78
|
* Generate embedding using OpenAI API
|
|
123
79
|
* @param {string} text - Text to embed
|
|
124
80
|
* @returns {Promise<number[]>} Embedding vector
|
|
125
81
|
* @private
|
|
126
82
|
*/
|
|
127
|
-
_embedOpenAI(text:
|
|
83
|
+
_embedOpenAI(text: any): Promise<any>;
|
|
128
84
|
/**
|
|
129
85
|
* Generate embedding using Cohere API
|
|
130
86
|
* @param {string} text - Text to embed
|
|
131
87
|
* @returns {Promise<number[]>} Embedding vector
|
|
132
88
|
* @private
|
|
133
89
|
*/
|
|
134
|
-
_embedCohere(text:
|
|
90
|
+
_embedCohere(text: any): Promise<any>;
|
|
135
91
|
/**
|
|
136
92
|
* Normalize vector to unit length
|
|
137
93
|
* @param {number[]} vector - Vector to normalize
|
|
138
94
|
* @returns {number[]} Normalized vector
|
|
139
95
|
* @private
|
|
140
96
|
*/
|
|
141
|
-
_normalize(vector:
|
|
97
|
+
_normalize(vector: any): any;
|
|
142
98
|
/**
|
|
143
99
|
* Generate cache key from text
|
|
144
100
|
* @param {string} text - Text to generate key from
|
|
145
101
|
* @returns {string} Cache key
|
|
146
102
|
* @private
|
|
147
103
|
*/
|
|
148
|
-
_getCacheKey(text:
|
|
149
|
-
_setCache(key:
|
|
104
|
+
_getCacheKey(text: any): string;
|
|
105
|
+
_setCache(key: any, value: any): void;
|
|
150
106
|
/**
|
|
151
107
|
* Get service statistics
|
|
152
108
|
* @returns {Object} Statistics object
|
|
153
109
|
*/
|
|
154
|
-
getStats():
|
|
110
|
+
getStats(): {
|
|
111
|
+
modelType: any;
|
|
112
|
+
modelName: any;
|
|
113
|
+
dimension: any;
|
|
114
|
+
initialized: any;
|
|
115
|
+
totalEmbeddings: any;
|
|
116
|
+
cacheHits: any;
|
|
117
|
+
cacheMisses: any;
|
|
118
|
+
cacheSize: any;
|
|
119
|
+
cacheMaxSize: any;
|
|
120
|
+
cacheHitRate: number;
|
|
121
|
+
batchCount: any;
|
|
122
|
+
batchSize: any;
|
|
123
|
+
normalize: any;
|
|
124
|
+
};
|
|
155
125
|
/**
|
|
156
126
|
* Clear the embedding cache
|
|
157
127
|
*/
|