@henrychong-ai/mcp-neo4j-knowledge-graph 2.4.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -0
- package/dist/KnowledgeGraphManager.d.ts +18 -0
- package/dist/KnowledgeGraphManager.js +52 -1
- package/dist/KnowledgeGraphManager.js.map +1 -1
- package/dist/cli/generate-embeddings.js +12 -0
- package/dist/cli/generate-embeddings.js.map +1 -1
- package/dist/embeddings/EmbeddingJobManager.js +4 -26
- package/dist/embeddings/EmbeddingJobManager.js.map +1 -1
- package/dist/embeddings/EmbeddingServiceFactory.d.ts +35 -0
- package/dist/embeddings/EmbeddingServiceFactory.js +122 -18
- package/dist/embeddings/EmbeddingServiceFactory.js.map +1 -1
- package/dist/embeddings/OpenAIEmbeddingService.d.ts +6 -0
- package/dist/embeddings/OpenAIEmbeddingService.js +1 -1
- package/dist/embeddings/OpenAIEmbeddingService.js.map +1 -1
- package/dist/embeddings/entityText.d.ts +12 -0
- package/dist/embeddings/entityText.js +39 -0
- package/dist/embeddings/entityText.js.map +1 -0
- package/dist/index.js +65 -53
- package/dist/index.js.map +1 -1
- package/dist/retrieval/RerankerService.d.ts +57 -0
- package/dist/retrieval/RerankerService.js +117 -0
- package/dist/retrieval/RerankerService.js.map +1 -0
- package/dist/server/setup.d.ts +10 -0
- package/dist/server/setup.js +16 -1
- package/dist/server/setup.js.map +1 -1
- package/dist/storage/neo4j/Neo4jStorageProvider.d.ts +10 -0
- package/dist/storage/neo4j/Neo4jStorageProvider.js +56 -9
- package/dist/storage/neo4j/Neo4jStorageProvider.js.map +1 -1
- package/example.env +61 -0
- package/package.json +16 -14
|
@@ -66,44 +66,147 @@ export class EmbeddingServiceFactory {
|
|
|
66
66
|
* @returns An embedding service implementation
|
|
67
67
|
*/
|
|
68
68
|
static createFromEnvironment() {
|
|
69
|
-
//
|
|
70
|
-
|
|
69
|
+
// Surface dimension misconfig at startup, not at first write (the storage
|
|
70
|
+
// layer's dimension guard remains the enforcement).
|
|
71
|
+
const dimensionWarning = EmbeddingServiceFactory.checkDimensionConsistency();
|
|
72
|
+
if (dimensionWarning) {
|
|
73
|
+
logger.warn(`EmbeddingServiceFactory: ${dimensionWarning}`);
|
|
74
|
+
}
|
|
75
|
+
// Check if we should use mock embeddings (for testing / explicit opt-in).
|
|
76
|
+
// NEVER honoured in production — a stale MOCK_EMBEDDINGS=true must not beat a
|
|
77
|
+
// real key (or silently produce random vectors) on a production deployment.
|
|
78
|
+
const isProduction = process.env.NODE_ENV === 'production';
|
|
79
|
+
const useMockEmbeddings = process.env.MOCK_EMBEDDINGS === 'true' && !isProduction;
|
|
80
|
+
if (process.env.MOCK_EMBEDDINGS === 'true' && isProduction) {
|
|
81
|
+
logger.error('EmbeddingServiceFactory: MOCK_EMBEDDINGS=true ignored under NODE_ENV=production — ' +
|
|
82
|
+
'random vectors must never drive a production store.');
|
|
83
|
+
}
|
|
84
|
+
// New EMBEDDING_* env vars (provider-neutral) fall back to the legacy OPENAI_* names,
|
|
85
|
+
// so existing deployments are unaffected. Point EMBEDDING_API_ENDPOINT / EMBEDDING_API_BASE_URL
|
|
86
|
+
// at any OpenAI-compatible endpoint (e.g. Cloudflare Workers AI) to switch providers.
|
|
87
|
+
const apiKey = process.env.EMBEDDING_API_KEY || process.env.OPENAI_API_KEY;
|
|
88
|
+
const embeddingModel = process.env.EMBEDDING_MODEL || process.env.OPENAI_EMBEDDING_MODEL || 'text-embedding-3-small';
|
|
89
|
+
const apiEndpoint = process.env.EMBEDDING_API_ENDPOINT ||
|
|
90
|
+
(process.env.EMBEDDING_API_BASE_URL
|
|
91
|
+
? `${process.env.EMBEDDING_API_BASE_URL.replace(/\/$/, '')}/embeddings`
|
|
92
|
+
: undefined);
|
|
93
|
+
const dimensions = process.env.EMBEDDING_DIMENSIONS
|
|
94
|
+
? Number.parseInt(process.env.EMBEDDING_DIMENSIONS, 10)
|
|
95
|
+
: undefined;
|
|
71
96
|
logger.debug('EmbeddingServiceFactory: Creating service from environment variables', {
|
|
72
97
|
mockEmbeddings: useMockEmbeddings,
|
|
73
|
-
|
|
74
|
-
embeddingModel
|
|
98
|
+
apiKeyPresent: !!apiKey,
|
|
99
|
+
embeddingModel,
|
|
100
|
+
apiEndpoint: apiEndpoint || 'openai-default',
|
|
101
|
+
dimensions: dimensions ?? 'default',
|
|
75
102
|
});
|
|
76
103
|
if (useMockEmbeddings) {
|
|
77
|
-
logger.info('EmbeddingServiceFactory: Using mock embeddings
|
|
78
|
-
return new DefaultEmbeddingService();
|
|
104
|
+
logger.info('EmbeddingServiceFactory: Using mock embeddings (MOCK_EMBEDDINGS=true)');
|
|
105
|
+
return new DefaultEmbeddingService(dimensions);
|
|
79
106
|
}
|
|
80
|
-
|
|
81
|
-
const embeddingModel = process.env.OPENAI_EMBEDDING_MODEL || 'text-embedding-3-small';
|
|
82
|
-
if (openaiApiKey) {
|
|
107
|
+
if (apiKey) {
|
|
83
108
|
try {
|
|
84
|
-
logger.debug('EmbeddingServiceFactory: Creating OpenAI embedding service', {
|
|
109
|
+
logger.debug('EmbeddingServiceFactory: Creating OpenAI-compatible embedding service', {
|
|
85
110
|
model: embeddingModel,
|
|
111
|
+
apiEndpoint: apiEndpoint || 'openai-default',
|
|
86
112
|
});
|
|
87
113
|
const service = new OpenAIEmbeddingService({
|
|
88
|
-
apiKey
|
|
114
|
+
apiKey,
|
|
89
115
|
model: embeddingModel,
|
|
116
|
+
apiEndpoint,
|
|
117
|
+
dimensions,
|
|
90
118
|
});
|
|
91
|
-
logger.info('EmbeddingServiceFactory: OpenAI embedding service created
|
|
119
|
+
logger.info('EmbeddingServiceFactory: OpenAI-compatible embedding service created', {
|
|
92
120
|
model: service.getModelInfo().name,
|
|
93
121
|
dimensions: service.getModelInfo().dimensions,
|
|
94
122
|
});
|
|
95
123
|
return service;
|
|
96
124
|
}
|
|
97
125
|
catch (error) {
|
|
98
|
-
logger.error('EmbeddingServiceFactory: Failed to create OpenAI service', error);
|
|
126
|
+
logger.error('EmbeddingServiceFactory: Failed to create OpenAI-compatible service', error);
|
|
99
127
|
logger.info('EmbeddingServiceFactory: Falling back to default embedding service');
|
|
100
|
-
// Fallback to default if
|
|
101
|
-
return new DefaultEmbeddingService();
|
|
128
|
+
// Fallback to default if service creation fails
|
|
129
|
+
return new DefaultEmbeddingService(dimensions);
|
|
102
130
|
}
|
|
103
131
|
}
|
|
104
|
-
// No
|
|
105
|
-
|
|
106
|
-
|
|
132
|
+
// No API key: default (random) service. Callers should consult hasEmbeddingProvider()
|
|
133
|
+
// to decide whether to enable semantic search — index.ts runs keyword-only when no
|
|
134
|
+
// provider is configured rather than generating meaningless random-vector embeddings.
|
|
135
|
+
logger.info('EmbeddingServiceFactory: No embedding API key found, using default embedding service');
|
|
136
|
+
return new DefaultEmbeddingService(dimensions);
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Whether a real embedding provider is configured: an API key (EMBEDDING_API_KEY or
|
|
140
|
+
* OPENAI_API_KEY) or MOCK_EMBEDDINGS=true. When false, the server should run in
|
|
141
|
+
* keyword-only mode rather than generate meaningless random-vector embeddings.
|
|
142
|
+
*
|
|
143
|
+
* Production guard (v2.6.0): MOCK_EMBEDDINGS counts as a provider ONLY outside
|
|
144
|
+
* NODE_ENV=production — random vectors must never drive a production store.
|
|
145
|
+
*
|
|
146
|
+
* @param env - Environment to evaluate (defaults to process.env; injectable for tests)
|
|
147
|
+
* @returns true if embeddings should be enabled
|
|
148
|
+
*/
|
|
149
|
+
static hasEmbeddingProvider(env = process.env) {
|
|
150
|
+
if (env.EMBEDDING_API_KEY || env.OPENAI_API_KEY) {
|
|
151
|
+
return true;
|
|
152
|
+
}
|
|
153
|
+
return env.MOCK_EMBEDDINGS === 'true' && env.NODE_ENV !== 'production';
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Whether the resolved embedding service is safe to WRITE embeddings with.
|
|
157
|
+
* Returns false when NODE_ENV=production and the service is the random
|
|
158
|
+
* DefaultEmbeddingService — covering both MOCK_EMBEDDINGS and the silent
|
|
159
|
+
* fallback path (a configured key whose service construction fell back).
|
|
160
|
+
* Callers should run keyword-only (no EmbeddingJobManager) when false.
|
|
161
|
+
*
|
|
162
|
+
* @param service - The embedding service resolved for this process
|
|
163
|
+
* @param env - Environment to evaluate (defaults to process.env; injectable for tests)
|
|
164
|
+
* @returns true if the service may write embeddings
|
|
165
|
+
*/
|
|
166
|
+
static shouldWriteEmbeddings(service, env = process.env) {
|
|
167
|
+
if (env.NODE_ENV !== 'production') {
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
// instanceof + model-name sentinel: instanceof alone is brittle across
|
|
171
|
+
// duplicate module copies (npx cache / symlinked dev installs); every mock
|
|
172
|
+
// service's model name ends in '-mock'.
|
|
173
|
+
const isRandomService = service instanceof DefaultEmbeddingService ||
|
|
174
|
+
(service.getModelInfo()?.name ?? '').endsWith('-mock');
|
|
175
|
+
return !isRandomService;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Config-consistency check: when both EMBEDDING_DIMENSIONS and
|
|
179
|
+
* NEO4J_VECTOR_DIMENSIONS are set, they must match — a mismatch means every
|
|
180
|
+
* embedding write will fail the storage layer's dimension guard.
|
|
181
|
+
*
|
|
182
|
+
* @param env - Environment to evaluate (defaults to process.env; injectable for tests)
|
|
183
|
+
* @returns a human-readable warning when inconsistent, else null
|
|
184
|
+
*/
|
|
185
|
+
static checkDimensionConsistency(env = process.env) {
|
|
186
|
+
const rawEmbedding = env.EMBEDDING_DIMENSIONS;
|
|
187
|
+
const rawIndex = env.NEO4J_VECTOR_DIMENSIONS;
|
|
188
|
+
const parsedEmbedding = rawEmbedding ? Number.parseInt(rawEmbedding, 10) : undefined;
|
|
189
|
+
const parsedIndex = rawIndex ? Number.parseInt(rawIndex, 10) : undefined;
|
|
190
|
+
// Invalid values are worse than mismatched ones: parseInt('abc') = NaN is
|
|
191
|
+
// falsy, which would silently DISABLE the write-path dimension guard.
|
|
192
|
+
if (rawEmbedding && (!Number.isFinite(parsedEmbedding) || parsedEmbedding <= 0)) {
|
|
193
|
+
return (`EMBEDDING_DIMENSIONS ('${rawEmbedding}') is not a positive integer — ` +
|
|
194
|
+
`embedding configuration is invalid; fix it to a positive integer matching the ` +
|
|
195
|
+
`embedding model's native output dimension.`);
|
|
196
|
+
}
|
|
197
|
+
if (rawIndex && (!Number.isFinite(parsedIndex) || parsedIndex <= 0)) {
|
|
198
|
+
return (`NEO4J_VECTOR_DIMENSIONS ('${rawIndex}') is not a positive integer — ` +
|
|
199
|
+
`the write-path dimension guard is DISABLED until this is fixed to a positive integer.`);
|
|
200
|
+
}
|
|
201
|
+
// Numeric comparison (not raw string) so '1024' vs '01024' compare equal.
|
|
202
|
+
if (parsedEmbedding !== undefined &&
|
|
203
|
+
parsedIndex !== undefined &&
|
|
204
|
+
parsedEmbedding !== parsedIndex) {
|
|
205
|
+
return (`EMBEDDING_DIMENSIONS (${parsedEmbedding}) != NEO4J_VECTOR_DIMENSIONS (${parsedIndex}) — ` +
|
|
206
|
+
`embedding writes WILL fail the dimension guard. Align both with the embedding ` +
|
|
207
|
+
`model's native output dimension.`);
|
|
208
|
+
}
|
|
209
|
+
return null;
|
|
107
210
|
}
|
|
108
211
|
/**
|
|
109
212
|
* Create an OpenAI embedding service
|
|
@@ -142,6 +245,7 @@ EmbeddingServiceFactory.registerProvider('openai', (config = {}) => {
|
|
|
142
245
|
apiKey: config.apiKey,
|
|
143
246
|
model: config.model,
|
|
144
247
|
dimensions: config.dimensions,
|
|
248
|
+
apiEndpoint: config.apiEndpoint,
|
|
145
249
|
});
|
|
146
250
|
});
|
|
147
251
|
//# sourceMappingURL=EmbeddingServiceFactory.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"EmbeddingServiceFactory.js","sourceRoot":"","sources":["../../src/embeddings/EmbeddingServiceFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"EmbeddingServiceFactory.js","sourceRoot":"","sources":["../../src/embeddings/EmbeddingServiceFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAoBrE;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAClC;;OAEG;IACK,MAAM,CAAC,SAAS,GAA6C,EAAE,CAAC;IAExE;;;;;OAKG;IACH,MAAM,CAAC,gBAAgB,CAAC,IAAY,EAAE,QAAkC;QACtE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,GAAG,QAAQ,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa;QAClB,uBAAuB,CAAC,SAAS,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,qBAAqB;QAC1B,OAAO,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,aAAa,CAAC,SAAiC,EAAE;QACtD,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,MAAM,CAAC,KAAK,CAAC,4DAA4D,YAAY,GAAG,CAAC,CAAC;QAE1F,MAAM,UAAU,GAAG,uBAAuB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAEnE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,CAAC,KAAK,CACV,wEAAwE,YAAY,GAAG,EACvF;oBACE,SAAS,EAAE,OAAO,CAAC,YAAY,EAAE;iBAClC,CACF,CAAC;gBACF,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,oEAAoE,YAAY,GAAG,EACnF,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,sCAAsC,YAAY,qBAAqB,CAAC,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,aAAa,YAAY,qBAAqB,CAAC,CAAC;IAClE,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,qBAAqB;QAC1B,0EAA0E;QAC1E,oDAAoD;QACpD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,yBAAyB,EAAE,CAAC;QAC7E,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,4BAA4B,gBAAgB,EAAE,CAAC,CAAC;QAC9D,CAAC;QAED,0EAA0E;QAC1E,8EAA8E;QAC9E,4EAA4E;QAC5E,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;QAC3D,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,IAAI,CAAC,YAAY,CAAC;QAClF,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,IAAI,YAAY,EAAE,CAAC;YAC3D,MAAM,CAAC,KAAK,CACV,oFAAoF;gBAClF,qDAAqD,CACxD,CAAC;QACJ,CAAC;QAED,sFAAsF;QACtF,gGAAgG;QAChG,sFAAsF;QACtF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC3E,MAAM,cAAc,GAClB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,wBAAwB,CAAC;QAChG,MAAM,WAAW,GACf,OAAO,CAAC,GAAG,CAAC,sBAAsB;YAClC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB;gBACjC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa;gBACvE,CAAC,CAAC,SAAS,CAAC,CAAC;QACjB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB;YACjD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;YACvD,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,CAAC,KAAK,CAAC,sEAAsE,EAAE;YACnF,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,CAAC,CAAC,MAAM;YACvB,cAAc;YACd,WAAW,EAAE,WAAW,IAAI,gBAAgB;YAC5C,UAAU,EAAE,UAAU,IAAI,SAAS;SACpC,CAAC,CAAC;QAEH,IAAI,iBAAiB,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;YACrF,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,uEAAuE,EAAE;oBACpF,KAAK,EAAE,cAAc;oBACrB,WAAW,EAAE,WAAW,IAAI,gBAAgB;iBAC7C,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC;oBACzC,MAAM;oBACN,KAAK,EAAE,cAAc;oBACrB,WAAW;oBACX,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,sEAAsE,EAAE;oBAClF,KAAK,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;oBAClC,UAAU,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU;iBAC9C,CAAC,CAAC;gBACH,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,qEAAqE,EAAE,KAAK,CAAC,CAAC;gBAC3F,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;gBAClF,gDAAgD;gBAChD,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,sFAAsF;QACtF,mFAAmF;QACnF,sFAAsF;QACtF,MAAM,CAAC,IAAI,CACT,sFAAsF,CACvF,CAAC;QACF,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,oBAAoB,CAAC,MAAyB,OAAO,CAAC,GAAG;QAC9D,IAAI,GAAG,CAAC,iBAAiB,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,GAAG,CAAC,eAAe,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC;IACzE,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,qBAAqB,CAC1B,OAAyB,EACzB,MAAyB,OAAO,CAAC,GAAG;QAEpC,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,uEAAuE;QACvE,2EAA2E;QAC3E,wCAAwC;QACxC,MAAM,eAAe,GACnB,OAAO,YAAY,uBAAuB;YAC1C,CAAC,OAAO,CAAC,YAAY,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACzD,OAAO,CAAC,eAAe,CAAC;IAC1B,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,yBAAyB,CAAC,MAAyB,OAAO,CAAC,GAAG;QACnE,MAAM,YAAY,GAAG,GAAG,CAAC,oBAAoB,CAAC;QAC9C,MAAM,QAAQ,GAAG,GAAG,CAAC,uBAAuB,CAAC;QAC7C,MAAM,eAAe,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACrF,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAEzE,0EAA0E;QAC1E,sEAAsE;QACtE,IAAI,YAAY,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAAC,IAAK,eAA0B,IAAI,CAAC,CAAC,EAAE,CAAC;YAC5F,OAAO,CACL,0BAA0B,YAAY,iCAAiC;gBACvE,gFAAgF;gBAChF,4CAA4C,CAC7C,CAAC;QACJ,CAAC;QACD,IAAI,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAK,WAAsB,IAAI,CAAC,CAAC,EAAE,CAAC;YAChF,OAAO,CACL,6BAA6B,QAAQ,iCAAiC;gBACtE,uFAAuF,CACxF,CAAC;QACJ,CAAC;QACD,0EAA0E;QAC1E,IACE,eAAe,KAAK,SAAS;YAC7B,WAAW,KAAK,SAAS;YACzB,eAAe,KAAK,WAAW,EAC/B,CAAC;YACD,OAAO,CACL,yBAAyB,eAAe,iCAAiC,WAAW,MAAM;gBAC1F,gFAAgF;gBAChF,kCAAkC,CACnC,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,mBAAmB,CACxB,MAAc,EACd,KAAc,EACd,UAAmB;QAEnB,OAAO,IAAI,sBAAsB,CAAC;YAChC,MAAM;YACN,KAAK;YACL,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,oBAAoB,CAAC,UAAmB;QAC7C,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;;AAGH,8BAA8B;AAC9B,uBAAuB,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE;IAClE,OAAO,IAAI,uBAAuB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,uBAAuB,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE;IACjE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,IAAI,sBAAsB,CAAC;QAChC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -19,6 +19,12 @@ export interface OpenAIEmbeddingConfig {
|
|
|
19
19
|
* Optional version string
|
|
20
20
|
*/
|
|
21
21
|
version?: string;
|
|
22
|
+
/**
|
|
23
|
+
* Optional full embeddings endpoint URL. Defaults to OpenAI. Point this at any
|
|
24
|
+
* OpenAI-compatible `/embeddings` endpoint (e.g. Cloudflare Workers AI:
|
|
25
|
+
* `https://api.cloudflare.com/client/v4/accounts/<id>/ai/v1/embeddings`).
|
|
26
|
+
*/
|
|
27
|
+
apiEndpoint?: string;
|
|
22
28
|
}
|
|
23
29
|
/**
|
|
24
30
|
* Service implementation that generates embeddings using OpenAI's API
|
|
@@ -28,7 +28,7 @@ export class OpenAIEmbeddingService extends EmbeddingService {
|
|
|
28
28
|
this.model = config.model || 'text-embedding-3-small';
|
|
29
29
|
this.dimensions = config.dimensions || 1536; // text-embedding-3-small has 1536 dimensions
|
|
30
30
|
this.version = config.version || '3.0.0';
|
|
31
|
-
this.apiEndpoint = 'https://api.openai.com/v1/embeddings';
|
|
31
|
+
this.apiEndpoint = config.apiEndpoint || 'https://api.openai.com/v1/embeddings';
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
34
|
* Generate an embedding for a single text
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddingService.js","sourceRoot":"","sources":["../../src/embeddings/OpenAIEmbeddingService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,gBAAgB,EAA2B,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddingService.js","sourceRoot":"","sources":["../../src/embeddings/OpenAIEmbeddingService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,gBAAgB,EAA2B,MAAM,uBAAuB,CAAC;AAuDlF;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,gBAAgB;IAClD,MAAM,CAAS;IACf,KAAK,CAAS;IACd,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,WAAW,CAAS;IAE5B;;;;OAIG;IACH,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QAED,kFAAkF;QAClF,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,6CAA6C;QAC1F,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,sCAAsC,CAAC;IAClF,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE;YACnC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;YAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,WAAW,EAChB;gBACE,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,EACD;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;oBACtC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,OAAO,EAAE,MAAM,EAAE,oBAAoB;aACtC,CACF,CAAC;YAEF,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAElD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9E,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC/E,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAElD,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtE,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChE,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBAClC,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;aAClC,CAAC,CAAC;YAEH,mCAAmC;YACnC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,SAAS,CAAC;gBAC/D,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE;gBACnC,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aAC9B,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,mCAAmC;YACnC,MAAM,UAAU,GAAG,KAOlB,CAAC;YACF,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAC/C,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC;gBAE/C,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;oBAC/B,MAAM,EAAE,UAAU;oBAClB,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,UAAU,CAAC,OAAO;iBAC5B,CAAC,CAAC;gBAEH,8BAA8B;gBAC9B,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;gBACxE,CAAC;qBAAM,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACtE,CAAC;qBAAM,IAAI,UAAU,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBAC3C,MAAM,IAAI,KAAK,CAAC,4BAA4B,UAAU,qBAAqB,CAAC,CAAC;gBAC/E,CAAC;gBAED,8CAA8C;gBAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAE3F,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,sBAAsB;YACtB,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,kBAAkB,CAAC,KAAe;QAC/C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,WAAW,EAChB;gBACE,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,EACD;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;oBACtC,cAAc,EAAE,kBAAkB;iBACnC;aACF,CACF,CAAC;YAEF,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAElE,kCAAkC;YAClC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACnC,CAAC;YAED,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;OAIG;IACM,YAAY;QACnB,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,KAAK,CAAC,OAAO,CAAC;QACvB,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,MAAgB;QACvC,iDAAiD;QACjD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC7B,SAAS,IAAI,OAAO,GAAG,OAAO,CAAC;QACjC,CAAC;QACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEjC,yBAAyB;QACzB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,2BAA2B;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;YACzB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,oEAAoE;YACpE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Entity } from '../KnowledgeGraphManager.js';
|
|
2
|
+
/**
|
|
3
|
+
* Build the canonical text representation of an entity.
|
|
4
|
+
*
|
|
5
|
+
* Used for BOTH embedding generation and reranking so the two never drift —
|
|
6
|
+
* single source of truth for "what text represents this entity". Pure (no side
|
|
7
|
+
* effects); callers add their own logging.
|
|
8
|
+
*
|
|
9
|
+
* @param entity - Entity to render
|
|
10
|
+
* @returns Multi-line text: name, type, and observations
|
|
11
|
+
*/
|
|
12
|
+
export declare function prepareEntityText(entity: Entity): string;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the canonical text representation of an entity.
|
|
3
|
+
*
|
|
4
|
+
* Used for BOTH embedding generation and reranking so the two never drift —
|
|
5
|
+
* single source of truth for "what text represents this entity". Pure (no side
|
|
6
|
+
* effects); callers add their own logging.
|
|
7
|
+
*
|
|
8
|
+
* @param entity - Entity to render
|
|
9
|
+
* @returns Multi-line text: name, type, and observations
|
|
10
|
+
*/
|
|
11
|
+
export function prepareEntityText(entity) {
|
|
12
|
+
const lines = [`Name: ${entity.name}`, `Type: ${entity.entityType}`, 'Observations:'];
|
|
13
|
+
if (entity.observations) {
|
|
14
|
+
let observationsArray = entity.observations;
|
|
15
|
+
if (typeof entity.observations === 'string') {
|
|
16
|
+
try {
|
|
17
|
+
observationsArray = JSON.parse(entity.observations);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
observationsArray = [entity.observations];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (!Array.isArray(observationsArray)) {
|
|
24
|
+
observationsArray = [String(observationsArray)];
|
|
25
|
+
}
|
|
26
|
+
const arr = observationsArray;
|
|
27
|
+
if (arr.length > 0) {
|
|
28
|
+
lines.push(...arr.map(obs => `- ${obs}`));
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
lines.push(' (No observations)');
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
lines.push(' (No observations)');
|
|
36
|
+
}
|
|
37
|
+
return lines.join('\n');
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=entityText.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entityText.js","sourceRoot":"","sources":["../../src/embeddings/entityText.ts"],"names":[],"mappings":"AAEA;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,MAAM,KAAK,GAAG,CAAC,SAAS,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,MAAM,CAAC,UAAU,EAAE,EAAE,eAAe,CAAC,CAAC;IAEtF,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,IAAI,iBAAiB,GAAY,MAAM,CAAC,YAAY,CAAC;QACrD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACtC,iBAAiB,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,GAAG,GAAG,iBAA6B,CAAC;QAC1C,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,7 @@ import { EmbeddingServiceFactory } from './embeddings/EmbeddingServiceFactory.js
|
|
|
7
7
|
import { Neo4jJobStore } from './embeddings/Neo4jJobStore.js';
|
|
8
8
|
import { KnowledgeGraphManager } from './KnowledgeGraphManager.js';
|
|
9
9
|
import { PrometheusMetrics } from './metrics/PrometheusMetrics.js';
|
|
10
|
+
import { RerankerService } from './retrieval/RerankerService.js';
|
|
10
11
|
import { setupServer } from './server/setup.js';
|
|
11
12
|
import { createAdaptedStorageProvider } from './storage/createAdaptedStorageProvider.js';
|
|
12
13
|
import { logger } from './utils/logger.js';
|
|
@@ -25,66 +26,77 @@ const staleClaimMs = process.env.EMBEDDING_STALE_CLAIM_MS
|
|
|
25
26
|
let prometheusMetrics = null;
|
|
26
27
|
// Initialize embedding job manager only if storage provider supports it
|
|
27
28
|
let embeddingJobManager;
|
|
28
|
-
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
logger.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
29
|
+
if (!EmbeddingServiceFactory.hasEmbeddingProvider()) {
|
|
30
|
+
// No embedding provider configured (no EMBEDDING_API_KEY/OPENAI_API_KEY, not MOCK_EMBEDDINGS).
|
|
31
|
+
// Run keyword-only rather than generate meaningless random-vector "semantic" results.
|
|
32
|
+
logger.info('No embedding provider configured — semantic search disabled, running in keyword-only mode. ' +
|
|
33
|
+
'Set EMBEDDING_API_KEY (any OpenAI-compatible endpoint, e.g. Cloudflare Workers AI) or ' +
|
|
34
|
+
'OPENAI_API_KEY to enable embeddings, or MOCK_EMBEDDINGS=true for test vectors.');
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
try {
|
|
38
|
+
logger.debug(`Storage provider type: ${process.env.MEMORY_STORAGE_TYPE || 'default'}`);
|
|
39
|
+
// Initialize the embedding service (OpenAI-compatible: OpenAI / Cloudflare Workers AI / any)
|
|
40
|
+
const embeddingService = EmbeddingServiceFactory.createFromEnvironment();
|
|
41
|
+
logger.debug(`Embedding service model info: ${JSON.stringify(embeddingService.getModelInfo())}`);
|
|
42
|
+
// Production guard (v2.6.0): never wire a writing job manager to a random/mock
|
|
43
|
+
// service. Catches BOTH MOCK_EMBEDDINGS and the silent fallback-to-Default path
|
|
44
|
+
// (configured key whose service construction failed). Thrown error is handled
|
|
45
|
+
// by the catch below → embeddingJobManager stays undefined → keyword-only mode.
|
|
46
|
+
if (!EmbeddingServiceFactory.shouldWriteEmbeddings(embeddingService)) {
|
|
47
|
+
throw new Error('NODE_ENV=production with a random/mock embedding service (DefaultEmbeddingService) — ' +
|
|
48
|
+
'refusing to write embeddings. Set EMBEDDING_API_KEY or OPENAI_API_KEY to a real ' +
|
|
49
|
+
'provider. Falling back to keyword-only mode.');
|
|
50
|
+
}
|
|
51
|
+
// Configure rate limiting options - stricter limits to prevent OpenAI API abuse
|
|
52
|
+
const rateLimiterOptions = {
|
|
53
|
+
tokensPerInterval: process.env.EMBEDDING_RATE_LIMIT_TOKENS
|
|
54
|
+
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_TOKENS, 10)
|
|
55
|
+
: 20, // Default: 20 requests per minute
|
|
56
|
+
interval: process.env.EMBEDDING_RATE_LIMIT_INTERVAL
|
|
57
|
+
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_INTERVAL, 10)
|
|
58
|
+
: 60 * 1000, // Default: 1 minute
|
|
59
|
+
};
|
|
60
|
+
logger.info('Initializing EmbeddingJobManager', {
|
|
61
|
+
rateLimiterOptions,
|
|
62
|
+
model: embeddingService.getModelInfo().name,
|
|
63
|
+
storageType: process.env.MEMORY_STORAGE_TYPE || 'neo4j',
|
|
64
|
+
});
|
|
65
|
+
const adaptedStorageProvider = createAdaptedStorageProvider(storageProvider);
|
|
66
|
+
// Pull the Neo4j connection out of the storage provider so the queue
|
|
67
|
+
// shares the same driver/pool. Falls back to throwing if missing — we
|
|
68
|
+
// only support Neo4j storage in v2.x.
|
|
69
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
70
|
+
const getConnectionManager = storageProvider.getConnectionManager;
|
|
71
|
+
if (typeof getConnectionManager !== 'function') {
|
|
72
|
+
throw new Error('Storage provider does not expose getConnectionManager(); EmbeddingJobManager v2.4.0+ requires Neo4j.');
|
|
73
|
+
}
|
|
74
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
75
|
+
const connectionManager = getConnectionManager.call(storageProvider);
|
|
76
|
+
const jobStore = new Neo4jJobStore(connectionManager);
|
|
77
|
+
// Create the embedding job manager with adapted storage provider + Neo4j-backed queue
|
|
78
|
+
embeddingJobManager = new EmbeddingJobManager(
|
|
79
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
80
|
+
adaptedStorageProvider, embeddingService, rateLimiterOptions, null, // Use default cache options
|
|
81
|
+
logger, jobStore, staleClaimMs);
|
|
82
|
+
logger.info('EmbeddingJobManager initialized (background jobs will start only in production)');
|
|
39
83
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
: 20, // Default: 20 requests per minute
|
|
48
|
-
interval: process.env.EMBEDDING_RATE_LIMIT_INTERVAL
|
|
49
|
-
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_INTERVAL, 10)
|
|
50
|
-
: 60 * 1000, // Default: 1 minute
|
|
51
|
-
};
|
|
52
|
-
logger.info('Initializing EmbeddingJobManager', {
|
|
53
|
-
rateLimiterOptions,
|
|
54
|
-
model: embeddingService.getModelInfo().name,
|
|
55
|
-
storageType: process.env.MEMORY_STORAGE_TYPE || 'neo4j',
|
|
56
|
-
});
|
|
57
|
-
const adaptedStorageProvider = createAdaptedStorageProvider(storageProvider);
|
|
58
|
-
// Pull the Neo4j connection out of the storage provider so the queue
|
|
59
|
-
// shares the same driver/pool. Falls back to throwing if missing — we
|
|
60
|
-
// only support Neo4j storage in v2.x.
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
62
|
-
const getConnectionManager = storageProvider.getConnectionManager;
|
|
63
|
-
if (typeof getConnectionManager !== 'function') {
|
|
64
|
-
throw new Error('Storage provider does not expose getConnectionManager(); EmbeddingJobManager v2.4.0+ requires Neo4j.');
|
|
84
|
+
catch (error) {
|
|
85
|
+
// Fail gracefully if embedding job manager initialization fails
|
|
86
|
+
logger.error('Failed to initialize EmbeddingJobManager', {
|
|
87
|
+
error: error instanceof Error ? error.message : String(error),
|
|
88
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
89
|
+
});
|
|
90
|
+
embeddingJobManager = undefined;
|
|
65
91
|
}
|
|
66
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
67
|
-
const connectionManager = getConnectionManager.call(storageProvider);
|
|
68
|
-
const jobStore = new Neo4jJobStore(connectionManager);
|
|
69
|
-
// Create the embedding job manager with adapted storage provider + Neo4j-backed queue
|
|
70
|
-
embeddingJobManager = new EmbeddingJobManager(
|
|
71
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
72
|
-
adaptedStorageProvider, embeddingService, rateLimiterOptions, null, // Use default cache options
|
|
73
|
-
logger, jobStore, staleClaimMs);
|
|
74
|
-
logger.info('EmbeddingJobManager initialized (background jobs will start only in production)');
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
// Fail gracefully if embedding job manager initialization fails
|
|
78
|
-
logger.error('Failed to initialize EmbeddingJobManager', {
|
|
79
|
-
error: error instanceof Error ? error.message : String(error),
|
|
80
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
81
|
-
});
|
|
82
|
-
embeddingJobManager = undefined;
|
|
83
92
|
}
|
|
93
|
+
// Build the optional cross-encoder reranker (disabled unless RERANK_ENABLED + endpoint/key resolve).
|
|
94
|
+
const reranker = RerankerService.fromEnv();
|
|
84
95
|
// Create the KnowledgeGraphManager with the storage provider, embedding job manager, and vector store options
|
|
85
96
|
const knowledgeGraphManager = new KnowledgeGraphManager({
|
|
86
97
|
storageProvider,
|
|
87
98
|
embeddingJobManager,
|
|
99
|
+
reranker,
|
|
88
100
|
writeEmbeddingsLocally,
|
|
89
101
|
// Pass vector store options from storage provider if available
|
|
90
102
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,MAAM,2CAA2C,CAAC;AAEzF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,2DAA2D;AAC3D,cAAc,4BAA4B,CAAC;AAG3C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,sDAAsD;AACtD,MAAM,eAAe,GAAG,yBAAyB,EAAE,CAAC;AAEpD,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,OAAO,CAAC;AAChF,MAAM,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY,CAAC;AAClF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB;IACvD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElB,qFAAqF;AACrF,IAAI,iBAAiB,GAA6B,IAAI,CAAC;AAEvD,wEAAwE;AACxE,IAAI,mBAAoD,CAAC;AACzD,IAAI,CAAC
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,MAAM,2CAA2C,CAAC;AAEzF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,2DAA2D;AAC3D,cAAc,4BAA4B,CAAC;AAG3C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,sDAAsD;AACtD,MAAM,eAAe,GAAG,yBAAyB,EAAE,CAAC;AAEpD,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,OAAO,CAAC;AAChF,MAAM,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY,CAAC;AAClF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB;IACvD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElB,qFAAqF;AACrF,IAAI,iBAAiB,GAA6B,IAAI,CAAC;AAEvD,wEAAwE;AACxE,IAAI,mBAAoD,CAAC;AACzD,IAAI,CAAC,uBAAuB,CAAC,oBAAoB,EAAE,EAAE,CAAC;IACpD,+FAA+F;IAC/F,sFAAsF;IACtF,MAAM,CAAC,IAAI,CACT,6FAA6F;QAC3F,wFAAwF;QACxF,gFAAgF,CACnF,CAAC;AACJ,CAAC;KAAM,CAAC;IACN,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC,CAAC;QAEvF,6FAA6F;QAC7F,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,qBAAqB,EAAE,CAAC;QACzE,MAAM,CAAC,KAAK,CACV,iCAAiC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC,EAAE,CACnF,CAAC;QAEF,+EAA+E;QAC/E,gFAAgF;QAChF,8EAA8E;QAC9E,gFAAgF;QAChF,IAAI,CAAC,uBAAuB,CAAC,qBAAqB,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CACb,uFAAuF;gBACrF,kFAAkF;gBAClF,8CAA8C,CACjD,CAAC;QACJ,CAAC;QAED,gFAAgF;QAChF,MAAM,kBAAkB,GAAG;YACzB,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B;gBACxD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,CAAC;gBAC9D,CAAC,CAAC,EAAE,EAAE,kCAAkC;YAC1C,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,6BAA6B;gBACjD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,EAAE,CAAC;gBAChE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,oBAAoB;SACpC,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;YAC9C,kBAAkB;YAClB,KAAK,EAAE,gBAAgB,CAAC,YAAY,EAAE,CAAC,IAAI;YAC3C,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO;SACxD,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,4BAA4B,CAAC,eAAe,CAAC,CAAC;QAE7E,qEAAqE;QACrE,sEAAsE;QACtE,sCAAsC;QACtC,8DAA8D;QAC9D,MAAM,oBAAoB,GAAI,eAAuB,CAAC,oBAAoB,CAAC;QAC3E,IAAI,OAAO,oBAAoB,KAAK,UAAU,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,sGAAsG,CACvG,CAAC;QACJ,CAAC;QACD,8DAA8D;QAC9D,MAAM,iBAAiB,GAA2B,oBAAoB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7F,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC,iBAAiB,CAAC,CAAC;QAEtD,sFAAsF;QACtF,mBAAmB,GAAG,IAAI,mBAAmB;QAC3C,8DAA8D;QAC9D,sBAA6B,EAC7B,gBAAgB,EAChB,kBAAkB,EAClB,IAAI,EAAE,4BAA4B;QAClC,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IACjG,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gEAAgE;QAChE,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE;YACvD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACxD,CAAC,CAAC;QACH,mBAAmB,GAAG,SAAS,CAAC;IAClC,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;AAE3C,8GAA8G;AAC9G,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;IACtD,eAAe;IACf,mBAAmB;IACnB,QAAQ;IACR,sBAAsB;IACtB,+DAA+D;IAC/D,8DAA8D;IAC9D,kBAAkB,EAAG,eAAuB,CAAC,kBAAkB;CAChE,CAAC,CAAC;AAEH,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC5B,MAAM,CAAC,IAAI,CACT,+EAA+E;QAC7E,wFAAwF;QACxF,yBAAyB,CAC5B,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,8DAA8D;AAC9D,8DAA8D;AAC9D,MAAM,wBAAwB,GAAG,qBAA4B,CAAC;AAE9D,IACE,wBAAwB,CAAC,eAAe;IACxC,OAAO,wBAAwB,CAAC,eAAe,CAAC,iBAAiB,KAAK,UAAU,EAChF,CAAC;IACD,2DAA2D;IAC3D,wBAAwB,CAAC,eAAe,CAAC,iBAAiB,GAAG,KAAK,EAChE,IAAY;IACZ,8DAA8D;IAC9D,SAAc,EACd,EAAE;QACF,MAAM,CAAC,KAAK,CAAC,qEAAqE,IAAI,EAAE,EAAE;YACxF,aAAa,EAAE,OAAO,SAAS;YAC/B,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW;YACtD,KAAK,EAAE,SAAS,EAAE,KAAK,IAAI,UAAU;SACtC,CAAC,CAAC;QAEH,0CAA0C;QAC1C,MAAM,kBAAkB,GAAG;YACzB,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,SAAS;YACrC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS;YACnC,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;SACjD,CAAC;QAEF,IAAI,OAAO,wBAAwB,CAAC,eAAe,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;YACzF,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,CACV,wEAAwE,IAAI,EAAE,CAC/E,CAAC;gBACF,OAAO,MAAM,wBAAwB,CAAC,eAAe,CAAC,qBAAqB,CACzE,IAAI,EACJ,kBAAkB,CACnB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,uEAAuE,IAAI,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kFAAkF,IAAI,EAAE,CAAC;YAC1G,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,CAAC,IAAI,CACT,4FAA4F,CAC7F,CAAC;AACJ,CAAC;AAED,4GAA4G;AAC5G,IAAI,qBAAqB,IAAI,OAAO,qBAAqB,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;IACxF,MAAM,sBAAsB,GAAG,qBAAqB,CAAC,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAChG,qBAAqB,CAAC,cAAc,GAAG,KAAK,WAAW,QAAQ;QAC7D,wDAAwD;QACxD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,mBAAmB,IAAI,sBAAsB,EAAE,CAAC;YAClD,IAAI,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,6DAA6D,EAAE;oBACzE,WAAW,EAAE,QAAQ,CAAC,MAAM;oBAC5B,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;iBAClD,CAAC,CAAC;gBACH,MAAM,mBAAmB,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE;oBAC1D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED,kDAAkD;AAClD,MAAM,MAAM,GAAG,WAAW,CAAC,qBAAqB,CAAC,CAAC;AAElD,mCAAmC;AACnC,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,iEAAiE;AACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;IACnE,qFAAqF;IACrF,iBAAiB,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;IACpD,iBAAiB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAEpC,8FAA8F;IAC9F,IAAI,mBAAmB,IAAI,sBAAsB,EAAE,CAAC;QAClD,MAAM,0BAA0B,GAAG,MAAM,CAAC,CAAC,wCAAwC;QACnF,WAAW,CAAC,KAAK,IAAI,EAAE;YACrB,IAAI,CAAC;gBACH,iCAAiC;gBACjC,MAAM,mBAAmB,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,4BAA4B;gBAC5B,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;oBAChD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAE/B,+FAA+F;QAC/F,2FAA2F;QAC3F,+EAA+E;QAC/E,IAAI,CAAC,QAAQ,CACX,qBAAqB,EACrB,KAAK,IAAI,EAAE;YACT,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE;gBACzD,QAAQ,EAAE,qBAAqB;aAChC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,mBAAmB,EAAE,+BAA+B,EAAE,CAAC;gBACpF,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;oBAChD,iBAAiB,EAAE,cAAc;oBACjC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE;oBAC9C,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACvD,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EACD;YACE,QAAQ,EAAE,KAAK;SAChB,CACF,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;YAC7C,kBAAkB,EAAE,SAAS,0BAA0B,IAAI;YAC3D,gBAAgB,EAAE,qBAAqB;YACvC,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QACnB,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for the neural reranker stage.
|
|
3
|
+
*
|
|
4
|
+
* Disabled by default — reranking is strictly additive. When `enabled` is false
|
|
5
|
+
* (no config, or `RERANK_ENABLED` unset) the search pipeline returns its
|
|
6
|
+
* vector/hybrid ordering unchanged.
|
|
7
|
+
*/
|
|
8
|
+
export interface RerankConfig {
|
|
9
|
+
enabled: boolean;
|
|
10
|
+
/** Full native run URL for the reranker model. */
|
|
11
|
+
endpoint: string;
|
|
12
|
+
model: string;
|
|
13
|
+
apiKey: string;
|
|
14
|
+
/** Candidates sent to the reranker (recall set). */
|
|
15
|
+
topN: number;
|
|
16
|
+
/** Results kept after rerank. */
|
|
17
|
+
topK: number;
|
|
18
|
+
/** Per-passage truncation for scoring only (reranker context limits). */
|
|
19
|
+
maxPassageChars: number;
|
|
20
|
+
timeoutMs: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Thin client for a cross-encoder reranker (default: Cloudflare Workers AI
|
|
24
|
+
* `@cf/baai/bge-reranker-base`). OpenAI-style/native HTTP; reuses `axios`.
|
|
25
|
+
*
|
|
26
|
+
* The service itself MAY throw (transport/parse errors) — callers are expected
|
|
27
|
+
* to fail open (return the pre-rerank order). It never enables itself without a
|
|
28
|
+
* resolved endpoint + API key.
|
|
29
|
+
*/
|
|
30
|
+
export declare class RerankerService {
|
|
31
|
+
private readonly cfg;
|
|
32
|
+
constructor(cfg: RerankConfig);
|
|
33
|
+
get enabled(): boolean;
|
|
34
|
+
get topN(): number;
|
|
35
|
+
get topK(): number;
|
|
36
|
+
get model(): string;
|
|
37
|
+
/**
|
|
38
|
+
* Build a reranker from environment variables. Returns a DISABLED instance
|
|
39
|
+
* unless `RERANK_ENABLED=true` AND an endpoint + API key resolve.
|
|
40
|
+
*
|
|
41
|
+
* Env: RERANK_ENABLED, RERANK_MODEL (default @cf/baai/bge-reranker-base),
|
|
42
|
+
* RERANK_ENDPOINT (or derived from RERANK_ACCOUNT_ID/CF_ACCOUNT_ID + model),
|
|
43
|
+
* RERANK_API_KEY (falls back to EMBEDDING_API_KEY), RERANK_TOP_N (20),
|
|
44
|
+
* RERANK_TOP_K (10), RERANK_MAX_PASSAGE_CHARS (2000), RERANK_TIMEOUT_MS (5000).
|
|
45
|
+
*/
|
|
46
|
+
static fromEnv(): RerankerService;
|
|
47
|
+
/**
|
|
48
|
+
* Rerank `passages` against `query`. Returns candidate indices reordered
|
|
49
|
+
* descending by relevance, length <= topK. THROWS on transport/parse error —
|
|
50
|
+
* the caller fail-opens.
|
|
51
|
+
*
|
|
52
|
+
* @param query - Search query
|
|
53
|
+
* @param passages - Candidate passages (index-aligned with the caller's results)
|
|
54
|
+
* @returns Indices into `passages`, best-first
|
|
55
|
+
*/
|
|
56
|
+
rerank(query: string, passages: string[]): Promise<number[]>;
|
|
57
|
+
}
|