@henrychong-ai/mcp-neo4j-knowledge-graph 2.4.0 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/KnowledgeGraphManager.d.ts +18 -0
- package/dist/KnowledgeGraphManager.js +52 -1
- package/dist/KnowledgeGraphManager.js.map +1 -1
- package/dist/embeddings/EmbeddingJobManager.d.ts +21 -2
- package/dist/embeddings/EmbeddingJobManager.js +31 -44
- package/dist/embeddings/EmbeddingJobManager.js.map +1 -1
- package/dist/embeddings/EmbeddingServiceFactory.d.ts +10 -0
- package/dist/embeddings/EmbeddingServiceFactory.js +46 -17
- package/dist/embeddings/EmbeddingServiceFactory.js.map +1 -1
- package/dist/embeddings/OpenAIEmbeddingService.d.ts +6 -0
- package/dist/embeddings/OpenAIEmbeddingService.js +1 -1
- package/dist/embeddings/OpenAIEmbeddingService.js.map +1 -1
- package/dist/embeddings/entityText.d.ts +12 -0
- package/dist/embeddings/entityText.js +39 -0
- package/dist/embeddings/entityText.js.map +1 -0
- package/dist/index.js +56 -53
- package/dist/index.js.map +1 -1
- package/dist/retrieval/RerankerService.d.ts +57 -0
- package/dist/retrieval/RerankerService.js +117 -0
- package/dist/retrieval/RerankerService.js.map +1 -0
- package/dist/storage/createAdaptedStorageProvider.js +12 -0
- package/dist/storage/createAdaptedStorageProvider.js.map +1 -1
- package/dist/storage/neo4j/Neo4jStorageProvider.d.ts +13 -0
- package/dist/storage/neo4j/Neo4jStorageProvider.js +20 -0
- package/dist/storage/neo4j/Neo4jStorageProvider.js.map +1 -1
- package/package.json +14 -13
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"EmbeddingServiceFactory.js","sourceRoot":"","sources":["../../src/embeddings/EmbeddingServiceFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"EmbeddingServiceFactory.js","sourceRoot":"","sources":["../../src/embeddings/EmbeddingServiceFactory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAEvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAoBrE;;GAEG;AACH,MAAM,OAAO,uBAAuB;IAClC;;OAEG;IACK,MAAM,CAAC,SAAS,GAA6C,EAAE,CAAC;IAExE;;;;;OAKG;IACH,MAAM,CAAC,gBAAgB,CAAC,IAAY,EAAE,QAAkC;QACtE,uBAAuB,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,GAAG,QAAQ,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa;QAClB,uBAAuB,CAAC,SAAS,GAAG,EAAE,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,qBAAqB;QAC1B,OAAO,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,aAAa,CAAC,SAAiC,EAAE;QACtD,MAAM,YAAY,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QAClE,MAAM,CAAC,KAAK,CAAC,4DAA4D,YAAY,GAAG,CAAC,CAAC;QAE1F,MAAM,UAAU,GAAG,uBAAuB,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAEnE,IAAI,UAAU,EAAE,CAAC;YACf,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,CAAC,KAAK,CACV,wEAAwE,YAAY,GAAG,EACvF;oBACE,SAAS,EAAE,OAAO,CAAC,YAAY,EAAE;iBAClC,CACF,CAAC;gBACF,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,oEAAoE,YAAY,GAAG,EACnF,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,sCAAsC,YAAY,qBAAqB,CAAC,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,aAAa,YAAY,qBAAqB,CAAC,CAAC;IAClE,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,qBAAqB;QAC1B,yEAAyE;QACzE,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,CAAC;QAEjE,sFAAsF;QACtF,gGAAgG;QAChG,sFAAsF;QACtF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC3E,MAAM,cAAc,GAClB,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,wBAAwB,CAAC;QAChG,MAAM,WAAW,GACf,OAAO,CAAC,GAAG,CAAC,sBAAsB;YAClC,CAAC,OAAO,CAAC,GAAG,CAAC,sBAAsB;gBACjC,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa;gBACvE,CAAC,CAAC,SAAS,CAAC,CAAC;QACjB,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB;YACjD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC;YACvD,CAAC,CAAC,SAAS,CAAC;QAEd,MAAM,CAAC,KAAK,CAAC,sEAAsE,EAAE;YACnF,cAAc,EAAE,iBAAiB;YACjC,aAAa,EAAE,CAAC,CAAC,MAAM;YACvB,cAAc;YACd,WAAW,EAAE,WAAW,IAAI,gBAAgB;YAC5C,UAAU,EAAE,UAAU,IAAI,SAAS;SACpC,CAAC,CAAC;QAEH,IAAI,iBAAiB,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,uEAAuE,CAAC,CAAC;YACrF,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,CAAC,uEAAuE,EAAE;oBACpF,KAAK,EAAE,cAAc;oBACrB,WAAW,EAAE,WAAW,IAAI,gBAAgB;iBAC7C,CAAC,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,sBAAsB,CAAC;oBACzC,MAAM;oBACN,KAAK,EAAE,cAAc;oBACrB,WAAW;oBACX,UAAU;iBACX,CAAC,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,sEAAsE,EAAE;oBAClF,KAAK,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,IAAI;oBAClC,UAAU,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,UAAU;iBAC9C,CAAC,CAAC;gBACH,OAAO,OAAO,CAAC;YACjB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,qEAAqE,EAAE,KAAK,CAAC,CAAC;gBAC3F,MAAM,CAAC,IAAI,CAAC,oEAAoE,CAAC,CAAC;gBAClF,gDAAgD;gBAChD,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,sFAAsF;QACtF,mFAAmF;QACnF,sFAAsF;QACtF,MAAM,CAAC,IAAI,CACT,sFAAsF,CACvF,CAAC;QACF,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,oBAAoB;QACzB,OAAO,CACL,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM;YACtC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAChE,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,mBAAmB,CACxB,MAAc,EACd,KAAc,EACd,UAAmB;QAEnB,OAAO,IAAI,sBAAsB,CAAC;YAChC,MAAM;YACN,KAAK;YACL,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,oBAAoB,CAAC,UAAmB;QAC7C,OAAO,IAAI,uBAAuB,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;;AAGH,8BAA8B;AAC9B,uBAAuB,CAAC,gBAAgB,CAAC,SAAS,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE;IAClE,OAAO,IAAI,uBAAuB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,uBAAuB,CAAC,gBAAgB,CAAC,QAAQ,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE;IACjE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,IAAI,sBAAsB,CAAC;QAChC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -19,6 +19,12 @@ export interface OpenAIEmbeddingConfig {
|
|
|
19
19
|
* Optional version string
|
|
20
20
|
*/
|
|
21
21
|
version?: string;
|
|
22
|
+
/**
|
|
23
|
+
* Optional full embeddings endpoint URL. Defaults to OpenAI. Point this at any
|
|
24
|
+
* OpenAI-compatible `/embeddings` endpoint (e.g. Cloudflare Workers AI:
|
|
25
|
+
* `https://api.cloudflare.com/client/v4/accounts/<id>/ai/v1/embeddings`).
|
|
26
|
+
*/
|
|
27
|
+
apiEndpoint?: string;
|
|
22
28
|
}
|
|
23
29
|
/**
|
|
24
30
|
* Service implementation that generates embeddings using OpenAI's API
|
|
@@ -28,7 +28,7 @@ export class OpenAIEmbeddingService extends EmbeddingService {
|
|
|
28
28
|
this.model = config.model || 'text-embedding-3-small';
|
|
29
29
|
this.dimensions = config.dimensions || 1536; // text-embedding-3-small has 1536 dimensions
|
|
30
30
|
this.version = config.version || '3.0.0';
|
|
31
|
-
this.apiEndpoint = 'https://api.openai.com/v1/embeddings';
|
|
31
|
+
this.apiEndpoint = config.apiEndpoint || 'https://api.openai.com/v1/embeddings';
|
|
32
32
|
}
|
|
33
33
|
/**
|
|
34
34
|
* Generate an embedding for a single text
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"OpenAIEmbeddingService.js","sourceRoot":"","sources":["../../src/embeddings/OpenAIEmbeddingService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,gBAAgB,EAA2B,MAAM,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"OpenAIEmbeddingService.js","sourceRoot":"","sources":["../../src/embeddings/OpenAIEmbeddingService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,gBAAgB,EAA2B,MAAM,uBAAuB,CAAC;AAuDlF;;GAEG;AACH,MAAM,OAAO,sBAAuB,SAAQ,gBAAgB;IAClD,MAAM,CAAS;IACf,KAAK,CAAS;IACd,UAAU,CAAS;IACnB,OAAO,CAAS;IAChB,WAAW,CAAS;IAE5B;;;;OAIG;IACH,YAAY,MAA6B;QACvC,KAAK,EAAE,CAAC;QAER,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QAED,kFAAkF;QAClF,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,wBAAwB,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,IAAI,CAAC,CAAC,6CAA6C;QAC1F,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,sCAAsC,CAAC;IAClF,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,iBAAiB,CAAC,IAAY;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE;YACnC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;YAC/B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW,EAAE,IAAI,CAAC,WAAW;SAC9B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,WAAW,EAChB;gBACE,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,EACD;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;oBACtC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,OAAO,EAAE,MAAM,EAAE,oBAAoB;aACtC,CACF,CAAC;YAEF,MAAM,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;YAElD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC9E,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;YAC/E,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAElD,IAAI,CAAC,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtE,MAAM,CAAC,KAAK,CAAC,4BAA4B,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;gBAC1D,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChE,CAAC;YAED,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE;gBAClC,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7B,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;aAClC,CAAC,CAAC;YAEH,mCAAmC;YACnC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,SAAS,CAAC;gBAC/D,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC3D,CAAC;YAED,iCAAiC;YACjC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,sBAAsB,EAAE;gBACnC,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;aAC9B,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC;QACnB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,mCAAmC;YACnC,MAAM,UAAU,GAAG,KAOlB,CAAC;YACF,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAC/C,MAAM,YAAY,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC;gBAE/C,MAAM,CAAC,KAAK,CAAC,kBAAkB,EAAE;oBAC/B,MAAM,EAAE,UAAU;oBAClB,IAAI,EAAE,YAAY;oBAClB,OAAO,EAAE,UAAU,CAAC,OAAO;iBAC5B,CAAC,CAAC;gBAEH,8BAA8B;gBAC9B,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBACvB,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;gBACxE,CAAC;qBAAM,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gBACtE,CAAC;qBAAM,IAAI,UAAU,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;oBAC3C,MAAM,IAAI,KAAK,CAAC,4BAA4B,UAAU,qBAAqB,CAAC,CAAC;gBAC/E,CAAC;gBAED,8CAA8C;gBAC9C,MAAM,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAE3F,MAAM,IAAI,KAAK,CAAC,qBAAqB,UAAU,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC,CAAC;YAClF,CAAC;YAED,sBAAsB;YACtB,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,kBAAkB,CAAC,KAAe;QAC/C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,WAAW,EAChB;gBACE,KAAK,EAAE,KAAK;gBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,EACD;gBACE,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;oBACtC,cAAc,EAAE,kBAAkB;iBACnC;aACF,CACF,CAAC;YAEF,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAElE,kCAAkC;YAClC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;gBACnC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;YACnC,CAAC;YAED,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAClD,MAAM,IAAI,KAAK,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED;;;;OAIG;IACM,YAAY;QACnB,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,KAAK;YAChB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,KAAK,CAAC,OAAO,CAAC;QACvB,CAAC;QACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACK,gBAAgB,CAAC,MAAgB;QACvC,iDAAiD;QACjD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC7B,SAAS,IAAI,OAAO,GAAG,OAAO,CAAC;QACjC,CAAC;QACD,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEjC,yBAAyB;QACzB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,2BAA2B;YAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;YACzB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,oEAAoE;YACpE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Entity } from '../KnowledgeGraphManager.js';
|
|
2
|
+
/**
|
|
3
|
+
* Build the canonical text representation of an entity.
|
|
4
|
+
*
|
|
5
|
+
* Used for BOTH embedding generation and reranking so the two never drift —
|
|
6
|
+
* single source of truth for "what text represents this entity". Pure (no side
|
|
7
|
+
* effects); callers add their own logging.
|
|
8
|
+
*
|
|
9
|
+
* @param entity - Entity to render
|
|
10
|
+
* @returns Multi-line text: name, type, and observations
|
|
11
|
+
*/
|
|
12
|
+
export declare function prepareEntityText(entity: Entity): string;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build the canonical text representation of an entity.
|
|
3
|
+
*
|
|
4
|
+
* Used for BOTH embedding generation and reranking so the two never drift —
|
|
5
|
+
* single source of truth for "what text represents this entity". Pure (no side
|
|
6
|
+
* effects); callers add their own logging.
|
|
7
|
+
*
|
|
8
|
+
* @param entity - Entity to render
|
|
9
|
+
* @returns Multi-line text: name, type, and observations
|
|
10
|
+
*/
|
|
11
|
+
export function prepareEntityText(entity) {
|
|
12
|
+
const lines = [`Name: ${entity.name}`, `Type: ${entity.entityType}`, 'Observations:'];
|
|
13
|
+
if (entity.observations) {
|
|
14
|
+
let observationsArray = entity.observations;
|
|
15
|
+
if (typeof entity.observations === 'string') {
|
|
16
|
+
try {
|
|
17
|
+
observationsArray = JSON.parse(entity.observations);
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
observationsArray = [entity.observations];
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (!Array.isArray(observationsArray)) {
|
|
24
|
+
observationsArray = [String(observationsArray)];
|
|
25
|
+
}
|
|
26
|
+
const arr = observationsArray;
|
|
27
|
+
if (arr.length > 0) {
|
|
28
|
+
lines.push(...arr.map(obs => `- ${obs}`));
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
lines.push(' (No observations)');
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
lines.push(' (No observations)');
|
|
36
|
+
}
|
|
37
|
+
return lines.join('\n');
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=entityText.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"entityText.js","sourceRoot":"","sources":["../../src/embeddings/entityText.ts"],"names":[],"mappings":"AAEA;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,MAAM,KAAK,GAAG,CAAC,SAAS,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,MAAM,CAAC,UAAU,EAAE,EAAE,eAAe,CAAC,CAAC;IAEtF,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;QACxB,IAAI,iBAAiB,GAAY,MAAM,CAAC,YAAY,CAAC;QACrD,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACtD,CAAC;YAAC,MAAM,CAAC;gBACP,iBAAiB,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACtC,iBAAiB,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,GAAG,GAAG,iBAA6B,CAAC;QAC1C,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IACpC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7,6 +7,7 @@ import { EmbeddingServiceFactory } from './embeddings/EmbeddingServiceFactory.js
|
|
|
7
7
|
import { Neo4jJobStore } from './embeddings/Neo4jJobStore.js';
|
|
8
8
|
import { KnowledgeGraphManager } from './KnowledgeGraphManager.js';
|
|
9
9
|
import { PrometheusMetrics } from './metrics/PrometheusMetrics.js';
|
|
10
|
+
import { RerankerService } from './retrieval/RerankerService.js';
|
|
10
11
|
import { setupServer } from './server/setup.js';
|
|
11
12
|
import { createAdaptedStorageProvider } from './storage/createAdaptedStorageProvider.js';
|
|
12
13
|
import { logger } from './utils/logger.js';
|
|
@@ -25,66 +26,68 @@ const staleClaimMs = process.env.EMBEDDING_STALE_CLAIM_MS
|
|
|
25
26
|
let prometheusMetrics = null;
|
|
26
27
|
// Initialize embedding job manager only if storage provider supports it
|
|
27
28
|
let embeddingJobManager;
|
|
28
|
-
|
|
29
|
-
//
|
|
30
|
-
|
|
31
|
-
logger.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
29
|
+
if (!EmbeddingServiceFactory.hasEmbeddingProvider()) {
|
|
30
|
+
// No embedding provider configured (no EMBEDDING_API_KEY/OPENAI_API_KEY, not MOCK_EMBEDDINGS).
|
|
31
|
+
// Run keyword-only rather than generate meaningless random-vector "semantic" results.
|
|
32
|
+
logger.info('No embedding provider configured — semantic search disabled, running in keyword-only mode. ' +
|
|
33
|
+
'Set EMBEDDING_API_KEY (any OpenAI-compatible endpoint, e.g. Cloudflare Workers AI) or ' +
|
|
34
|
+
'OPENAI_API_KEY to enable embeddings, or MOCK_EMBEDDINGS=true for test vectors.');
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
try {
|
|
38
|
+
logger.debug(`Storage provider type: ${process.env.MEMORY_STORAGE_TYPE || 'default'}`);
|
|
39
|
+
// Initialize the embedding service (OpenAI-compatible: OpenAI / Cloudflare Workers AI / any)
|
|
40
|
+
const embeddingService = EmbeddingServiceFactory.createFromEnvironment();
|
|
41
|
+
logger.debug(`Embedding service model info: ${JSON.stringify(embeddingService.getModelInfo())}`);
|
|
42
|
+
// Configure rate limiting options - stricter limits to prevent OpenAI API abuse
|
|
43
|
+
const rateLimiterOptions = {
|
|
44
|
+
tokensPerInterval: process.env.EMBEDDING_RATE_LIMIT_TOKENS
|
|
45
|
+
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_TOKENS, 10)
|
|
46
|
+
: 20, // Default: 20 requests per minute
|
|
47
|
+
interval: process.env.EMBEDDING_RATE_LIMIT_INTERVAL
|
|
48
|
+
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_INTERVAL, 10)
|
|
49
|
+
: 60 * 1000, // Default: 1 minute
|
|
50
|
+
};
|
|
51
|
+
logger.info('Initializing EmbeddingJobManager', {
|
|
52
|
+
rateLimiterOptions,
|
|
53
|
+
model: embeddingService.getModelInfo().name,
|
|
54
|
+
storageType: process.env.MEMORY_STORAGE_TYPE || 'neo4j',
|
|
55
|
+
});
|
|
56
|
+
const adaptedStorageProvider = createAdaptedStorageProvider(storageProvider);
|
|
57
|
+
// Pull the Neo4j connection out of the storage provider so the queue
|
|
58
|
+
// shares the same driver/pool. Falls back to throwing if missing — we
|
|
59
|
+
// only support Neo4j storage in v2.x.
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
61
|
+
const getConnectionManager = storageProvider.getConnectionManager;
|
|
62
|
+
if (typeof getConnectionManager !== 'function') {
|
|
63
|
+
throw new Error('Storage provider does not expose getConnectionManager(); EmbeddingJobManager v2.4.0+ requires Neo4j.');
|
|
64
|
+
}
|
|
65
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
66
|
+
const connectionManager = getConnectionManager.call(storageProvider);
|
|
67
|
+
const jobStore = new Neo4jJobStore(connectionManager);
|
|
68
|
+
// Create the embedding job manager with adapted storage provider + Neo4j-backed queue
|
|
69
|
+
embeddingJobManager = new EmbeddingJobManager(
|
|
70
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
71
|
+
adaptedStorageProvider, embeddingService, rateLimiterOptions, null, // Use default cache options
|
|
72
|
+
logger, jobStore, staleClaimMs);
|
|
73
|
+
logger.info('EmbeddingJobManager initialized (background jobs will start only in production)');
|
|
39
74
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
: 20, // Default: 20 requests per minute
|
|
48
|
-
interval: process.env.EMBEDDING_RATE_LIMIT_INTERVAL
|
|
49
|
-
? Number.parseInt(process.env.EMBEDDING_RATE_LIMIT_INTERVAL, 10)
|
|
50
|
-
: 60 * 1000, // Default: 1 minute
|
|
51
|
-
};
|
|
52
|
-
logger.info('Initializing EmbeddingJobManager', {
|
|
53
|
-
rateLimiterOptions,
|
|
54
|
-
model: embeddingService.getModelInfo().name,
|
|
55
|
-
storageType: process.env.MEMORY_STORAGE_TYPE || 'neo4j',
|
|
56
|
-
});
|
|
57
|
-
const adaptedStorageProvider = createAdaptedStorageProvider(storageProvider);
|
|
58
|
-
// Pull the Neo4j connection out of the storage provider so the queue
|
|
59
|
-
// shares the same driver/pool. Falls back to throwing if missing — we
|
|
60
|
-
// only support Neo4j storage in v2.x.
|
|
61
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
62
|
-
const getConnectionManager = storageProvider.getConnectionManager;
|
|
63
|
-
if (typeof getConnectionManager !== 'function') {
|
|
64
|
-
throw new Error('Storage provider does not expose getConnectionManager(); EmbeddingJobManager v2.4.0+ requires Neo4j.');
|
|
75
|
+
catch (error) {
|
|
76
|
+
// Fail gracefully if embedding job manager initialization fails
|
|
77
|
+
logger.error('Failed to initialize EmbeddingJobManager', {
|
|
78
|
+
error: error instanceof Error ? error.message : String(error),
|
|
79
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
80
|
+
});
|
|
81
|
+
embeddingJobManager = undefined;
|
|
65
82
|
}
|
|
66
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
67
|
-
const connectionManager = getConnectionManager.call(storageProvider);
|
|
68
|
-
const jobStore = new Neo4jJobStore(connectionManager);
|
|
69
|
-
// Create the embedding job manager with adapted storage provider + Neo4j-backed queue
|
|
70
|
-
embeddingJobManager = new EmbeddingJobManager(
|
|
71
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
72
|
-
adaptedStorageProvider, embeddingService, rateLimiterOptions, null, // Use default cache options
|
|
73
|
-
logger, jobStore, staleClaimMs);
|
|
74
|
-
logger.info('EmbeddingJobManager initialized (background jobs will start only in production)');
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
// Fail gracefully if embedding job manager initialization fails
|
|
78
|
-
logger.error('Failed to initialize EmbeddingJobManager', {
|
|
79
|
-
error: error instanceof Error ? error.message : String(error),
|
|
80
|
-
stack: error instanceof Error ? error.stack : undefined,
|
|
81
|
-
});
|
|
82
|
-
embeddingJobManager = undefined;
|
|
83
83
|
}
|
|
84
|
+
// Build the optional cross-encoder reranker (disabled unless RERANK_ENABLED + endpoint/key resolve).
|
|
85
|
+
const reranker = RerankerService.fromEnv();
|
|
84
86
|
// Create the KnowledgeGraphManager with the storage provider, embedding job manager, and vector store options
|
|
85
87
|
const knowledgeGraphManager = new KnowledgeGraphManager({
|
|
86
88
|
storageProvider,
|
|
87
89
|
embeddingJobManager,
|
|
90
|
+
reranker,
|
|
88
91
|
writeEmbeddingsLocally,
|
|
89
92
|
// Pass vector store options from storage provider if available
|
|
90
93
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,MAAM,2CAA2C,CAAC;AAEzF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,2DAA2D;AAC3D,cAAc,4BAA4B,CAAC;AAG3C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,sDAAsD;AACtD,MAAM,eAAe,GAAG,yBAAyB,EAAE,CAAC;AAEpD,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,OAAO,CAAC;AAChF,MAAM,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY,CAAC;AAClF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB;IACvD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElB,qFAAqF;AACrF,IAAI,iBAAiB,GAA6B,IAAI,CAAC;AAEvD,wEAAwE;AACxE,IAAI,mBAAoD,CAAC;AACzD,IAAI,CAAC
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,yBAAyB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,qCAAqC,CAAC;AAC1E,OAAO,EAAE,uBAAuB,EAAE,MAAM,yCAAyC,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,MAAM,2CAA2C,CAAC;AAEzF,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAE3C,2DAA2D;AAC3D,cAAc,4BAA4B,CAAC;AAG3C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,sDAAsD;AACtD,MAAM,eAAe,GAAG,yBAAyB,EAAE,CAAC;AAEpD,wEAAwE;AACxE,MAAM,sBAAsB,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,KAAK,OAAO,CAAC;AAChF,MAAM,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,YAAY,CAAC;AAClF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB;IACvD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,EAAE,CAAC;IAC3D,CAAC,CAAC,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAElB,qFAAqF;AACrF,IAAI,iBAAiB,GAA6B,IAAI,CAAC;AAEvD,wEAAwE;AACxE,IAAI,mBAAoD,CAAC;AACzD,IAAI,CAAC,uBAAuB,CAAC,oBAAoB,EAAE,EAAE,CAAC;IACpD,+FAA+F;IAC/F,sFAAsF;IACtF,MAAM,CAAC,IAAI,CACT,6FAA6F;QAC3F,wFAAwF;QACxF,gFAAgF,CACnF,CAAC;AACJ,CAAC;KAAM,CAAC;IACN,IAAI,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,0BAA0B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,SAAS,EAAE,CAAC,CAAC;QAEvF,6FAA6F;QAC7F,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,qBAAqB,EAAE,CAAC;QACzE,MAAM,CAAC,KAAK,CACV,iCAAiC,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,YAAY,EAAE,CAAC,EAAE,CACnF,CAAC;QAEF,gFAAgF;QAChF,MAAM,kBAAkB,GAAG;YACzB,iBAAiB,EAAE,OAAO,CAAC,GAAG,CAAC,2BAA2B;gBACxD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,CAAC;gBAC9D,CAAC,CAAC,EAAE,EAAE,kCAAkC;YAC1C,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,6BAA6B;gBACjD,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,6BAA6B,EAAE,EAAE,CAAC;gBAChE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,oBAAoB;SACpC,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;YAC9C,kBAAkB;YAClB,KAAK,EAAE,gBAAgB,CAAC,YAAY,EAAE,CAAC,IAAI;YAC3C,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO;SACxD,CAAC,CAAC;QAEH,MAAM,sBAAsB,GAAG,4BAA4B,CAAC,eAAe,CAAC,CAAC;QAE7E,qEAAqE;QACrE,sEAAsE;QACtE,sCAAsC;QACtC,8DAA8D;QAC9D,MAAM,oBAAoB,GAAI,eAAuB,CAAC,oBAAoB,CAAC;QAC3E,IAAI,OAAO,oBAAoB,KAAK,UAAU,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,sGAAsG,CACvG,CAAC;QACJ,CAAC;QACD,8DAA8D;QAC9D,MAAM,iBAAiB,GAA2B,oBAAoB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC7F,MAAM,QAAQ,GAAG,IAAI,aAAa,CAAC,iBAAiB,CAAC,CAAC;QAEtD,sFAAsF;QACtF,mBAAmB,GAAG,IAAI,mBAAmB;QAC3C,8DAA8D;QAC9D,sBAA6B,EAC7B,gBAAgB,EAChB,kBAAkB,EAClB,IAAI,EAAE,4BAA4B;QAClC,MAAM,EACN,QAAQ,EACR,YAAY,CACb,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,iFAAiF,CAAC,CAAC;IACjG,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,gEAAgE;QAChE,MAAM,CAAC,KAAK,CAAC,0CAA0C,EAAE;YACvD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;YAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SACxD,CAAC,CAAC;QACH,mBAAmB,GAAG,SAAS,CAAC;IAClC,CAAC;AACH,CAAC;AAED,qGAAqG;AACrG,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,CAAC;AAE3C,8GAA8G;AAC9G,MAAM,qBAAqB,GAAG,IAAI,qBAAqB,CAAC;IACtD,eAAe;IACf,mBAAmB;IACnB,QAAQ;IACR,sBAAsB;IACtB,+DAA+D;IAC/D,8DAA8D;IAC9D,kBAAkB,EAAG,eAAuB,CAAC,kBAAkB;CAChE,CAAC,CAAC;AAEH,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAC5B,MAAM,CAAC,IAAI,CACT,+EAA+E;QAC7E,wFAAwF;QACxF,yBAAyB,CAC5B,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,8DAA8D;AAC9D,8DAA8D;AAC9D,MAAM,wBAAwB,GAAG,qBAA4B,CAAC;AAE9D,IACE,wBAAwB,CAAC,eAAe;IACxC,OAAO,wBAAwB,CAAC,eAAe,CAAC,iBAAiB,KAAK,UAAU,EAChF,CAAC;IACD,2DAA2D;IAC3D,wBAAwB,CAAC,eAAe,CAAC,iBAAiB,GAAG,KAAK,EAChE,IAAY;IACZ,8DAA8D;IAC9D,SAAc,EACd,EAAE;QACF,MAAM,CAAC,KAAK,CAAC,qEAAqE,IAAI,EAAE,EAAE;YACxF,aAAa,EAAE,OAAO,SAAS;YAC/B,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW;YACtD,KAAK,EAAE,SAAS,EAAE,KAAK,IAAI,UAAU;SACtC,CAAC,CAAC;QAEH,0CAA0C;QAC1C,MAAM,kBAAkB,GAAG;YACzB,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,SAAS;YACrC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS;YACnC,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;SACjD,CAAC;QAEF,IAAI,OAAO,wBAAwB,CAAC,eAAe,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;YACzF,IAAI,CAAC;gBACH,MAAM,CAAC,KAAK,CACV,wEAAwE,IAAI,EAAE,CAC/E,CAAC;gBACF,OAAO,MAAM,wBAAwB,CAAC,eAAe,CAAC,qBAAqB,CACzE,IAAI,EACJ,kBAAkB,CACnB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,uEAAuE,IAAI,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,QAAQ,GAAG,kFAAkF,IAAI,EAAE,CAAC;YAC1G,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,CAAC,IAAI,CACT,4FAA4F,CAC7F,CAAC;AACJ,CAAC;AAED,4GAA4G;AAC5G,IAAI,qBAAqB,IAAI,OAAO,qBAAqB,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;IACxF,MAAM,sBAAsB,GAAG,qBAAqB,CAAC,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAChG,qBAAqB,CAAC,cAAc,GAAG,KAAK,WAAW,QAAQ;QAC7D,wDAAwD;QACxD,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAEtD,IAAI,mBAAmB,IAAI,sBAAsB,EAAE,CAAC;YAClD,IAAI,CAAC;gBACH,MAAM,CAAC,IAAI,CAAC,6DAA6D,EAAE;oBACzE,WAAW,EAAE,QAAQ,CAAC,MAAM;oBAC5B,WAAW,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;iBAClD,CAAC,CAAC;gBACH,MAAM,mBAAmB,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YACzD,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE;oBAC1D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC;AACJ,CAAC;AAED,kDAAkD;AAClD,MAAM,MAAM,GAAG,WAAW,CAAC,qBAAqB,CAAC,CAAC;AAElD,mCAAmC;AACnC,MAAM,CAAC,KAAK,UAAU,IAAI;IACxB,MAAM,SAAS,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC7C,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;AAClC,CAAC;AAED,iEAAiE;AACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;IACnE,qFAAqF;IACrF,iBAAiB,GAAG,iBAAiB,CAAC,WAAW,EAAE,CAAC;IACpD,iBAAiB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAEpC,8FAA8F;IAC9F,IAAI,mBAAmB,IAAI,sBAAsB,EAAE,CAAC;QAClD,MAAM,0BAA0B,GAAG,MAAM,CAAC,CAAC,wCAAwC;QACnF,WAAW,CAAC,KAAK,IAAI,EAAE;YACrB,IAAI,CAAC;gBACH,iCAAiC;gBACjC,MAAM,mBAAmB,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,4BAA4B;gBAC5B,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE;oBAChD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;iBACxD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAE/B,+FAA+F;QAC/F,2FAA2F;QAC3F,+EAA+E;QAC/E,IAAI,CAAC,QAAQ,CACX,qBAAqB,EACrB,KAAK,IAAI,EAAE;YACT,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE;gBACzD,QAAQ,EAAE,qBAAqB;aAChC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACH,MAAM,cAAc,GAAG,MAAM,mBAAmB,EAAE,+BAA+B,EAAE,CAAC;gBACpF,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;oBAChD,iBAAiB,EAAE,cAAc;oBACjC,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE;oBAC9C,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;oBACvD,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC,EACD;YACE,QAAQ,EAAE,KAAK;SAChB,CACF,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE;YAC7C,kBAAkB,EAAE,SAAS,0BAA0B,IAAI;YAC3D,gBAAgB,EAAE,qBAAqB;YACvC,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;QACnB,wCAAwC;QACxC,MAAM,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Configuration for the neural reranker stage.
|
|
3
|
+
*
|
|
4
|
+
* Disabled by default — reranking is strictly additive. When `enabled` is false
|
|
5
|
+
* (no config, or `RERANK_ENABLED` unset) the search pipeline returns its
|
|
6
|
+
* vector/hybrid ordering unchanged.
|
|
7
|
+
*/
|
|
8
|
+
export interface RerankConfig {
|
|
9
|
+
enabled: boolean;
|
|
10
|
+
/** Full native run URL for the reranker model. */
|
|
11
|
+
endpoint: string;
|
|
12
|
+
model: string;
|
|
13
|
+
apiKey: string;
|
|
14
|
+
/** Candidates sent to the reranker (recall set). */
|
|
15
|
+
topN: number;
|
|
16
|
+
/** Results kept after rerank. */
|
|
17
|
+
topK: number;
|
|
18
|
+
/** Per-passage truncation for scoring only (reranker context limits). */
|
|
19
|
+
maxPassageChars: number;
|
|
20
|
+
timeoutMs: number;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Thin client for a cross-encoder reranker (default: Cloudflare Workers AI
|
|
24
|
+
* `@cf/baai/bge-reranker-base`). OpenAI-style/native HTTP; reuses `axios`.
|
|
25
|
+
*
|
|
26
|
+
* The service itself MAY throw (transport/parse errors) — callers are expected
|
|
27
|
+
* to fail open (return the pre-rerank order). It never enables itself without a
|
|
28
|
+
* resolved endpoint + API key.
|
|
29
|
+
*/
|
|
30
|
+
export declare class RerankerService {
|
|
31
|
+
private readonly cfg;
|
|
32
|
+
constructor(cfg: RerankConfig);
|
|
33
|
+
get enabled(): boolean;
|
|
34
|
+
get topN(): number;
|
|
35
|
+
get topK(): number;
|
|
36
|
+
get model(): string;
|
|
37
|
+
/**
|
|
38
|
+
* Build a reranker from environment variables. Returns a DISABLED instance
|
|
39
|
+
* unless `RERANK_ENABLED=true` AND an endpoint + API key resolve.
|
|
40
|
+
*
|
|
41
|
+
* Env: RERANK_ENABLED, RERANK_MODEL (default @cf/baai/bge-reranker-base),
|
|
42
|
+
* RERANK_ENDPOINT (or derived from RERANK_ACCOUNT_ID/CF_ACCOUNT_ID + model),
|
|
43
|
+
* RERANK_API_KEY (falls back to EMBEDDING_API_KEY), RERANK_TOP_N (20),
|
|
44
|
+
* RERANK_TOP_K (10), RERANK_MAX_PASSAGE_CHARS (2000), RERANK_TIMEOUT_MS (5000).
|
|
45
|
+
*/
|
|
46
|
+
static fromEnv(): RerankerService;
|
|
47
|
+
/**
|
|
48
|
+
* Rerank `passages` against `query`. Returns candidate indices reordered
|
|
49
|
+
* descending by relevance, length <= topK. THROWS on transport/parse error —
|
|
50
|
+
* the caller fail-opens.
|
|
51
|
+
*
|
|
52
|
+
* @param query - Search query
|
|
53
|
+
* @param passages - Candidate passages (index-aligned with the caller's results)
|
|
54
|
+
* @returns Indices into `passages`, best-first
|
|
55
|
+
*/
|
|
56
|
+
rerank(query: string, passages: string[]): Promise<number[]>;
|
|
57
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import axios from 'axios';
|
|
2
|
+
import { logger } from '../utils/logger.js';
|
|
3
|
+
function intEnv(name, fallback) {
|
|
4
|
+
const raw = process.env[name];
|
|
5
|
+
if (!raw)
|
|
6
|
+
return fallback;
|
|
7
|
+
const parsed = Number.parseInt(raw, 10);
|
|
8
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Thin client for a cross-encoder reranker (default: Cloudflare Workers AI
|
|
12
|
+
* `@cf/baai/bge-reranker-base`). OpenAI-style/native HTTP; reuses `axios`.
|
|
13
|
+
*
|
|
14
|
+
* The service itself MAY throw (transport/parse errors) — callers are expected
|
|
15
|
+
* to fail open (return the pre-rerank order). It never enables itself without a
|
|
16
|
+
* resolved endpoint + API key.
|
|
17
|
+
*/
|
|
18
|
+
export class RerankerService {
|
|
19
|
+
cfg;
|
|
20
|
+
constructor(cfg) {
|
|
21
|
+
this.cfg = cfg;
|
|
22
|
+
}
|
|
23
|
+
get enabled() {
|
|
24
|
+
return this.cfg.enabled;
|
|
25
|
+
}
|
|
26
|
+
get topN() {
|
|
27
|
+
return this.cfg.topN;
|
|
28
|
+
}
|
|
29
|
+
get topK() {
|
|
30
|
+
return this.cfg.topK;
|
|
31
|
+
}
|
|
32
|
+
get model() {
|
|
33
|
+
return this.cfg.model;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Build a reranker from environment variables. Returns a DISABLED instance
|
|
37
|
+
* unless `RERANK_ENABLED=true` AND an endpoint + API key resolve.
|
|
38
|
+
*
|
|
39
|
+
* Env: RERANK_ENABLED, RERANK_MODEL (default @cf/baai/bge-reranker-base),
|
|
40
|
+
* RERANK_ENDPOINT (or derived from RERANK_ACCOUNT_ID/CF_ACCOUNT_ID + model),
|
|
41
|
+
* RERANK_API_KEY (falls back to EMBEDDING_API_KEY), RERANK_TOP_N (20),
|
|
42
|
+
* RERANK_TOP_K (10), RERANK_MAX_PASSAGE_CHARS (2000), RERANK_TIMEOUT_MS (5000).
|
|
43
|
+
*/
|
|
44
|
+
static fromEnv() {
|
|
45
|
+
const requested = process.env.RERANK_ENABLED === 'true';
|
|
46
|
+
const model = process.env.RERANK_MODEL || '@cf/baai/bge-reranker-base';
|
|
47
|
+
const apiKey = process.env.RERANK_API_KEY || process.env.EMBEDDING_API_KEY || '';
|
|
48
|
+
let endpoint = process.env.RERANK_ENDPOINT || '';
|
|
49
|
+
const accountId = process.env.RERANK_ACCOUNT_ID || process.env.CF_ACCOUNT_ID;
|
|
50
|
+
if (!endpoint && accountId) {
|
|
51
|
+
endpoint = `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${model}`;
|
|
52
|
+
}
|
|
53
|
+
const enabled = requested && endpoint.length > 0 && apiKey.length > 0;
|
|
54
|
+
if (requested && !enabled) {
|
|
55
|
+
logger.warn('RerankerService: RERANK_ENABLED=true but endpoint or API key could not be resolved — ' +
|
|
56
|
+
'reranking DISABLED (fail-open). Set RERANK_ENDPOINT (or RERANK_ACCOUNT_ID) and RERANK_API_KEY.');
|
|
57
|
+
}
|
|
58
|
+
else if (enabled) {
|
|
59
|
+
logger.info(`RerankerService: enabled (model ${model})`);
|
|
60
|
+
}
|
|
61
|
+
return new RerankerService({
|
|
62
|
+
enabled,
|
|
63
|
+
endpoint,
|
|
64
|
+
model,
|
|
65
|
+
apiKey,
|
|
66
|
+
topN: intEnv('RERANK_TOP_N', 20),
|
|
67
|
+
topK: intEnv('RERANK_TOP_K', 10),
|
|
68
|
+
maxPassageChars: intEnv('RERANK_MAX_PASSAGE_CHARS', 2000),
|
|
69
|
+
timeoutMs: intEnv('RERANK_TIMEOUT_MS', 5000),
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Rerank `passages` against `query`. Returns candidate indices reordered
|
|
74
|
+
* descending by relevance, length <= topK. THROWS on transport/parse error —
|
|
75
|
+
* the caller fail-opens.
|
|
76
|
+
*
|
|
77
|
+
* @param query - Search query
|
|
78
|
+
* @param passages - Candidate passages (index-aligned with the caller's results)
|
|
79
|
+
* @returns Indices into `passages`, best-first
|
|
80
|
+
*/
|
|
81
|
+
async rerank(query, passages) {
|
|
82
|
+
const contexts = passages
|
|
83
|
+
.slice(0, this.cfg.topN)
|
|
84
|
+
.map(text => ({ text: (text || '').slice(0, this.cfg.maxPassageChars) }));
|
|
85
|
+
const response = await axios.post(this.cfg.endpoint, { query: query.slice(0, this.cfg.maxPassageChars), contexts }, {
|
|
86
|
+
headers: {
|
|
87
|
+
Authorization: `Bearer ${this.cfg.apiKey}`,
|
|
88
|
+
'Content-Type': 'application/json',
|
|
89
|
+
},
|
|
90
|
+
timeout: this.cfg.timeoutMs,
|
|
91
|
+
});
|
|
92
|
+
const ranked = response.data?.result?.response;
|
|
93
|
+
if (!Array.isArray(ranked)) {
|
|
94
|
+
throw new Error('Reranker returned a malformed response (missing result.response array)');
|
|
95
|
+
}
|
|
96
|
+
// Defensive: keep only integer, in-range, UNIQUE ids (CF returns request-array indices).
|
|
97
|
+
// Fractional / duplicate / out-of-range ids are dropped so a malformed-but-arrayed response
|
|
98
|
+
// can't corrupt ordering or duplicate results — the caller still fully fail-opens on a throw.
|
|
99
|
+
const seen = new Set();
|
|
100
|
+
const indices = [];
|
|
101
|
+
for (const row of ranked) {
|
|
102
|
+
const id = row?.id;
|
|
103
|
+
if (typeof id === 'number' &&
|
|
104
|
+
Number.isInteger(id) &&
|
|
105
|
+
id >= 0 &&
|
|
106
|
+
id < contexts.length &&
|
|
107
|
+
!seen.has(id)) {
|
|
108
|
+
seen.add(id);
|
|
109
|
+
indices.push(id);
|
|
110
|
+
if (indices.length >= this.cfg.topK)
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return indices;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=RerankerService.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RerankerService.js","sourceRoot":"","sources":["../../src/retrieval/RerankerService.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AA6B5C,SAAS,MAAM,CAAC,IAAY,EAAE,QAAgB;IAC5C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,CAAC,GAAG;QAAE,OAAO,QAAQ,CAAC;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AACnE,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,eAAe;IACT,GAAG,CAAe;IAEnC,YAAY,GAAiB;QAC3B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;IAC1B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;IACvB,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;IACxB,CAAC;IAED;;;;;;;;OAQG;IACH,MAAM,CAAC,OAAO;QACZ,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,MAAM,CAAC;QACxD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,4BAA4B,CAAC;QACvE,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;QACjF,IAAI,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;QAC7E,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC3B,QAAQ,GAAG,iDAAiD,SAAS,WAAW,KAAK,EAAE,CAAC;QAC1F,CAAC;QAED,MAAM,OAAO,GAAG,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACtE,IAAI,SAAS,IAAI,CAAC,OAAO,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CACT,uFAAuF;gBACrF,gGAAgG,CACnG,CAAC;QACJ,CAAC;aAAM,IAAI,OAAO,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,mCAAmC,KAAK,GAAG,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,IAAI,eAAe,CAAC;YACzB,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,IAAI,EAAE,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,IAAI,EAAE,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC;YAChC,eAAe,EAAE,MAAM,CAAC,0BAA0B,EAAE,IAAI,CAAC;YACzD,SAAS,EAAE,MAAM,CAAC,mBAAmB,EAAE,IAAI,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAkB;QAC5C,MAAM,QAAQ,GAAG,QAAQ;aACtB,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;aACvB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5E,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAC/B,IAAI,CAAC,GAAG,CAAC,QAAQ,EACjB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE,EAC7D;YACE,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE;gBAC1C,cAAc,EAAE,kBAAkB;aACnC;YACD,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS;SAC5B,CACF,CAAC;QAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC;QAC/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC5F,CAAC;QAED,yFAAyF;QACzF,4FAA4F;QAC5F,8FAA8F;QAC9F,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,GAAG,EAAE,EAAE,CAAC;YACnB,IACE,OAAO,EAAE,KAAK,QAAQ;gBACtB,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACpB,EAAE,IAAI,CAAC;gBACP,EAAE,GAAG,QAAQ,CAAC,MAAM;gBACpB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EACb,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACb,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACjB,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI;oBAAE,MAAM;YAC7C,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -22,6 +22,18 @@ export function createAdaptedStorageProvider(storageProvider) {
|
|
|
22
22
|
}
|
|
23
23
|
throw new Error('Underlying storage provider has no loadGraph method');
|
|
24
24
|
},
|
|
25
|
+
// v2.4.1+: prefer the dedicated Cypher predicate over a full loadGraph
|
|
26
|
+
// when scheduling incremental regeneration. Falls through to undefined if
|
|
27
|
+
// the underlying provider doesn't expose this method, in which case the
|
|
28
|
+
// manager falls back to the loadGraph-and-filter approach.
|
|
29
|
+
getEntityNamesMissingEmbeddings: async () => {
|
|
30
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
31
|
+
const fn = storageProvider.getEntityNamesMissingEmbeddings;
|
|
32
|
+
if (typeof fn !== 'function') {
|
|
33
|
+
throw new Error('Underlying storage provider has no getEntityNamesMissingEmbeddings method');
|
|
34
|
+
}
|
|
35
|
+
return fn.call(storageProvider);
|
|
36
|
+
},
|
|
25
37
|
getEntity: async (name) => {
|
|
26
38
|
if (typeof storageProvider.getEntity === 'function') {
|
|
27
39
|
return storageProvider.getEntity(name);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createAdaptedStorageProvider.js","sourceRoot":"","sources":["../../src/storage/createAdaptedStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C;;;;;;;;;;;;GAYG;AACH,8DAA8D;AAC9D,MAAM,UAAU,4BAA4B,CAAC,eAAgC;IAC3E,OAAO;QACL,GAAG,eAAe;QAElB,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,IAAI,OAAO,eAAe,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpD,OAAO,eAAe,CAAC,SAAS,EAAE,CAAC;YACrC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,SAAS,EAAE,KAAK,EAAE,IAAY,EAAE,EAAE;YAChC,IAAI,OAAO,eAAe,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpD,OAAO,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACpC,CAAC;QAED,8DAA8D;QAC9D,iBAAiB,EAAE,KAAK,EAAE,IAAY,EAAE,SAAc,EAAE,EAAE;YACxD,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,EAAE,EAAE;gBAClE,aAAa,EAAE,OAAO,SAAS;gBAC/B,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW;gBACtD,KAAK,EAAE,SAAS,EAAE,KAAK,IAAI,UAAU;aACtC,CAAC,CAAC;YAEH,MAAM,kBAAkB,GAAG;gBACzB,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,SAAS;gBACrC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS;gBACnC,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;aACjD,CAAC;YAEF,8DAA8D;YAC9D,IAAI,OAAQ,eAAuB,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;gBACzE,IAAI,CAAC;oBACH,8DAA8D;oBAC9D,OAAO,MAAO,eAAuB,CAAC,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBACxF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC7E,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YACD,MAAM,QAAQ,GAAG,4DAA4D,IAAI,EAAE,CAAC;YACpF,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
1
|
+
{"version":3,"file":"createAdaptedStorageProvider.js","sourceRoot":"","sources":["../../src/storage/createAdaptedStorageProvider.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C;;;;;;;;;;;;GAYG;AACH,8DAA8D;AAC9D,MAAM,UAAU,4BAA4B,CAAC,eAAgC;IAC3E,OAAO;QACL,GAAG,eAAe;QAElB,SAAS,EAAE,KAAK,IAAI,EAAE;YACpB,IAAI,OAAO,eAAe,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpD,OAAO,eAAe,CAAC,SAAS,EAAE,CAAC;YACrC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;QAED,uEAAuE;QACvE,0EAA0E;QAC1E,wEAAwE;QACxE,2DAA2D;QAC3D,+BAA+B,EAAE,KAAK,IAAuB,EAAE;YAC7D,8DAA8D;YAC9D,MAAM,EAAE,GAAI,eAAuB,CAAC,+BAA+B,CAAC;YACpE,IAAI,OAAO,EAAE,KAAK,UAAU,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClC,CAAC;QAED,SAAS,EAAE,KAAK,EAAE,IAAY,EAAE,EAAE;YAChC,IAAI,OAAO,eAAe,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;gBACpD,OAAO,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzC,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACpC,CAAC;QAED,8DAA8D;QAC9D,iBAAiB,EAAE,KAAK,EAAE,IAAY,EAAE,SAAc,EAAE,EAAE;YACxD,MAAM,CAAC,KAAK,CAAC,+CAA+C,IAAI,EAAE,EAAE;gBAClE,aAAa,EAAE,OAAO,SAAS;gBAC/B,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW;gBACtD,KAAK,EAAE,SAAS,EAAE,KAAK,IAAI,UAAU;aACtC,CAAC,CAAC;YAEH,MAAM,kBAAkB,GAAG;gBACzB,MAAM,EAAE,SAAS,CAAC,MAAM,IAAI,SAAS;gBACrC,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,SAAS;gBACnC,WAAW,EAAE,SAAS,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;aACjD,CAAC;YAEF,8DAA8D;YAC9D,IAAI,OAAQ,eAAuB,CAAC,qBAAqB,KAAK,UAAU,EAAE,CAAC;gBACzE,IAAI,CAAC;oBACH,8DAA8D;oBAC9D,OAAO,MAAO,eAAuB,CAAC,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBACxF,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,iDAAiD,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;oBAC7E,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YACD,MAAM,QAAQ,GAAG,4DAA4D,IAAI,EAAE,CAAC;YACpF,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -224,6 +224,19 @@ export declare class Neo4jStorageProvider implements StorageProvider {
|
|
|
224
224
|
* based on their age and the configured decay settings
|
|
225
225
|
*/
|
|
226
226
|
getDecayedGraph(): Promise<KnowledgeGraph>;
|
|
227
|
+
/**
|
|
228
|
+
* Return the names of currently-valid entities that have no embedding.
|
|
229
|
+
*
|
|
230
|
+
* Direct Cypher predicate (`e.embedding IS NULL`) so the database does the
|
|
231
|
+
* filtering instead of materialising every entity into memory and inspecting
|
|
232
|
+
* a stripped-down JS object — `loadGraph()` discards the `embedding` field
|
|
233
|
+
* via `nodeToEntity`, so consumers that filtered on `entity.embedding` were
|
|
234
|
+
* always seeing 100% of entities as "missing". This method is the right
|
|
235
|
+
* primitive for `EmbeddingJobManager.scheduleIncrementalRegeneration`.
|
|
236
|
+
*
|
|
237
|
+
* @returns Array of entity names that need an embedding job scheduled.
|
|
238
|
+
*/
|
|
239
|
+
getEntityNamesMissingEmbeddings(): Promise<string[]>;
|
|
227
240
|
/**
|
|
228
241
|
* Store or update the embedding vector for an entity
|
|
229
242
|
* @param entityName The name of the entity to update
|
|
@@ -1662,6 +1662,26 @@ export class Neo4jStorageProvider {
|
|
|
1662
1662
|
throw error;
|
|
1663
1663
|
}
|
|
1664
1664
|
}
|
|
1665
|
+
/**
|
|
1666
|
+
* Return the names of currently-valid entities that have no embedding.
|
|
1667
|
+
*
|
|
1668
|
+
* Direct Cypher predicate (`e.embedding IS NULL`) so the database does the
|
|
1669
|
+
* filtering instead of materialising every entity into memory and inspecting
|
|
1670
|
+
* a stripped-down JS object — `loadGraph()` discards the `embedding` field
|
|
1671
|
+
* via `nodeToEntity`, so consumers that filtered on `entity.embedding` were
|
|
1672
|
+
* always seeing 100% of entities as "missing". This method is the right
|
|
1673
|
+
* primitive for `EmbeddingJobManager.scheduleIncrementalRegeneration`.
|
|
1674
|
+
*
|
|
1675
|
+
* @returns Array of entity names that need an embedding job scheduled.
|
|
1676
|
+
*/
|
|
1677
|
+
async getEntityNamesMissingEmbeddings() {
|
|
1678
|
+
const result = await this.connectionManager.executeQuery(`
|
|
1679
|
+
MATCH (e:Entity)
|
|
1680
|
+
WHERE e.validTo IS NULL AND e.embedding IS NULL
|
|
1681
|
+
RETURN e.name AS name
|
|
1682
|
+
`, {});
|
|
1683
|
+
return result.records.map(r => String(r.get('name')));
|
|
1684
|
+
}
|
|
1665
1685
|
/**
|
|
1666
1686
|
* Store or update the embedding vector for an entity
|
|
1667
1687
|
* @param entityName The name of the entity to update
|