@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. package/.ragrc.schema.json +83 -192
  2. package/CHANGELOG.md +345 -0
  3. package/README.md +418 -117
  4. package/SECURITY.md +5 -5
  5. package/dist/experimental.cjs +2 -0
  6. package/dist/experimental.d.ts +1 -0
  7. package/dist/experimental.mjs +5 -0
  8. package/dist/index.cjs +4992 -1242
  9. package/dist/index.d.ts +80 -2
  10. package/dist/index.mjs +1 -1
  11. package/dist/types/agentic/agentic-pipeline.d.ts +103 -0
  12. package/dist/types/agentic/iterative-retriever.d.ts +85 -0
  13. package/dist/types/agentic/query-planner.d.ts +145 -0
  14. package/dist/types/agentic/self-critique.d.ts +45 -0
  15. package/dist/types/ai/adaptive-retrieval.d.ts +44 -0
  16. package/dist/types/ai/federation/federated-learning-coordinator.d.ts +62 -0
  17. package/dist/types/ai/multimodal-processing.d.ts +32 -0
  18. package/dist/types/ai/multimodal.d.ts +39 -0
  19. package/dist/types/ai/retrieval-engine.d.ts +37 -0
  20. package/dist/types/ai/training/model-training-orchestrator.d.ts +31 -0
  21. package/dist/types/chunking/chunking-engine.d.ts +70 -0
  22. package/dist/types/chunking/strategies/fixed-size.d.ts +17 -0
  23. package/dist/types/chunking/strategies/recursive.d.ts +20 -0
  24. package/dist/types/chunking/strategies/semantic.d.ts +19 -0
  25. package/dist/types/chunking/strategies/sentence.d.ts +13 -0
  26. package/dist/types/chunking/strategies/structure-aware.d.ts +19 -0
  27. package/dist/types/citation/citation-tracker.d.ts +64 -0
  28. package/dist/types/citation/hallucination-detector.d.ts +40 -0
  29. package/dist/types/citation/source-mapper.d.ts +37 -0
  30. package/dist/types/connectors/anthropic-connector.d.ts +92 -0
  31. package/dist/types/connectors/base-connector.d.ts +87 -0
  32. package/dist/types/connectors/cohere-connector.d.ts +89 -0
  33. package/dist/types/connectors/connector-utils.d.ts +110 -0
  34. package/dist/types/connectors/local-embedder.d.ts +86 -0
  35. package/dist/types/connectors/memory-retriever.d.ts +91 -0
  36. package/dist/types/connectors/ollama-connector.d.ts +110 -0
  37. package/dist/types/connectors/openai-connector.d.ts +114 -0
  38. package/dist/types/connectors/rag-prompt.d.ts +40 -0
  39. package/dist/types/cost/cost-calculator.d.ts +152 -0
  40. package/dist/types/cost/cost-tracker.d.ts +126 -0
  41. package/dist/types/cost/token-budget.d.ts +110 -0
  42. package/dist/types/dag/core/error-context.d.ts +239 -0
  43. package/dist/types/dag/core/execution-scheduler.d.ts +191 -0
  44. package/dist/types/dag/core/topology-validation.d.ts +226 -0
  45. package/dist/types/dag/dag-engine.d.ts +156 -0
  46. package/dist/types/dag/dag-node.d.ts +37 -0
  47. package/dist/types/dashboard/dashboard-generator.d.ts +127 -0
  48. package/dist/types/dashboard/metrics-aggregator.d.ts +218 -0
  49. package/dist/types/debug/execution-tracer.d.ts +57 -0
  50. package/dist/types/debug/trace-inspector.d.ts +31 -0
  51. package/dist/types/ecosystem/plugin-certification.d.ts +169 -0
  52. package/dist/types/enterprise/data-governance.d.ts +203 -0
  53. package/dist/types/enterprise/multi-tenancy.d.ts +137 -0
  54. package/dist/types/enterprise/sso-integration.d.ts +420 -0
  55. package/dist/types/evaluate/context-metrics.d.ts +55 -0
  56. package/dist/types/evaluate/faithfulness.d.ts +41 -0
  57. package/dist/types/evaluate/groundedness.d.ts +60 -0
  58. package/dist/types/evaluate/pipeline-evaluator.d.ts +88 -0
  59. package/dist/types/evaluate/relevance.d.ts +21 -0
  60. package/dist/types/evaluate/scoring.d.ts +123 -0
  61. package/dist/types/experimental.d.ts +15 -0
  62. package/dist/types/graph/entity-extractor.d.ts +84 -0
  63. package/dist/types/graph/graph-index.d.ts +49 -0
  64. package/dist/types/graph/graph-retriever.d.ts +81 -0
  65. package/dist/types/graph/knowledge-graph.d.ts +129 -0
  66. package/dist/types/guardrails/guardrails-pipeline.d.ts +63 -0
  67. package/dist/types/guardrails/injection-detector.d.ts +47 -0
  68. package/dist/types/guardrails/post-generation-guard.d.ts +61 -0
  69. package/dist/types/guardrails/pre-retrieval-guard.d.ts +64 -0
  70. package/dist/types/guardrails/retrieval-guard.d.ts +57 -0
  71. package/dist/types/mcp/mcp-server.d.ts +85 -0
  72. package/dist/types/mcp/mcp-tool-builder.d.ts +55 -0
  73. package/dist/types/reranker/cascade-reranker.d.ts +40 -0
  74. package/dist/types/reranker/embedding-reranker.d.ts +77 -0
  75. package/dist/types/reranker/scoring-reranker.d.ts +41 -0
  76. package/dist/types/retrieval/bm25-search.d.ts +43 -0
  77. package/dist/types/retrieval/file-vector-store.d.ts +87 -0
  78. package/dist/types/retrieval/hybrid-retriever.d.ts +35 -0
  79. package/dist/types/retrieval/rank-fusion.d.ts +10 -0
  80. package/dist/types/security/jwt-config.d.ts +80 -0
  81. package/dist/types/security/jwt-validator.d.ts +195 -0
  82. package/dist/types/security/plugin-sandbox.d.ts +158 -0
  83. package/dist/types/security/plugin-signature-verifier.d.ts +139 -0
  84. package/dist/types/security/validator.d.ts +5 -0
  85. package/dist/types/streaming/embedding-stream.d.ts +65 -0
  86. package/dist/types/streaming/sse-adapter.d.ts +59 -0
  87. package/dist/types/streaming/stream-router.d.ts +88 -0
  88. package/dist/types/streaming/streaming-embedder.d.ts +99 -0
  89. package/dist/types/streaming/websocket-adapter.d.ts +71 -0
  90. package/dist/types/templates/scaffold.d.ts +55 -0
  91. package/dist/types/templates/template-registry.d.ts +63 -0
  92. package/dist/types/utils/embed-query.d.ts +20 -0
  93. package/dist/types/utils/logger.d.ts +50 -0
  94. package/dist/types/utils/rate-limiter.d.ts +66 -0
  95. package/dist/types/utils/retry.d.ts +27 -0
  96. package/dist/types/utils/run-abort.d.ts +34 -0
  97. package/dist/types/utils/secure-logging.d.ts +51 -0
  98. package/dist/types/utils/structured-logger.d.ts +126 -0
  99. package/package.json +53 -29
  100. package/src/agentic/agentic-pipeline.js +75 -7
  101. package/src/agentic/iterative-retriever.js +47 -10
  102. package/src/agentic/query-planner.js +329 -78
  103. package/src/citation/citation-tracker.js +6 -4
  104. package/src/citation/hallucination-detector.js +2 -2
  105. package/src/citation/source-mapper.js +15 -28
  106. package/src/cli/commands/ai.js +101 -0
  107. package/src/cli/commands/plugin-hub/event-handlers.js +6 -0
  108. package/src/cli/commands/plugin-hub/hub-commands.js +0 -74
  109. package/src/cli/commands/plugin-hub/local-commands.js +143 -0
  110. package/src/cli/commands/plugin-hub.js +96 -36
  111. package/src/cli/diagnostics/config-checker.js +76 -63
  112. package/src/cli/diagnostics/environment-checker.js +62 -31
  113. package/src/cli/diagnostics/plugin-checker.js +36 -89
  114. package/src/cli/doctor-command.js +39 -5
  115. package/src/cli/enhanced-cli-commands.js +72 -217
  116. package/src/cli/handlers/ai-handler.js +364 -0
  117. package/src/cli/handlers/diagnostics-handler.js +78 -31
  118. package/src/cli/handlers/ingest-handler.js +28 -22
  119. package/src/cli/handlers/init-handler.js +148 -25
  120. package/src/cli/handlers/query-handler.js +41 -26
  121. package/src/cli/handlers/run-handler.js +16 -167
  122. package/src/cli/handlers/serve-handler.js +60 -0
  123. package/src/cli/services/bench-stats.js +54 -0
  124. package/src/cli/services/benchmark.js +161 -0
  125. package/src/cli/services/feedback.js +155 -0
  126. package/src/cli/services/rag-service.js +322 -0
  127. package/src/cli/shared/cli-utils.js +24 -7
  128. package/src/cli/shared/config-context.js +21 -0
  129. package/src/cli/wizard/config-builder.js +81 -0
  130. package/src/cli/wizard/init-wizard.js +617 -0
  131. package/src/config/component-factory.js +125 -0
  132. package/src/config/load-config.js +29 -10
  133. package/src/config/load-plugin-config.js +46 -2
  134. package/src/connectors/anthropic-connector.js +93 -55
  135. package/src/connectors/base-connector.js +74 -19
  136. package/src/connectors/cohere-connector.js +88 -32
  137. package/src/connectors/connector-utils.js +302 -0
  138. package/src/connectors/local-embedder.js +43 -5
  139. package/src/connectors/memory-retriever.js +108 -15
  140. package/src/connectors/ollama-connector.js +83 -49
  141. package/src/connectors/openai-connector.js +108 -44
  142. package/src/connectors/rag-prompt.js +88 -0
  143. package/src/core/create-pipeline.js +209 -60
  144. package/src/core/plugin-registry.js +87 -11
  145. package/src/cost/cost-calculator.js +205 -32
  146. package/src/dev/dev-server.js +166 -11
  147. package/src/dev/hot-reload.js +40 -1
  148. package/src/ecosystem/plugin-checks.js +181 -0
  149. package/src/ecosystem/plugin-hub.js +50 -125
  150. package/src/ecosystem/plugin-installer.js +298 -0
  151. package/src/ecosystem/plugin-probe.js +127 -0
  152. package/src/ecosystem/plugin-scanner.js +185 -0
  153. package/src/ecosystem/tar-extract.js +114 -0
  154. package/src/enterprise/audit-logging.js +158 -29
  155. package/src/enterprise/sso-integration.js +715 -228
  156. package/src/evaluate/context-metrics.js +28 -16
  157. package/src/evaluate/evaluator.js +3 -3
  158. package/src/evaluate/faithfulness.js +18 -14
  159. package/src/evaluate/groundedness.js +43 -12
  160. package/src/evaluate/pipeline-evaluator.js +35 -8
  161. package/src/evaluate/scoring.js +251 -3
  162. package/src/experimental.js +71 -0
  163. package/src/guardrails/guardrails-pipeline.js +69 -7
  164. package/src/guardrails/injection-detector.js +283 -0
  165. package/src/guardrails/post-generation-guard.js +19 -0
  166. package/src/guardrails/pre-retrieval-guard.js +36 -36
  167. package/src/guardrails/retrieval-guard.js +58 -8
  168. package/src/index.js +36 -1
  169. package/src/loader/document-reader.js +91 -0
  170. package/src/reranker/cascade-reranker.js +3 -0
  171. package/src/reranker/embedding-reranker.js +136 -21
  172. package/src/reranker/llm-reranker.js +79 -12
  173. package/src/reranker/scoring-reranker.js +3 -0
  174. package/src/retrieval/bm25-search.js +12 -6
  175. package/src/retrieval/file-vector-store.js +212 -0
  176. package/src/retrieval/hybrid-retriever.js +22 -19
  177. package/src/security/jwt-config.js +13 -2
  178. package/src/security/jwt-validator.js +134 -30
  179. package/src/security/plugin-sandbox.js +18 -1
  180. package/src/security/plugin-signature-verifier.js +245 -70
  181. package/src/utils/deprecate.js +69 -0
  182. package/src/utils/embed-query.js +33 -0
  183. package/src/utils/input-sanitizer.js +19 -4
  184. package/src/utils/logger.js +34 -4
  185. package/src/utils/run-abort.js +102 -0
  186. package/src/ai/federated-learning.js +0 -211
  187. package/src/ai/federation.js +0 -71
  188. package/src/ai/index.js +0 -22
  189. package/src/ai/model-trainer.js +0 -217
  190. package/src/ai/model-training.js +0 -202
  191. package/src/ai/multimodal/multi-modal-processor.js +0 -345
  192. package/src/ai/orchestrator.js +0 -65
  193. package/src/ai/retrieval/adaptive-retrieval-engine.js +0 -398
  194. package/src/cli/commands/ai-ml/adaptive-commands.js +0 -178
  195. package/src/cli/commands/ai-ml/federated-commands.js +0 -207
  196. package/src/cli/commands/ai-ml/index.js +0 -17
  197. package/src/cli/commands/ai-ml/multimodal-commands.js +0 -187
  198. package/src/cli/commands/ai-ml/shared.js +0 -51
  199. package/src/cli/commands/ai-ml/training-commands.js +0 -228
  200. package/src/cli/commands/ai-ml.js +0 -386
  201. package/src/cli/commands/docs.js +0 -228
  202. package/src/cli/commands/dx.js +0 -538
  203. package/src/cli/commands/wizard.js +0 -627
  204. package/src/cli/input-sanitization-middleware.js +0 -356
  205. package/src/cli/interactive-wizard.js +0 -373
  206. package/src/cli/templates/development.template.json +0 -64
  207. package/src/cli/templates/minimal.template.json +0 -33
  208. package/src/cli/templates/production.template.json +0 -104
  209. package/src/cli/templates/testing.template.json +0 -45
  210. package/src/cli/wizard/configuration.js +0 -148
  211. package/src/cli/wizard/observability.js +0 -79
  212. package/src/cli/wizard/performance.js +0 -76
  213. package/src/cli/wizard/plugin-selection.js +0 -297
  214. package/src/cli/wizard/preview-save.js +0 -172
  215. package/src/cli/wizard/project-setup.js +0 -66
  216. package/src/core/pipeline-factory.js +0 -59
  217. package/src/dag/enhanced-dag-engine.js +0 -894
  218. package/src/dx/index.js +0 -40
  219. package/src/dx/integration-templates.js +0 -662
  220. package/src/dx/performance-profiler.js +0 -968
  221. package/src/dx/realtime-debugger.js +0 -755
  222. package/src/dx/visual-pipeline-builder.js +0 -578
  223. package/src/ingest.js +0 -59
  224. package/src/pipeline/pipeline.js +0 -18
  225. package/src/query.js +0 -117
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Pre-retrieval guard that validates and sanitizes user queries before
3
+ * they reach the retrieval layer. Checks for injection attacks, topic
4
+ * violations, and input length constraints.
5
+ *
6
+ * @extends EventEmitter
7
+ */
8
+ export class PreRetrievalGuard extends EventEmitter<[never]> {
9
+ /**
10
+ * @param {object} [options] - Override default configuration
11
+ * @param {number} [options.maxQueryLength=10000] - Maximum allowed query length
12
+ * @param {boolean} [options.enableInjectionDetection=true] - Whether to check for prompt injection
13
+ * @param {boolean} [options.enableTopicFiltering=false] - Whether to enforce topic constraints
14
+ * @param {string[]} [options.blockedTopics=[]] - Topics to reject
15
+ * @param {string[]} [options.allowedTopics=[]] - If non-empty, only these topics are permitted
16
+ * @param {RegExp[]} [options.injectionPatterns] - Additional injection patterns (merged with defaults)
17
+ */
18
+ constructor(options?: {
19
+ maxQueryLength?: number;
20
+ enableInjectionDetection?: boolean;
21
+ enableTopicFiltering?: boolean;
22
+ blockedTopics?: string[];
23
+ allowedTopics?: string[];
24
+ injectionPatterns?: RegExp[];
25
+ });
26
+ config: any;
27
+ /** @private Extra patterns supplied by the caller */
28
+ private _customPatterns;
29
+ /**
30
+ * Check a query for safety issues, injection attempts, and topic violations.
31
+ *
32
+ * @param {string} query - The user query to validate
33
+ * @returns {{
34
+ * safe: boolean,
35
+ * issues: string[],
36
+ * sanitizedQuery: string,
37
+ * injection: { detected: boolean, matches: Array<{id: string, pattern: string, match: string}> }
38
+ * }}
39
+ * `sanitizedQuery` is the query truncated to maxQueryLength; it is never
40
+ * rewritten. Injection findings are reported in `issues` and `injection`
41
+ * and make `safe` false; blocking is up to the caller (GuardrailsPipeline
42
+ * blocks in strict mode).
43
+ * @fires PreRetrievalGuard#checked
44
+ */
45
+ check(query: string): {
46
+ safe: boolean;
47
+ issues: string[];
48
+ sanitizedQuery: string;
49
+ injection: {
50
+ detected: boolean;
51
+ matches: Array<{
52
+ id: string;
53
+ pattern: string;
54
+ match: string;
55
+ }>;
56
+ };
57
+ };
58
+ }
59
+ /**
60
+ * Default configuration for the PreRetrievalGuard.
61
+ * @type {object}
62
+ */
63
+ export const DEFAULT_CONFIG: object;
64
+ import { EventEmitter } from "events";
@@ -0,0 +1,57 @@
1
+ /**
2
+ * Retrieval guard that filters search results based on relevance score,
3
+ * document freshness, and access control lists. Applied between the
4
+ * retrieval and generation stages of the pipeline.
5
+ *
6
+ * @extends EventEmitter
7
+ */
8
+ export class RetrievalGuard extends EventEmitter<[never]> {
9
+ /**
10
+ * @param {object} [options] - Override default configuration
11
+ * @param {number} [options.minRelevanceScore=0] - Minimum relevance score threshold
12
+ * @param {number} [options.maxAgeDays=0] - Maximum document age in days (0 = disabled)
13
+ * @param {boolean} [options.enforceACL=false] - Whether to enforce access control
14
+ * @param {boolean} [options.detectInjection=false] - Scan document text for prompt injection
15
+ * @param {'remove'|'flag'} [options.injectionAction='remove'] - Remove flagged documents or keep and mark them
16
+ * @param {RegExp[]} [options.injectionPatterns=[]] - Extra injection patterns for the scan
17
+ */
18
+ constructor(options?: {
19
+ minRelevanceScore?: number;
20
+ maxAgeDays?: number;
21
+ enforceACL?: boolean;
22
+ detectInjection?: boolean;
23
+ injectionAction?: "remove" | "flag";
24
+ injectionPatterns?: RegExp[];
25
+ });
26
+ config: any;
27
+ /**
28
+ * Filter retrieval results based on configured guards.
29
+ *
30
+ * @param {Array<object>} results - Array of retrieval results, each with .score and optional .metadata
31
+ * @param {object} [context={}] - Request context
32
+ * @param {string[]} [context.userPermissions] - User's permission roles for ACL filtering
33
+ * @returns {{
34
+ * results: Array<object>,
35
+ * removed: Array<object>,
36
+ * metadata: { reasons: Object<string, number> }
37
+ * }}
38
+ * @fires RetrievalGuard#filtered
39
+ */
40
+ filter(results: Array<object>, context?: {
41
+ userPermissions?: string[];
42
+ }): {
43
+ results: Array<object>;
44
+ removed: Array<object>;
45
+ metadata: {
46
+ reasons: {
47
+ [x: string]: number;
48
+ };
49
+ };
50
+ };
51
+ }
52
+ /**
53
+ * Default configuration for the RetrievalGuard.
54
+ * @type {object}
55
+ */
56
+ export const DEFAULT_CONFIG: object;
57
+ import { EventEmitter } from "events";
@@ -0,0 +1,85 @@
1
+ /**
2
+ * MCP server that registers RAG pipelines as callable MCP tools.
3
+ *
4
+ * @extends EventEmitter
5
+ * @fires MCPServer#tool_use
6
+ *
7
+ * @example
8
+ * const server = MCPServer.fromPipeline(myPipeline, { name: 'my_tool' });
9
+ * const tools = server.getToolDefinitions();
10
+ * const result = await server.handleToolUse('my_tool', { query: 'What is X?' });
11
+ */
12
+ export class MCPServer extends EventEmitter<[never]> {
13
+ /**
14
+ * Convenience factory: create an MCPServer with a single pipeline registered.
15
+ *
16
+ * @param {Object} pipeline - Pipeline instance with a `run` method
17
+ * @param {Object} [options={}] - Server and registration options
18
+ * @param {string} [options.name='rag_query'] - Tool name for the pipeline
19
+ * @param {Object} [options.config] - Pipeline-specific configuration
20
+ * @returns {MCPServer} Configured server instance
21
+ */
22
+ static fromPipeline(pipeline: any, options?: {
23
+ name?: string;
24
+ config?: any;
25
+ }): MCPServer;
26
+ /**
27
+ * @param {Object} [options={}] - Server configuration
28
+ * @param {boolean} [options.includeMetadata] - Include metadata in responses
29
+ * @param {number} [options.maxResponseLength] - Maximum response text length
30
+ * @param {Object} [options.toolOptions] - Options forwarded to MCPToolBuilder
31
+ */
32
+ constructor(options?: {
33
+ includeMetadata?: boolean;
34
+ maxResponseLength?: number;
35
+ toolOptions?: any;
36
+ });
37
+ config: any;
38
+ toolBuilder: MCPToolBuilder;
39
+ /** @type {Map<string, {pipeline: Object, config: Object, toolDefinition: Object}>} */
40
+ pipelines: Map<string, {
41
+ pipeline: any;
42
+ config: any;
43
+ toolDefinition: any;
44
+ }>;
45
+ /**
46
+ * Register a pipeline that can be called as an MCP tool.
47
+ *
48
+ * @param {string} name - Tool name used to invoke this pipeline
49
+ * @param {Object} pipeline - Pipeline instance with a `run` method
50
+ * @param {Object} [config={}] - Pipeline-specific configuration
51
+ * @returns {void}
52
+ */
53
+ registerPipeline(name: string, pipeline: any, config?: any): void;
54
+ /**
55
+ * Return array of all registered tool definitions.
56
+ * Each is a valid MCP tool schema.
57
+ *
58
+ * @returns {Array<Object>} Array of MCP tool definition objects
59
+ */
60
+ getToolDefinitions(): Array<any>;
61
+ /**
62
+ * Handle an MCP tool_use request by running the matching pipeline.
63
+ *
64
+ * @param {string} toolName - Name of the tool to invoke
65
+ * @param {Object} toolInput - Tool input parameters
66
+ * @param {string} toolInput.query - The search query (required)
67
+ * @param {number} [toolInput.topK] - Number of results to retrieve
68
+ * @param {boolean} [toolInput.stream] - Enable streaming response
69
+ * @param {boolean} [toolInput.citations] - Include source citations
70
+ * @param {boolean} [toolInput.evaluate] - Include quality evaluation metrics
71
+ * @returns {Promise<Object>} MCP-formatted response with type and text fields
72
+ * @throws {Error} If tool name is not registered or query is missing
73
+ */
74
+ handleToolUse(toolName: string, toolInput: {
75
+ query: string;
76
+ topK?: number;
77
+ stream?: boolean;
78
+ citations?: boolean;
79
+ evaluate?: boolean;
80
+ }): Promise<any>;
81
+ }
82
+ /** @type {Object} Default configuration for the MCP server */
83
+ export const DEFAULT_CONFIG: any;
84
+ import { EventEmitter } from "events";
85
+ import { MCPToolBuilder } from "./mcp-tool-builder";
@@ -0,0 +1,55 @@
1
+ /**
2
+ * Builds MCP-compatible tool definitions from RAG pipeline configuration.
3
+ *
4
+ * @example
5
+ * const builder = new MCPToolBuilder({ toolName: 'my_rag_tool' });
6
+ * const definition = builder.buildToolDefinition({ capabilities: { stream: true } });
7
+ */
8
+ export class MCPToolBuilder {
9
+ /**
10
+ * @param {Object} [options={}] - Builder configuration
11
+ * @param {string} [options.toolName] - Name for the generated MCP tool
12
+ * @param {string} [options.toolDescription] - Description for the generated MCP tool
13
+ */
14
+ constructor(options?: {
15
+ toolName?: string;
16
+ toolDescription?: string;
17
+ });
18
+ config: any;
19
+ /**
20
+ * Generate an MCP-compatible tool definition from pipeline configuration.
21
+ *
22
+ * @param {Object} [pipelineConfig={}] - Pipeline-specific configuration
23
+ * @param {Object} [pipelineConfig.capabilities] - Pipeline capabilities to reflect in schema
24
+ * @param {boolean} [pipelineConfig.capabilities.stream] - Whether streaming is supported
25
+ * @param {boolean} [pipelineConfig.capabilities.citations] - Whether citations are supported
26
+ * @param {boolean} [pipelineConfig.capabilities.evaluate] - Whether evaluation is supported
27
+ * @param {number} [pipelineConfig.capabilities.maxTopK] - Maximum allowed topK value
28
+ * @returns {Object} MCP tool definition with name, description, and input_schema
29
+ */
30
+ buildToolDefinition(pipelineConfig?: {
31
+ capabilities?: {
32
+ stream?: boolean;
33
+ citations?: boolean;
34
+ evaluate?: boolean;
35
+ maxTopK?: number;
36
+ };
37
+ }): any;
38
+ /**
39
+ * Build tool definitions for multiple named pipelines.
40
+ *
41
+ * @param {Array<Object>} pipelines - Array of pipeline descriptors
42
+ * @param {string} pipelines[].name - Tool name for this pipeline
43
+ * @param {string} [pipelines[].description] - Tool description
44
+ * @param {Object} [pipelines[].pipeline] - Pipeline instance (unused during definition building)
45
+ * @param {Object} [pipelines[].config] - Pipeline configuration passed to buildToolDefinition
46
+ * @returns {Array<Object>} Array of MCP tool definitions
47
+ */
48
+ buildMultipleTools(pipelines: Array<any>): Array<any>;
49
+ }
50
+ /**
51
+ * @module mcp/mcp-tool-builder
52
+ * @description Builds MCP tool definitions from pipeline configuration.
53
+ */
54
+ /** @type {Object} Default configuration for the MCP tool builder */
55
+ export const DEFAULT_CONFIG: any;
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Chains multiple rerankers in sequence, passing the output of each
3
+ * stage as input to the next. Implements the reranker contract:
4
+ * rerank(query, documents, options).
5
+ *
6
+ * @extends EventEmitter
7
+ */
8
+ export class CascadeReranker extends EventEmitter<[never]> {
9
+ /**
10
+ * @param {object[]} [rerankers=[]] - Array of objects with .rerank() method
11
+ * @param {object} [options]
12
+ */
13
+ constructor(rerankers?: object[], options?: object);
14
+ rerankers: any[];
15
+ config: any;
16
+ /**
17
+ * Run documents through each reranker in sequence.
18
+ *
19
+ * @param {string} query - The search query
20
+ * @param {Array<object|string>} documents - Documents to rerank
21
+ * @param {object} [options]
22
+ * @param {number} [options.topK] - Maximum number of results to return
23
+ * @param {AbortSignal} [options.signal] - Forwarded to every stage; checked between stages
24
+ * @returns {Promise<object[]>} Reranked documents from the final stage
25
+ * @throws {Error} If no rerankers are configured
26
+ */
27
+ rerank(query: string, documents: Array<object | string>, options?: {
28
+ topK?: number;
29
+ signal?: AbortSignal;
30
+ }): Promise<object[]>;
31
+ /**
32
+ * Add a reranker to the end of the cascade.
33
+ *
34
+ * @param {object} reranker - Must have a .rerank() method
35
+ */
36
+ addReranker(reranker: object): void;
37
+ }
38
+ /** @type {object} */
39
+ export const DEFAULT_CONFIG: object;
40
+ import { EventEmitter } from "events";
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Reranks documents by computing cosine similarity between query and
3
+ * document embeddings produced by an external embedder.
4
+ * Implements the reranker contract: rerank(query, documents, options).
5
+ *
6
+ * Document embeddings are cached in a bounded LRU keyed by a SHA-256 of the
7
+ * document text, and embedding calls run with bounded concurrency.
8
+ *
9
+ * @extends EventEmitter
10
+ */
11
+ export class EmbeddingReranker extends EventEmitter<[never]> {
12
+ /**
13
+ * @param {object} embedder - Must expose .embedQuery(text) or .embed(text)
14
+ * @param {object} [options]
15
+ * @param {number} [options.cacheSize=1000] - Max cached document embeddings (0 disables caching)
16
+ * @param {number} [options.concurrency=4] - Max concurrent embedding calls
17
+ */
18
+ constructor(embedder: object, options?: {
19
+ cacheSize?: number;
20
+ concurrency?: number;
21
+ });
22
+ embedder: any;
23
+ config: {
24
+ cacheSize: number;
25
+ concurrency: number;
26
+ };
27
+ /** @type {Map<string, Promise<number[]>>} LRU: oldest entry first */
28
+ _cache: Map<string, Promise<number[]>>;
29
+ cacheStats: {
30
+ hits: number;
31
+ misses: number;
32
+ };
33
+ /**
34
+ * Call the embedder, forwarding the signal only when one is given.
35
+ * @private
36
+ */
37
+ private _embed;
38
+ /**
39
+ * Embed a document, using the LRU cache. In-flight promises are cached so
40
+ * duplicate documents in one batch are embedded once; failed embeddings
41
+ * are evicted so they can be retried.
42
+ * @private
43
+ */
44
+ private _embedDocument;
45
+ /**
46
+ * Clear the document embedding cache.
47
+ */
48
+ clearCache(): void;
49
+ /**
50
+ * Rerank documents by cosine similarity to the query embedding.
51
+ *
52
+ * @param {string} query - The search query
53
+ * @param {Array<object|string>} documents - Documents to rerank
54
+ * @param {object} [options]
55
+ * @param {number} [options.topK] - Maximum number of results to return
56
+ * @param {AbortSignal} [options.signal] - Cancels pending embedding calls
57
+ * @returns {Promise<object[]>} Reranked documents with .relevanceScore
58
+ */
59
+ rerank(query: string, documents: Array<object | string>, options?: {
60
+ topK?: number;
61
+ signal?: AbortSignal;
62
+ }): Promise<object[]>;
63
+ /**
64
+ * Compute cosine similarity between two vectors.
65
+ *
66
+ * @param {number[]} a - First vector
67
+ * @param {number[]} b - Second vector
68
+ * @returns {number} Cosine similarity in [-1, 1], or 0 for zero vectors
69
+ */
70
+ _cosineSimilarity(a: number[], b: number[]): number;
71
+ }
72
+ /** @type {{ cacheSize: number, concurrency: number }} */
73
+ export const DEFAULT_CONFIG: {
74
+ cacheSize: number;
75
+ concurrency: number;
76
+ };
77
+ import { EventEmitter } from "events";
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Reranks documents using BM25-like scoring with IDF weighting.
3
+ * Implements the reranker contract: rerank(query, documents, options).
4
+ *
5
+ * @extends EventEmitter
6
+ */
7
+ export class ScoringReranker extends EventEmitter<[never]> {
8
+ /**
9
+ * @param {object} [options]
10
+ * @param {number} [options.k1=1.2] - Term frequency saturation parameter
11
+ * @param {number} [options.b=0.75] - Length normalisation parameter
12
+ */
13
+ constructor(options?: {
14
+ k1?: number;
15
+ b?: number;
16
+ });
17
+ config: {
18
+ k1: number;
19
+ b: number;
20
+ };
21
+ /**
22
+ * Rerank documents against a query using BM25 scoring.
23
+ *
24
+ * @param {string} query - The search query
25
+ * @param {Array<object|string>} documents - Documents to rerank
26
+ * @param {object} [options]
27
+ * @param {number} [options.topK] - Maximum number of results to return
28
+ * @param {AbortSignal} [options.signal] - Rejects with AbortError when already aborted
29
+ * @returns {Promise<object[]>} Reranked documents with .relevanceScore
30
+ */
31
+ rerank(query: string, documents: Array<object | string>, options?: {
32
+ topK?: number;
33
+ signal?: AbortSignal;
34
+ }): Promise<object[]>;
35
+ }
36
+ /** @type {{ k1: number, b: number }} */
37
+ export const DEFAULT_CONFIG: {
38
+ k1: number;
39
+ b: number;
40
+ };
41
+ import { EventEmitter } from "events";
@@ -0,0 +1,43 @@
1
+ export class BM25Search extends EventEmitter<[never]> {
2
+ constructor(options?: {});
3
+ config: {
4
+ k1: number;
5
+ b: number;
6
+ };
7
+ invertedIndex: Map<any, any>;
8
+ documents: any[];
9
+ /** @type {number[]} token count per document, computed at index time */
10
+ docLengths: number[];
11
+ avgDocLength: number;
12
+ docCount: number;
13
+ /**
14
+ * Index documents for BM25 search
15
+ * @param {Array} documents - [{id, content|text, metadata?}]
16
+ */
17
+ index(documents: any[]): void;
18
+ /**
19
+ * Search using BM25 scoring
20
+ * @param {string} query
21
+ * @param {object} options - {k: 10}
22
+ * @returns {Array} [{id, score, content, metadata}] sorted by BM25 score
23
+ */
24
+ search(query: string, options?: object): any[];
25
+ /**
26
+ * Clear the index
27
+ */
28
+ clear(): void;
29
+ /**
30
+ * Get index stats
31
+ * @returns {{docCount: number, uniqueTerms: number, avgDocLength: number}}
32
+ */
33
+ getStats(): {
34
+ docCount: number;
35
+ uniqueTerms: number;
36
+ avgDocLength: number;
37
+ };
38
+ }
39
+ export namespace DEFAULT_CONFIG {
40
+ let k1: number;
41
+ let b: number;
42
+ }
43
+ import { EventEmitter } from "events";
@@ -0,0 +1,87 @@
1
+ /**
2
+ * Vector store persisted to a single JSON file, so separate processes (e.g.
3
+ * `rag-pipeline ingest` then `rag-pipeline query`) share one index. Intended
4
+ * for local use and small corpora: every query is a linear scan.
5
+ *
6
+ * The index records which embedder produced the vectors (name, dimensions and
7
+ * optional serialized state) and refuses queries whose vector dimension does
8
+ * not match, instead of silently comparing incompatible vectors.
9
+ */
10
+ export class FileVectorStore {
11
+ /**
12
+ * @param {object} options
13
+ * @param {string} options.path - Index file location
14
+ */
15
+ constructor(options?: {
16
+ path: string;
17
+ });
18
+ path: string;
19
+ _index: any;
20
+ _emptyIndex(): {
21
+ version: number;
22
+ embedder: any;
23
+ documents: {};
24
+ };
25
+ /**
26
+ * Read the index from disk (cached after the first call).
27
+ * @returns {{version: number, embedder: object|null, documents: object}}
28
+ */
29
+ load(): {
30
+ version: number;
31
+ embedder: object | null;
32
+ documents: object;
33
+ };
34
+ /** Write the index atomically: temp file in the same directory, then rename. */
35
+ save(): void;
36
+ /** @returns {object|null} Embedder metadata recorded with the vectors */
37
+ getEmbedderInfo(): object | null;
38
+ /** @param {{name: string, dimensions: number, state?: object}} info */
39
+ setEmbedderInfo(info: {
40
+ name: string;
41
+ dimensions: number;
42
+ state?: object;
43
+ }): void;
44
+ /**
45
+ * Add or replace documents. Ids are caller-chosen; re-storing an id
46
+ * replaces the previous entry.
47
+ *
48
+ * @param {Array<{id: string, vector: number[], content: string, metadata?: object}>} docs
49
+ */
50
+ store(docs: Array<{
51
+ id: string;
52
+ vector: number[];
53
+ content: string;
54
+ metadata?: object;
55
+ }>): Promise<void>;
56
+ /**
57
+ * Remove every document whose metadata.source equals `source`.
58
+ * @param {string} source
59
+ * @returns {number} Number of documents removed
60
+ */
61
+ removeBySource(source: string): number;
62
+ /** Remove every document and the recorded embedder info. */
63
+ clear(): void;
64
+ /** @returns {Array<{id: string, content: string, metadata: object}>} */
65
+ list(): Array<{
66
+ id: string;
67
+ content: string;
68
+ metadata: object;
69
+ }>;
70
+ /** @returns {number} */
71
+ size(): number;
72
+ /**
73
+ * Cosine-similarity search.
74
+ *
75
+ * @param {{queryVector: number[], topK?: number}} request
76
+ * @returns {Promise<Array<{id: string, score: number, content: string, metadata: object}>>}
77
+ */
78
+ retrieve(request?: {
79
+ queryVector: number[];
80
+ topK?: number;
81
+ }): Promise<Array<{
82
+ id: string;
83
+ score: number;
84
+ content: string;
85
+ metadata: object;
86
+ }>>;
87
+ }
@@ -0,0 +1,35 @@
1
+ export class HybridRetriever extends EventEmitter<[never]> {
2
+ /**
3
+ * @param {Array<{retriever: object, weight: number, name: string}>} retrievers
4
+ * @param {object} options
5
+ */
6
+ constructor(retrievers?: Array<{
7
+ retriever: object;
8
+ weight: number;
9
+ name: string;
10
+ }>, options?: object);
11
+ retrievers: {
12
+ retriever: any;
13
+ weight: number;
14
+ name: string;
15
+ }[];
16
+ config: any;
17
+ /**
18
+ * Run all retrievers in parallel, fuse results
19
+ * @param {object} query - {query: string, queryVector?: number[], topK?: number}
20
+ * @returns {Promise<Array>} Fused results
21
+ */
22
+ retrieve(query: object): Promise<any[]>;
23
+ /**
24
+ * Add a retriever
25
+ * @param {object} retriever - Object with .retrieve() or .search() method
26
+ * @param {number} weight - Weight for rank fusion
27
+ * @param {string} name - Name for this retriever
28
+ */
29
+ addRetriever(retriever: object, weight?: number, name?: string): void;
30
+ }
31
+ export namespace DEFAULT_CONFIG {
32
+ let topK: number;
33
+ let fusionK: number;
34
+ }
35
+ import { EventEmitter } from "events";
@@ -0,0 +1,10 @@
1
+ /**
2
+ * Reciprocal Rank Fusion - merges multiple ranked result lists
3
+ * @param {Array<Array>} resultSets - Array of result arrays, each result has {id, score, ...}
4
+ * @param {object} options - {k: 60, weights: number[] (per result set)}
5
+ * @returns {Array} Fused, deduplicated, sorted results
6
+ */
7
+ export function reciprocalRankFusion(resultSets: Array<any[]>, options?: object): any[];
8
+ export namespace DEFAULT_CONFIG {
9
+ let k: number;
10
+ }
@@ -0,0 +1,80 @@
1
+ /**
2
+ * Load JWT configuration from environment or defaults
3
+ *
4
+ * Environment variables:
5
+ * - JWT_SECRET: Secret for HMAC algorithms (min 32 bytes)
6
+ * - JWT_ALGORITHM: Algorithm to use (default: HS256)
7
+ * - JWT_ISSUER: Token issuer (default: rag-pipeline-utils)
8
+ * - JWT_AUDIENCE: Token audience (default: rag-pipeline-api)
9
+ * - JWT_EXPIRY: Token expiration in seconds (default: 3600)
10
+ * - JWT_CLOCK_TOLERANCE: Clock skew tolerance in seconds (default: 30)
11
+ * - JWT_PUBLIC_KEY_PATH: Path to public key file (for asymmetric algorithms)
12
+ * - JWT_PRIVATE_KEY_PATH: Path to private key file (for asymmetric algorithms)
13
+ * - JWT_ENABLE_JTI_TRACKING: Enable JTI replay protection (default: true)
14
+ *
15
+ * @returns {Object} JWT configuration
16
+ */
17
+ export function loadJWTConfig(): any;
18
+ /**
19
+ * Create JWT configuration with overrides
20
+ *
21
+ * @param {Object} overrides - Configuration overrides
22
+ * @returns {Object} Merged configuration
23
+ *
24
+ * @example
25
+ * const config = createJWTConfig({
26
+ * algorithm: 'RS256',
27
+ * publicKey: fs.readFileSync('public.pem'),
28
+ * privateKey: fs.readFileSync('private.pem'),
29
+ * });
30
+ */
31
+ export function createJWTConfig(overrides?: any): any;
32
+ /**
33
+ * Validate JWT configuration
34
+ *
35
+ * @param {Object} config - JWT configuration to validate
36
+ * @throws {Error} If configuration is invalid
37
+ */
38
+ export function validateJWTConfig(config: any): void;
39
+ /**
40
+ * Get environment-specific JWT configuration presets
41
+ *
42
+ * @param {string} environment - Environment name (development, production, test)
43
+ * @returns {Object} Environment-specific configuration
44
+ */
45
+ export function getEnvironmentPreset(environment: string): any;
46
+ /**
47
+ * Generate a secure JWT secret
48
+ *
49
+ * @param {number} bytes - Number of bytes (default: 64)
50
+ * @returns {string} Hex-encoded secret
51
+ *
52
+ * @example
53
+ * const secret = generateJWTSecret();
54
+ * console.log('Add to .env: JWT_SECRET=' + secret);
55
+ */
56
+ export function generateJWTSecret(bytes?: number): string;
57
+ /**
58
+ * Generate RSA key pair for JWT signing
59
+ *
60
+ * @param {number} modulusLength - Key size in bits (default: 2048)
61
+ * @returns {Promise<Object>} Key pair { publicKey, privateKey }
62
+ *
63
+ * @example
64
+ * const { publicKey, privateKey } = await generateRSAKeyPair();
65
+ * fs.writeFileSync('jwt-public.pem', publicKey);
66
+ * fs.writeFileSync('jwt-private.pem', privateKey);
67
+ */
68
+ export function generateRSAKeyPair(modulusLength?: number): Promise<any>;
69
+ /**
70
+ * Generate ECDSA key pair for JWT signing
71
+ *
72
+ * @param {string} namedCurve - Curve name (default: 'prime256v1' for ES256)
73
+ * @returns {Promise<Object>} Key pair { publicKey, privateKey }
74
+ *
75
+ * @example
76
+ * const { publicKey, privateKey } = await generateECDSAKeyPair('prime256v1');
77
+ * fs.writeFileSync('jwt-public-ec.pem', publicKey);
78
+ * fs.writeFileSync('jwt-private-ec.pem', privateKey);
79
+ */
80
+ export function generateECDSAKeyPair(namedCurve?: string): Promise<any>;