@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (225) hide show
  1. package/.ragrc.schema.json +83 -192
  2. package/CHANGELOG.md +345 -0
  3. package/README.md +418 -117
  4. package/SECURITY.md +5 -5
  5. package/dist/experimental.cjs +2 -0
  6. package/dist/experimental.d.ts +1 -0
  7. package/dist/experimental.mjs +5 -0
  8. package/dist/index.cjs +4992 -1242
  9. package/dist/index.d.ts +80 -2
  10. package/dist/index.mjs +1 -1
  11. package/dist/types/agentic/agentic-pipeline.d.ts +103 -0
  12. package/dist/types/agentic/iterative-retriever.d.ts +85 -0
  13. package/dist/types/agentic/query-planner.d.ts +145 -0
  14. package/dist/types/agentic/self-critique.d.ts +45 -0
  15. package/dist/types/ai/adaptive-retrieval.d.ts +44 -0
  16. package/dist/types/ai/federation/federated-learning-coordinator.d.ts +62 -0
  17. package/dist/types/ai/multimodal-processing.d.ts +32 -0
  18. package/dist/types/ai/multimodal.d.ts +39 -0
  19. package/dist/types/ai/retrieval-engine.d.ts +37 -0
  20. package/dist/types/ai/training/model-training-orchestrator.d.ts +31 -0
  21. package/dist/types/chunking/chunking-engine.d.ts +70 -0
  22. package/dist/types/chunking/strategies/fixed-size.d.ts +17 -0
  23. package/dist/types/chunking/strategies/recursive.d.ts +20 -0
  24. package/dist/types/chunking/strategies/semantic.d.ts +19 -0
  25. package/dist/types/chunking/strategies/sentence.d.ts +13 -0
  26. package/dist/types/chunking/strategies/structure-aware.d.ts +19 -0
  27. package/dist/types/citation/citation-tracker.d.ts +64 -0
  28. package/dist/types/citation/hallucination-detector.d.ts +40 -0
  29. package/dist/types/citation/source-mapper.d.ts +37 -0
  30. package/dist/types/connectors/anthropic-connector.d.ts +92 -0
  31. package/dist/types/connectors/base-connector.d.ts +87 -0
  32. package/dist/types/connectors/cohere-connector.d.ts +89 -0
  33. package/dist/types/connectors/connector-utils.d.ts +110 -0
  34. package/dist/types/connectors/local-embedder.d.ts +86 -0
  35. package/dist/types/connectors/memory-retriever.d.ts +91 -0
  36. package/dist/types/connectors/ollama-connector.d.ts +110 -0
  37. package/dist/types/connectors/openai-connector.d.ts +114 -0
  38. package/dist/types/connectors/rag-prompt.d.ts +40 -0
  39. package/dist/types/cost/cost-calculator.d.ts +152 -0
  40. package/dist/types/cost/cost-tracker.d.ts +126 -0
  41. package/dist/types/cost/token-budget.d.ts +110 -0
  42. package/dist/types/dag/core/error-context.d.ts +239 -0
  43. package/dist/types/dag/core/execution-scheduler.d.ts +191 -0
  44. package/dist/types/dag/core/topology-validation.d.ts +226 -0
  45. package/dist/types/dag/dag-engine.d.ts +156 -0
  46. package/dist/types/dag/dag-node.d.ts +37 -0
  47. package/dist/types/dashboard/dashboard-generator.d.ts +127 -0
  48. package/dist/types/dashboard/metrics-aggregator.d.ts +218 -0
  49. package/dist/types/debug/execution-tracer.d.ts +57 -0
  50. package/dist/types/debug/trace-inspector.d.ts +31 -0
  51. package/dist/types/ecosystem/plugin-certification.d.ts +169 -0
  52. package/dist/types/enterprise/data-governance.d.ts +203 -0
  53. package/dist/types/enterprise/multi-tenancy.d.ts +137 -0
  54. package/dist/types/enterprise/sso-integration.d.ts +420 -0
  55. package/dist/types/evaluate/context-metrics.d.ts +55 -0
  56. package/dist/types/evaluate/faithfulness.d.ts +41 -0
  57. package/dist/types/evaluate/groundedness.d.ts +60 -0
  58. package/dist/types/evaluate/pipeline-evaluator.d.ts +88 -0
  59. package/dist/types/evaluate/relevance.d.ts +21 -0
  60. package/dist/types/evaluate/scoring.d.ts +123 -0
  61. package/dist/types/experimental.d.ts +15 -0
  62. package/dist/types/graph/entity-extractor.d.ts +84 -0
  63. package/dist/types/graph/graph-index.d.ts +49 -0
  64. package/dist/types/graph/graph-retriever.d.ts +81 -0
  65. package/dist/types/graph/knowledge-graph.d.ts +129 -0
  66. package/dist/types/guardrails/guardrails-pipeline.d.ts +63 -0
  67. package/dist/types/guardrails/injection-detector.d.ts +47 -0
  68. package/dist/types/guardrails/post-generation-guard.d.ts +61 -0
  69. package/dist/types/guardrails/pre-retrieval-guard.d.ts +64 -0
  70. package/dist/types/guardrails/retrieval-guard.d.ts +57 -0
  71. package/dist/types/mcp/mcp-server.d.ts +85 -0
  72. package/dist/types/mcp/mcp-tool-builder.d.ts +55 -0
  73. package/dist/types/reranker/cascade-reranker.d.ts +40 -0
  74. package/dist/types/reranker/embedding-reranker.d.ts +77 -0
  75. package/dist/types/reranker/scoring-reranker.d.ts +41 -0
  76. package/dist/types/retrieval/bm25-search.d.ts +43 -0
  77. package/dist/types/retrieval/file-vector-store.d.ts +87 -0
  78. package/dist/types/retrieval/hybrid-retriever.d.ts +35 -0
  79. package/dist/types/retrieval/rank-fusion.d.ts +10 -0
  80. package/dist/types/security/jwt-config.d.ts +80 -0
  81. package/dist/types/security/jwt-validator.d.ts +195 -0
  82. package/dist/types/security/plugin-sandbox.d.ts +158 -0
  83. package/dist/types/security/plugin-signature-verifier.d.ts +139 -0
  84. package/dist/types/security/validator.d.ts +5 -0
  85. package/dist/types/streaming/embedding-stream.d.ts +65 -0
  86. package/dist/types/streaming/sse-adapter.d.ts +59 -0
  87. package/dist/types/streaming/stream-router.d.ts +88 -0
  88. package/dist/types/streaming/streaming-embedder.d.ts +99 -0
  89. package/dist/types/streaming/websocket-adapter.d.ts +71 -0
  90. package/dist/types/templates/scaffold.d.ts +55 -0
  91. package/dist/types/templates/template-registry.d.ts +63 -0
  92. package/dist/types/utils/embed-query.d.ts +20 -0
  93. package/dist/types/utils/logger.d.ts +50 -0
  94. package/dist/types/utils/rate-limiter.d.ts +66 -0
  95. package/dist/types/utils/retry.d.ts +27 -0
  96. package/dist/types/utils/run-abort.d.ts +34 -0
  97. package/dist/types/utils/secure-logging.d.ts +51 -0
  98. package/dist/types/utils/structured-logger.d.ts +126 -0
  99. package/package.json +53 -29
  100. package/src/agentic/agentic-pipeline.js +75 -7
  101. package/src/agentic/iterative-retriever.js +47 -10
  102. package/src/agentic/query-planner.js +329 -78
  103. package/src/citation/citation-tracker.js +6 -4
  104. package/src/citation/hallucination-detector.js +2 -2
  105. package/src/citation/source-mapper.js +15 -28
  106. package/src/cli/commands/ai.js +101 -0
  107. package/src/cli/commands/plugin-hub/event-handlers.js +6 -0
  108. package/src/cli/commands/plugin-hub/hub-commands.js +0 -74
  109. package/src/cli/commands/plugin-hub/local-commands.js +143 -0
  110. package/src/cli/commands/plugin-hub.js +96 -36
  111. package/src/cli/diagnostics/config-checker.js +76 -63
  112. package/src/cli/diagnostics/environment-checker.js +62 -31
  113. package/src/cli/diagnostics/plugin-checker.js +36 -89
  114. package/src/cli/doctor-command.js +39 -5
  115. package/src/cli/enhanced-cli-commands.js +72 -217
  116. package/src/cli/handlers/ai-handler.js +364 -0
  117. package/src/cli/handlers/diagnostics-handler.js +78 -31
  118. package/src/cli/handlers/ingest-handler.js +28 -22
  119. package/src/cli/handlers/init-handler.js +148 -25
  120. package/src/cli/handlers/query-handler.js +41 -26
  121. package/src/cli/handlers/run-handler.js +16 -167
  122. package/src/cli/handlers/serve-handler.js +60 -0
  123. package/src/cli/services/bench-stats.js +54 -0
  124. package/src/cli/services/benchmark.js +161 -0
  125. package/src/cli/services/feedback.js +155 -0
  126. package/src/cli/services/rag-service.js +322 -0
  127. package/src/cli/shared/cli-utils.js +24 -7
  128. package/src/cli/shared/config-context.js +21 -0
  129. package/src/cli/wizard/config-builder.js +81 -0
  130. package/src/cli/wizard/init-wizard.js +617 -0
  131. package/src/config/component-factory.js +125 -0
  132. package/src/config/load-config.js +29 -10
  133. package/src/config/load-plugin-config.js +46 -2
  134. package/src/connectors/anthropic-connector.js +93 -55
  135. package/src/connectors/base-connector.js +74 -19
  136. package/src/connectors/cohere-connector.js +88 -32
  137. package/src/connectors/connector-utils.js +302 -0
  138. package/src/connectors/local-embedder.js +43 -5
  139. package/src/connectors/memory-retriever.js +108 -15
  140. package/src/connectors/ollama-connector.js +83 -49
  141. package/src/connectors/openai-connector.js +108 -44
  142. package/src/connectors/rag-prompt.js +88 -0
  143. package/src/core/create-pipeline.js +209 -60
  144. package/src/core/plugin-registry.js +87 -11
  145. package/src/cost/cost-calculator.js +205 -32
  146. package/src/dev/dev-server.js +166 -11
  147. package/src/dev/hot-reload.js +40 -1
  148. package/src/ecosystem/plugin-checks.js +181 -0
  149. package/src/ecosystem/plugin-hub.js +50 -125
  150. package/src/ecosystem/plugin-installer.js +298 -0
  151. package/src/ecosystem/plugin-probe.js +127 -0
  152. package/src/ecosystem/plugin-scanner.js +185 -0
  153. package/src/ecosystem/tar-extract.js +114 -0
  154. package/src/enterprise/audit-logging.js +158 -29
  155. package/src/enterprise/sso-integration.js +715 -228
  156. package/src/evaluate/context-metrics.js +28 -16
  157. package/src/evaluate/evaluator.js +3 -3
  158. package/src/evaluate/faithfulness.js +18 -14
  159. package/src/evaluate/groundedness.js +43 -12
  160. package/src/evaluate/pipeline-evaluator.js +35 -8
  161. package/src/evaluate/scoring.js +251 -3
  162. package/src/experimental.js +71 -0
  163. package/src/guardrails/guardrails-pipeline.js +69 -7
  164. package/src/guardrails/injection-detector.js +283 -0
  165. package/src/guardrails/post-generation-guard.js +19 -0
  166. package/src/guardrails/pre-retrieval-guard.js +36 -36
  167. package/src/guardrails/retrieval-guard.js +58 -8
  168. package/src/index.js +36 -1
  169. package/src/loader/document-reader.js +91 -0
  170. package/src/reranker/cascade-reranker.js +3 -0
  171. package/src/reranker/embedding-reranker.js +136 -21
  172. package/src/reranker/llm-reranker.js +79 -12
  173. package/src/reranker/scoring-reranker.js +3 -0
  174. package/src/retrieval/bm25-search.js +12 -6
  175. package/src/retrieval/file-vector-store.js +212 -0
  176. package/src/retrieval/hybrid-retriever.js +22 -19
  177. package/src/security/jwt-config.js +13 -2
  178. package/src/security/jwt-validator.js +134 -30
  179. package/src/security/plugin-sandbox.js +18 -1
  180. package/src/security/plugin-signature-verifier.js +245 -70
  181. package/src/utils/deprecate.js +69 -0
  182. package/src/utils/embed-query.js +33 -0
  183. package/src/utils/input-sanitizer.js +19 -4
  184. package/src/utils/logger.js +34 -4
  185. package/src/utils/run-abort.js +102 -0
  186. package/src/ai/federated-learning.js +0 -211
  187. package/src/ai/federation.js +0 -71
  188. package/src/ai/index.js +0 -22
  189. package/src/ai/model-trainer.js +0 -217
  190. package/src/ai/model-training.js +0 -202
  191. package/src/ai/multimodal/multi-modal-processor.js +0 -345
  192. package/src/ai/orchestrator.js +0 -65
  193. package/src/ai/retrieval/adaptive-retrieval-engine.js +0 -398
  194. package/src/cli/commands/ai-ml/adaptive-commands.js +0 -178
  195. package/src/cli/commands/ai-ml/federated-commands.js +0 -207
  196. package/src/cli/commands/ai-ml/index.js +0 -17
  197. package/src/cli/commands/ai-ml/multimodal-commands.js +0 -187
  198. package/src/cli/commands/ai-ml/shared.js +0 -51
  199. package/src/cli/commands/ai-ml/training-commands.js +0 -228
  200. package/src/cli/commands/ai-ml.js +0 -386
  201. package/src/cli/commands/docs.js +0 -228
  202. package/src/cli/commands/dx.js +0 -538
  203. package/src/cli/commands/wizard.js +0 -627
  204. package/src/cli/input-sanitization-middleware.js +0 -356
  205. package/src/cli/interactive-wizard.js +0 -373
  206. package/src/cli/templates/development.template.json +0 -64
  207. package/src/cli/templates/minimal.template.json +0 -33
  208. package/src/cli/templates/production.template.json +0 -104
  209. package/src/cli/templates/testing.template.json +0 -45
  210. package/src/cli/wizard/configuration.js +0 -148
  211. package/src/cli/wizard/observability.js +0 -79
  212. package/src/cli/wizard/performance.js +0 -76
  213. package/src/cli/wizard/plugin-selection.js +0 -297
  214. package/src/cli/wizard/preview-save.js +0 -172
  215. package/src/cli/wizard/project-setup.js +0 -66
  216. package/src/core/pipeline-factory.js +0 -59
  217. package/src/dag/enhanced-dag-engine.js +0 -894
  218. package/src/dx/index.js +0 -40
  219. package/src/dx/integration-templates.js +0 -662
  220. package/src/dx/performance-profiler.js +0 -968
  221. package/src/dx/realtime-debugger.js +0 -755
  222. package/src/dx/visual-pipeline-builder.js +0 -578
  223. package/src/ingest.js +0 -59
  224. package/src/pipeline/pipeline.js +0 -18
  225. package/src/query.js +0 -117
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Process multimodal content and store with embedding
3
+ * @param {string} tenantId - Tenant identifier
4
+ * @param {object} content - Content object with type and data
5
+ * @returns {Promise<object>} Processed content record with ID, modalities, metadata
6
+ */
7
+ export function processContent(tenantId: string, content: object): Promise<object>;
8
+ /**
9
+ * Generate description for content by ID
10
+ * @param {string} contentId - Content ID
11
+ * @returns {Promise<object>} Object with modality-specific descriptions and unified description
12
+ */
13
+ export function describeContent(contentId: string): Promise<object>;
14
+ /**
15
+ * Search multimodal content using query object
16
+ * @param {string} tenantId - Tenant identifier
17
+ * @param {object} queryObj - Query object
18
+ * @param {object} options - Search options
19
+ * @returns {Promise<object>} Object with results array and metadata
20
+ */
21
+ export function search(tenantId: string, queryObj: object, options?: object): Promise<object>;
22
+ /**
23
+ * Find similar content to given ID
24
+ * @param {string} contentId - Source content ID
25
+ * @param {object} options - Options with threshold
26
+ * @returns {Promise<Array>} Array of similar items with id and score
27
+ */
28
+ export function findSimilar(contentId: string, options?: object): Promise<any[]>;
29
+ /**
30
+ * Clear all stored content (for testing)
31
+ */
32
+ export function clear(): void;
@@ -0,0 +1,39 @@
1
+ /**
2
+ * MultiModalProcessor class
3
+ * Processes and integrates multiple content modalities
4
+ */
5
+ export class MultiModalProcessor {
6
+ supportedTypes: Set<string>;
7
+ processors: Map<any, any>;
8
+ embeddings: Map<any, any>;
9
+ /**
10
+ * Process multimodal content
11
+ * @param {string} tenantId - Tenant identifier
12
+ * @param {Array} content - Array of content objects with type and data
13
+ * @param {object} options - Processing options
14
+ * @returns {Promise<object>} Processing results
15
+ */
16
+ processContent(tenantId: string, content: any[], options?: object): Promise<object>;
17
+ /**
18
+ * Generate embeddings for multimodal content
19
+ * @param {string} tenantId - Tenant identifier
20
+ * @param {object} content - Content to embed
21
+ * @returns {Promise<Array>} Generated embeddings
22
+ */
23
+ generateEmbeddings(tenantId: string, content: object): Promise<any[]>;
24
+ /**
25
+ * Align embeddings across modalities
26
+ * @param {string} tenantId - Tenant identifier
27
+ * @param {Array} embeddings - Multi-modal embeddings
28
+ * @returns {Promise<Array>} Aligned embeddings
29
+ */
30
+ alignEmbeddings(tenantId: string, embeddings: any[]): Promise<any[]>;
31
+ /**
32
+ * Perform cross-modal search
33
+ * @param {string} tenantId - Tenant identifier
34
+ * @param {object} query - Query object with modality
35
+ * @param {object} options - Search options
36
+ * @returns {Promise<Array>} Search results
37
+ */
38
+ crossModalSearch(tenantId: string, query: object, options?: object): Promise<any[]>;
39
+ }
@@ -0,0 +1,37 @@
1
+ /**
2
+ * AdaptiveRetrievalEngine class
3
+ * Manages intelligent document retrieval with performance optimization
4
+ */
5
+ export class AdaptiveRetrievalEngine {
6
+ algorithms: Map<any, any>;
7
+ performanceMetrics: Map<any, any>;
8
+ adaptationThreshold: number;
9
+ /**
10
+ * Process retrieval query with adaptive algorithms
11
+ * @param {string} tenantId - Tenant identifier
12
+ * @param {string} query - Search query
13
+ * @param {object} options - Retrieval options
14
+ * @returns {Promise<object>} Retrieval results
15
+ */
16
+ retrieveDocuments(tenantId: string, query: string, options?: object): Promise<object>;
17
+ /**
18
+ * Optimize retrieval algorithm based on performance data
19
+ * @param {string} tenantId - Tenant identifier
20
+ * @param {object} performanceData - Performance metrics
21
+ * @returns {Promise<object>} Optimization results
22
+ */
23
+ optimizeRetrieval(tenantId: string, performanceData: object): Promise<object>;
24
+ /**
25
+ * Get current algorithm configuration
26
+ * @param {string} tenantId - Tenant identifier
27
+ * @returns {Promise<object>} Current algorithm configuration
28
+ */
29
+ getAlgorithmConfig(tenantId: string): Promise<object>;
30
+ /**
31
+ * Update retrieval strategy
32
+ * @param {string} tenantId - Tenant identifier
33
+ * @param {string} strategy - New retrieval strategy
34
+ * @returns {Promise<boolean>} Success status
35
+ */
36
+ updateStrategy(tenantId: string, strategy: string): Promise<boolean>;
37
+ }
@@ -0,0 +1,31 @@
1
+ export class ModelTrainingOrchestrator extends EventEmitter<[never]> {
2
+ constructor(options?: {});
3
+ config: {
4
+ batchSize: any;
5
+ learningRate: any;
6
+ epochs: any;
7
+ };
8
+ trainingJobs: Map<any, any>;
9
+ deployments: Map<any, any>;
10
+ optimizations: Map<any, any>;
11
+ createTrainingJob(tenantId: any, config?: {}): Promise<`${string}-${string}-${string}-${string}-${string}`>;
12
+ startTraining(jobId: any, data?: any, config?: {}): Promise<{
13
+ jobId: any;
14
+ status: string;
15
+ }>;
16
+ getTrainingStatus(jobId: any): any;
17
+ stopTraining(jobId: any): Promise<boolean>;
18
+ deployModel(jobId: any, deploymentConfig?: {}): Promise<`${string}-${string}-${string}-${string}-${string}`>;
19
+ getDeploymentStatus(deploymentId: any): Promise<any>;
20
+ optimizeHyperparameters(tenantId: any, optimizationConfig: any): Promise<`${string}-${string}-${string}-${string}-${string}`>;
21
+ getOptimizationResults(optimizationId: any): Promise<{
22
+ id: any;
23
+ status: any;
24
+ bestConfiguration: any;
25
+ bestScore: any;
26
+ trials: any;
27
+ startedAt: any;
28
+ completedAt: any;
29
+ }>;
30
+ }
31
+ import { EventEmitter } from "events";
@@ -0,0 +1,70 @@
1
+ /**
2
+ * ChunkingEngine orchestrates text chunking using pluggable strategies.
3
+ * Extends EventEmitter to emit 'chunk' events after each chunking operation.
4
+ *
5
+ * @extends EventEmitter
6
+ */
7
+ export class ChunkingEngine extends EventEmitter<[never]> {
8
+ /**
9
+ * Create a ChunkingEngine instance.
10
+ *
11
+ * @param {object} [options={}] - Configuration options merged with DEFAULT_CONFIG.
12
+ * @param {string} [options.strategy='recursive'] - Default strategy name.
13
+ * @param {number} [options.chunkSize=512] - Default chunk size.
14
+ * @param {number} [options.chunkOverlap=50] - Default chunk overlap.
15
+ * @param {number} [options.minChunkSize=50] - Default minimum chunk size.
16
+ * @param {string[]} [options.separators] - Default separators for recursive strategy.
17
+ * @param {number} [options.similarityThreshold=0.3] - Default similarity threshold for semantic strategy.
18
+ */
19
+ constructor(options?: {
20
+ strategy?: string;
21
+ chunkSize?: number;
22
+ chunkOverlap?: number;
23
+ minChunkSize?: number;
24
+ separators?: string[];
25
+ similarityThreshold?: number;
26
+ });
27
+ _config: any;
28
+ _strategies: Map<any, any>;
29
+ /**
30
+ * Chunk text using the configured or specified strategy.
31
+ *
32
+ * @param {string} text - The input text to chunk. Must be a non-empty string.
33
+ * @param {object} [options={}] - Per-call options that override the engine config.
34
+ * @param {string} [options.strategy] - Strategy name to use for this call.
35
+ * @returns {string[]} Array of text chunks.
36
+ * @throws {Error} If text is not a non-empty string.
37
+ * @throws {Error} If the specified strategy is not registered.
38
+ * @fires ChunkingEngine#chunk
39
+ */
40
+ chunk(text: string, options?: {
41
+ strategy?: string;
42
+ }): string[];
43
+ /**
44
+ * Register a custom chunking strategy.
45
+ *
46
+ * @param {string} name - Strategy name (used to select it via options.strategy).
47
+ * @param {function} fn - Strategy function with signature (text, options) => string[].
48
+ * @throws {Error} If name is not a non-empty string.
49
+ * @throws {Error} If fn is not a function.
50
+ */
51
+ registerStrategy(name: string, fn: Function): void;
52
+ /**
53
+ * List all registered strategy names.
54
+ *
55
+ * @returns {string[]} Array of strategy names.
56
+ */
57
+ listStrategies(): string[];
58
+ /**
59
+ * Get the current engine configuration.
60
+ *
61
+ * @returns {object} A copy of the current configuration.
62
+ */
63
+ getConfig(): object;
64
+ }
65
+ /**
66
+ * Default configuration for the chunking engine.
67
+ * @type {object}
68
+ */
69
+ export const DEFAULT_CONFIG: object;
70
+ import { EventEmitter } from "events";
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Fixed-size chunking strategy.
3
+ * Splits text by character count with configurable overlap.
4
+ * Attempts to break at word boundaries by looking back up to 20 characters.
5
+ *
6
+ * @param {string} text - The input text to chunk.
7
+ * @param {object} [options={}] - Chunking options.
8
+ * @param {number} [options.chunkSize=512] - Target chunk size in characters.
9
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
10
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
11
+ * @returns {string[]} Array of text chunks.
12
+ */
13
+ export function fixedSizeChunk(text: string, options?: {
14
+ chunkSize?: number;
15
+ chunkOverlap?: number;
16
+ minChunkSize?: number;
17
+ }): string[];
@@ -0,0 +1,20 @@
1
+ /**
2
+ * Recursive text chunking strategy.
3
+ * Splits text using a hierarchy of separators, merging consecutive pieces
4
+ * into chunks up to chunkSize. Pieces exceeding chunkSize are recursively
5
+ * split with the next separator in the hierarchy.
6
+ *
7
+ * @param {string} text - The input text to chunk.
8
+ * @param {object} [options={}] - Chunking options.
9
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
10
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
11
+ * @param {string[]} [options.separators] - Separator hierarchy for splitting.
12
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
13
+ * @returns {string[]} Array of text chunks.
14
+ */
15
+ export function recursiveChunk(text: string, options?: {
16
+ chunkSize?: number;
17
+ chunkOverlap?: number;
18
+ separators?: string[];
19
+ minChunkSize?: number;
20
+ }): string[];
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Semantic chunking strategy using topic-boundary detection.
3
+ * Splits text into sentences, computes Jaccard token overlap between
4
+ * consecutive sentence pairs, and inserts chunk boundaries where
5
+ * similarity drops below the threshold. Oversized segments fall back
6
+ * to recursive splitting.
7
+ *
8
+ * @param {string} text - The input text to chunk.
9
+ * @param {object} [options={}] - Chunking options.
10
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
11
+ * @param {number} [options.similarityThreshold=0.3] - Minimum similarity to keep sentences in the same chunk.
12
+ * @param {number} [options.minChunkSize=50] - Minimum chunk size; smaller chunks are filtered out.
13
+ * @returns {string[]} Array of text chunks.
14
+ */
15
+ export function semanticChunk(text: string, options?: {
16
+ chunkSize?: number;
17
+ similarityThreshold?: number;
18
+ minChunkSize?: number;
19
+ }): string[];
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Sentence-based chunking strategy.
3
+ * Extracted from MarkdownLoader._chunkText for backward compatibility.
4
+ * Splits text on sentence boundaries and merges into chunks up to maxLen.
5
+ *
6
+ * @param {string} text - The input text to chunk.
7
+ * @param {object} [options={}] - Chunking options.
8
+ * @param {number} [options.maxLen=500] - Maximum character length per chunk.
9
+ * @returns {string[]} Array of text chunks.
10
+ */
11
+ export function sentenceChunk(text: string, options?: {
12
+ maxLen?: number;
13
+ }): string[];
@@ -0,0 +1,19 @@
1
+ /**
2
+ * Structure-aware chunking strategy.
3
+ * Detects document format (Markdown, HTML, or plain) and respects
4
+ * structural boundaries. For Markdown, splits on headers and preserves
5
+ * code blocks. For HTML, splits on heading/section tags. Oversized
6
+ * sections are further split via the recursive strategy.
7
+ *
8
+ * @param {string} text - The input text to chunk.
9
+ * @param {object} [options={}] - Chunking options.
10
+ * @param {number} [options.chunkSize=512] - Maximum chunk size in characters.
11
+ * @param {number} [options.chunkOverlap=50] - Number of overlapping characters between chunks.
12
+ * @param {string} [options.format] - Document format: 'markdown', 'html', 'plain', or auto-detect.
13
+ * @returns {string[]} Array of text chunks.
14
+ */
15
+ export function structureAwareChunk(text: string, options?: {
16
+ chunkSize?: number;
17
+ chunkOverlap?: number;
18
+ format?: string;
19
+ }): string[];
@@ -0,0 +1,64 @@
1
+ /**
2
+ * Tracks citations between an LLM-generated answer and retrieved source documents.
3
+ * Emits a 'tracked' event when citation analysis completes.
4
+ *
5
+ * @extends EventEmitter
6
+ */
7
+ export class CitationTracker extends EventEmitter<[never]> {
8
+ /**
9
+ * @param {object} [options] - Override default configuration
10
+ * @param {number} [options.similarityThreshold=0.5] - Minimum similarity for a citation match
11
+ * @param {number} [options.maxCitationsPerSentence=3] - Max citations per sentence
12
+ * @param {boolean} [options.useIDFWeighting=true] - Whether to apply IDF weighting
13
+ */
14
+ constructor(options?: {
15
+ similarityThreshold?: number;
16
+ maxCitationsPerSentence?: number;
17
+ useIDFWeighting?: boolean;
18
+ });
19
+ config: any;
20
+ /**
21
+ * Normalize a document object to ensure it has a usable text content string.
22
+ * Looks for .content, .text, or .chunk fields. If .chunk is a function, calls it and joins the result.
23
+ * @param {object} doc
24
+ * @returns {object} Normalized doc with .content field
25
+ * @private
26
+ */
27
+ private _normalizeDoc;
28
+ /**
29
+ * Track citations for an answer against retrieved documents.
30
+ *
31
+ * @param {string} answer - The LLM-generated answer text
32
+ * @param {Array<object>} retrievedDocs - Array of source documents (each should have .content, .text, or .chunk)
33
+ * @returns {{
34
+ * citations: Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>,
35
+ * groundednessScore: number,
36
+ * hallucinationReport: object,
37
+ * metadata: {sentenceCount: number, docCount: number, config: object}
38
+ * }}
39
+ * @fires CitationTracker#tracked
40
+ */
41
+ track(answer: string, retrievedDocs: Array<object>): {
42
+ citations: Array<{
43
+ sentence: string;
44
+ sources: Array<{
45
+ docId: string | undefined;
46
+ docIndex: number;
47
+ score: number;
48
+ }>;
49
+ }>;
50
+ groundednessScore: number;
51
+ hallucinationReport: object;
52
+ metadata: {
53
+ sentenceCount: number;
54
+ docCount: number;
55
+ config: object;
56
+ };
57
+ };
58
+ }
59
+ /**
60
+ * Default configuration for the CitationTracker.
61
+ * @type {object}
62
+ */
63
+ export const DEFAULT_CONFIG: object;
64
+ import { EventEmitter } from "events";
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Detect hallucinations by analyzing citation results and classifying each sentence.
3
+ *
4
+ * Classification:
5
+ * - 'definite_hallucination': bestScore === 0 (no overlap with any source)
6
+ * - 'likely_hallucination': bestScore > 0 but below threshold
7
+ * - 'grounded': bestScore >= threshold
8
+ *
9
+ * @param {Array<{sentence: string, sources: Array<{docId: string|undefined, docIndex: number, score: number}>}>} citationResults
10
+ * @param {object} [options]
11
+ * @param {number} [options.threshold=0.5] - Score threshold for grounded classification
12
+ * @returns {{
13
+ * hallucinationRate: number,
14
+ * sentences: Array<{text: string, classification: string, bestScore: number, bestSource: object|null}>,
15
+ * summary: {grounded: number, likely_hallucination: number, definite_hallucination: number}
16
+ * }}
17
+ */
18
+ export function detectHallucinations(citationResults: Array<{
19
+ sentence: string;
20
+ sources: Array<{
21
+ docId: string | undefined;
22
+ docIndex: number;
23
+ score: number;
24
+ }>;
25
+ }>, options?: {
26
+ threshold?: number;
27
+ }): {
28
+ hallucinationRate: number;
29
+ sentences: Array<{
30
+ text: string;
31
+ classification: string;
32
+ bestScore: number;
33
+ bestSource: object | null;
34
+ }>;
35
+ summary: {
36
+ grounded: number;
37
+ likely_hallucination: number;
38
+ definite_hallucination: number;
39
+ };
40
+ };
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Map a sentence to its most relevant source documents by how much of the
3
+ * sentence each document supports (see computeSupportScore).
4
+ * @param {string} sentence - The sentence to map
5
+ * @param {Array<{content?: string, text?: string, id?: string}>} docs - Source documents
6
+ * @param {object} [options] - Configuration options
7
+ * @param {number} [options.maxCitations=3] - Maximum number of citations to return
8
+ * @param {number} [options.threshold=0.5] - Minimum similarity threshold
9
+ * @param {Map<string, number>|null} [options.idfWeights=null] - Optional IDF weights for weighted similarity
10
+ * @returns {Array<{docIndex: number, docId: string|undefined, score: number, matchedTokens: string[]}>}
11
+ */
12
+ export function mapSentenceToSources(sentence: string, docs: Array<{
13
+ content?: string;
14
+ text?: string;
15
+ id?: string;
16
+ }>, options?: {
17
+ maxCitations?: number;
18
+ threshold?: number;
19
+ idfWeights?: Map<string, number> | null;
20
+ }): Array<{
21
+ docIndex: number;
22
+ docId: string | undefined;
23
+ score: number;
24
+ matchedTokens: string[];
25
+ }>;
26
+ /**
27
+ * Build IDF (Inverse Document Frequency) weights from a collection of documents.
28
+ * IDF = log(1 + totalDocs / docsContainingToken). The +1 keeps weights
29
+ * positive, so tokens found in every document (and every token when only one
30
+ * document was retrieved) still count instead of weighing zero.
31
+ * @param {Array<{content?: string, text?: string}>} docs - Array of document objects
32
+ * @returns {Map<string, number>} Map of token to IDF weight
33
+ */
34
+ export function buildIDFWeights(docs: Array<{
35
+ content?: string;
36
+ text?: string;
37
+ }>): Map<string, number>;
@@ -0,0 +1,92 @@
1
+ /**
2
+ * LLM connector for the Anthropic Claude API.
3
+ * Uses axios for HTTP calls and supports streaming via SSE.
4
+ *
5
+ * @extends BaseConnector
6
+ */
7
+ export class AnthropicConnector extends BaseConnector {
8
+ /**
9
+ * @param {Object} [options]
10
+ * @param {string} [options.model='claude-3-5-sonnet-20241022'] - Claude model to use
11
+ * @param {string} [options.apiKey] - Anthropic API key (defaults to ANTHROPIC_API_KEY at construction)
12
+ * @param {string} [options.baseURL='https://api.anthropic.com'] - API base URL
13
+ * @param {string} [options.apiVersion='2023-06-01'] - API version header
14
+ * @param {number} [options.maxTokens=4096] - Max tokens in response
15
+ * @param {number} [options.temperature=0.7] - Sampling temperature
16
+ * @param {number} [options.maxRetries=3] - Retries for transient failures
17
+ * @param {number} [options.retryDelay=1000] - Base backoff delay in ms
18
+ * @param {string} [options.systemPrompt] - System prompt used when RAG context is supplied
19
+ */
20
+ constructor(options?: {
21
+ model?: string;
22
+ apiKey?: string;
23
+ baseURL?: string;
24
+ apiVersion?: string;
25
+ maxTokens?: number;
26
+ temperature?: number;
27
+ maxRetries?: number;
28
+ retryDelay?: number;
29
+ systemPrompt?: string;
30
+ });
31
+ config: any;
32
+ client: axios.AxiosInstance;
33
+ /**
34
+ * @throws {Error} code MISSING_API_KEY when no key is configured
35
+ * @private
36
+ */
37
+ private _requireApiKey;
38
+ /**
39
+ * Build the Messages API request body. Retrieved context goes into the
40
+ * user message as delimited, untrusted reference data; the system prompt
41
+ * only carries instructions.
42
+ * @private
43
+ */
44
+ private _buildBody;
45
+ /**
46
+ * Generate a response from Claude using the provided query and optional RAG context.
47
+ *
48
+ * @param {string} query - The user query / prompt
49
+ * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
50
+ * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
51
+ * @param {AbortSignal} [options.signal] - Cancels the request
52
+ * @returns {Promise<string>} The generated text
53
+ */
54
+ generate(query: string, context?: Array<any | string>, options?: {
55
+ signal?: AbortSignal;
56
+ }): Promise<string>;
57
+ /**
58
+ * Stream a response from Claude token-by-token via SSE.
59
+ *
60
+ * @param {string} query - The user query / prompt
61
+ * @param {Array<Object|string>} [context=[]] - Retrieved documents for RAG
62
+ * @param {Object} [options={}] - Override model, temperature, maxTokens per-call
63
+ * @param {AbortSignal} [options.signal] - Cancels the stream
64
+ * @yields {{ token: string, done: boolean }}
65
+ */
66
+ generateStream(query: string, context?: Array<any | string>, options?: {
67
+ signal?: AbortSignal;
68
+ }): AsyncGenerator<{
69
+ token: any;
70
+ done: boolean;
71
+ }, void, unknown>;
72
+ /**
73
+ * Build a single prompt string that includes RAG context when available.
74
+ * Kept for backward compatibility; the context is delimited and labelled
75
+ * as untrusted reference data.
76
+ *
77
+ * @param {string} query - User query
78
+ * @param {Array<Object|string>} context - RAG context documents
79
+ * @returns {string} Combined prompt
80
+ * @private
81
+ */
82
+ private _buildPrompt;
83
+ }
84
+ /**
85
+ * Default configuration for the Anthropic connector.
86
+ * `apiKey` is a getter so the environment is read when a connector is
87
+ * constructed, not when this module is first required.
88
+ * @type {Object}
89
+ */
90
+ export const DEFAULT_CONFIG: any;
91
+ import { BaseConnector } from "./base-connector";
92
+ import axios = require("axios");
@@ -0,0 +1,87 @@
1
+ /**
2
+ * Abstract base class for all connectors.
3
+ * Provides connection lifecycle, health checking, and retry logic.
4
+ * @extends EventEmitter
5
+ */
6
+ export class BaseConnector extends EventEmitter<[never]> {
7
+ /**
8
+ * @param {Object} options
9
+ * @param {string} [options.name] - Connector display name
10
+ */
11
+ constructor(options?: {
12
+ name?: string;
13
+ });
14
+ options: {
15
+ name?: string;
16
+ };
17
+ connected: boolean;
18
+ name: string;
19
+ /**
20
+ * Establish the connection.
21
+ * Override in subclass for custom connection logic.
22
+ * @returns {Promise<void>}
23
+ */
24
+ connect(): Promise<void>;
25
+ /**
26
+ * Tear down the connection.
27
+ * Override in subclass for custom disconnection logic.
28
+ * @returns {Promise<void>}
29
+ */
30
+ disconnect(): Promise<void>;
31
+ /**
32
+ * Check whether the connector is healthy and operational.
33
+ * Override in subclass for custom health checks.
34
+ * @returns {Promise<boolean>}
35
+ */
36
+ isHealthy(): Promise<boolean>;
37
+ /**
38
+ * Retry wrapper for transient failures.
39
+ *
40
+ * Only retryable errors are retried: network errors and HTTP
41
+ * 408/409/425/429/5xx (see {@link isRetryableError}). Other errors
42
+ * (400, 401, 403, 404, aborts, errors with no status or network code) are
43
+ * thrown immediately. Backoff is exponential with full jitter, starting at
44
+ * `delay` and capped at `opts.maxDelay`; a Retry-After header on the error
45
+ * is honoured (capped at `opts.maxDelay`). Emits a 'retry' event before
46
+ * each retry.
47
+ *
48
+ * @param {Function} fn - Async function to execute; receives the attempt number
49
+ * @param {number} [retries=3] - Maximum number of retry attempts
50
+ * @param {number} [delay=1000] - Base backoff delay in ms
51
+ * @param {object} [opts]
52
+ * @param {AbortSignal} [opts.signal] - Aborts the call and any pending backoff
53
+ * @param {number} [opts.maxDelay=30000] - Upper bound for a single backoff delay
54
+ * @param {(err: *) => boolean} [opts.shouldRetry] - Override retry classification
55
+ * @returns {Promise<*>} Result of the function
56
+ * @throws {Error} The first non-retryable error, or the last error once retries are exhausted
57
+ */
58
+ withRetry(fn: Function, retries?: number, delay?: number, opts?: {
59
+ signal?: AbortSignal;
60
+ maxDelay?: number;
61
+ shouldRetry?: (err: any) => boolean;
62
+ }): Promise<any>;
63
+ /**
64
+ * Wait between retries. Overridable in tests.
65
+ * @param {number} ms
66
+ * @param {AbortSignal} [signal]
67
+ * @returns {Promise<void>}
68
+ * @protected
69
+ */
70
+ protected _sleep(ms: number, signal?: AbortSignal): Promise<void>;
71
+ /**
72
+ * Jitter source for retry backoff. Overridable in tests.
73
+ * @returns {number} Value in [0, 1)
74
+ * @protected
75
+ */
76
+ protected _random(): number;
77
+ /**
78
+ * Return basic information about the connector.
79
+ * @returns {{ name: string, connected: boolean, type: string }}
80
+ */
81
+ getInfo(): {
82
+ name: string;
83
+ connected: boolean;
84
+ type: string;
85
+ };
86
+ }
87
+ import { EventEmitter } from "events";