@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +293 -361
  2. package/contracts/loader-contract.json +11 -6
  3. package/contracts/reranker-contract.json +18 -26
  4. package/dist/index.cjs +21190 -47
  5. package/dist/index.d.ts +298 -68
  6. package/dist/index.mjs +9 -76
  7. package/package.json +20 -8
  8. package/src/agentic/agentic-pipeline.js +235 -0
  9. package/src/agentic/iterative-retriever.js +213 -0
  10. package/src/agentic/query-planner.js +208 -0
  11. package/src/agentic/self-critique.js +102 -0
  12. package/src/ai/adaptive-retrieval.js +0 -3
  13. package/src/ai/federated-learning.js +0 -3
  14. package/src/ai/federation.js +1 -6
  15. package/src/ai/index.js +9 -20
  16. package/src/ai/model-training.js +0 -3
  17. package/src/ai/multimodal-processing.js +0 -3
  18. package/src/ai/multimodal.js +1 -6
  19. package/src/ai/orchestrator.js +1 -6
  20. package/src/ai/retrieval-engine.js +1 -6
  21. package/src/chunking/chunking-engine.js +142 -0
  22. package/src/chunking/strategies/fixed-size.js +54 -0
  23. package/src/chunking/strategies/recursive.js +127 -0
  24. package/src/chunking/strategies/semantic.js +128 -0
  25. package/src/chunking/strategies/sentence.js +32 -0
  26. package/src/chunking/strategies/structure-aware.js +138 -0
  27. package/src/citation/citation-tracker.js +161 -0
  28. package/src/citation/hallucination-detector.js +74 -0
  29. package/src/citation/source-mapper.js +95 -0
  30. package/src/cli/commands/ai-ml/adaptive-commands.js +178 -0
  31. package/src/cli/commands/ai-ml/federated-commands.js +207 -0
  32. package/src/cli/commands/ai-ml/index.js +17 -0
  33. package/src/cli/commands/ai-ml/multimodal-commands.js +187 -0
  34. package/src/cli/commands/ai-ml/shared.js +51 -0
  35. package/src/cli/commands/ai-ml/training-commands.js +228 -0
  36. package/src/cli/commands/ai-ml.js +176 -757
  37. package/src/cli/commands/plugin-hub/certification-commands.js +221 -0
  38. package/src/cli/commands/plugin-hub/event-handlers.js +64 -0
  39. package/src/cli/commands/plugin-hub/hub-commands.js +290 -0
  40. package/src/cli/commands/plugin-hub/review-commands.js +78 -0
  41. package/src/cli/commands/plugin-hub.js +106 -782
  42. package/src/cli/diagnostics/config-checker.js +81 -0
  43. package/src/cli/diagnostics/dependency-checker.js +69 -0
  44. package/src/cli/diagnostics/environment-checker.js +45 -0
  45. package/src/cli/diagnostics/performance-checker.js +51 -0
  46. package/src/cli/diagnostics/plugin-checker.js +106 -0
  47. package/src/cli/diagnostics/security-checker.js +61 -0
  48. package/src/cli/doctor-command.js +70 -369
  49. package/src/cli/enhanced-cli-commands.js +162 -706
  50. package/src/cli/handlers/config-handler.js +126 -0
  51. package/src/cli/handlers/diagnostics-handler.js +113 -0
  52. package/src/cli/handlers/ingest-handler.js +42 -0
  53. package/src/cli/handlers/init-handler.js +51 -0
  54. package/src/cli/handlers/query-handler.js +45 -0
  55. package/src/cli/handlers/run-handler.js +181 -0
  56. package/src/cli/interactive-wizard.js +99 -723
  57. package/src/cli/shared/cli-utils.js +89 -0
  58. package/src/cli/wizard/configuration.js +148 -0
  59. package/src/cli/wizard/observability.js +79 -0
  60. package/src/cli/wizard/performance.js +76 -0
  61. package/src/cli/wizard/plugin-selection.js +297 -0
  62. package/src/cli/wizard/preview-save.js +172 -0
  63. package/src/cli/wizard/project-setup.js +66 -0
  64. package/src/config/memory-optimization.js +3 -3
  65. package/src/connectors/anthropic-connector.js +179 -0
  66. package/src/connectors/base-connector.js +94 -0
  67. package/src/connectors/cohere-connector.js +143 -0
  68. package/src/connectors/local-embedder.js +171 -0
  69. package/src/connectors/memory-retriever.js +150 -0
  70. package/src/connectors/ollama-connector.js +205 -0
  71. package/src/connectors/openai-connector.js +173 -0
  72. package/src/core/create-pipeline.js +21 -1
  73. package/src/core/observability/tracing.js +2 -1
  74. package/src/core/pipeline-factory.js +0 -1
  75. package/src/core/plugin-registry.js +14 -14
  76. package/src/cost/cost-calculator.js +146 -0
  77. package/src/cost/cost-tracker.js +260 -0
  78. package/src/cost/token-budget.js +245 -0
  79. package/src/dag/core/execution-scheduler.js +2 -1
  80. package/src/dag/core/topology-validation.js +0 -3
  81. package/src/dag/dag-engine.js +3 -5
  82. package/src/dag/dag-node.js +0 -1
  83. package/src/dag/enhanced-dag-engine.js +171 -99
  84. package/src/dashboard/dashboard-generator.js +642 -0
  85. package/src/dashboard/metrics-aggregator.js +409 -0
  86. package/src/debug/execution-tracer.js +210 -0
  87. package/src/debug/trace-inspector.js +281 -0
  88. package/src/dev/dev-server.js +5 -5
  89. package/src/dev/hot-reload.js +3 -2
  90. package/src/dx/integration-templates.js +0 -3
  91. package/src/ecosystem/plugin-analytics-dashboard.js +2 -2
  92. package/src/ecosystem/plugin-hub.js +2 -2
  93. package/src/enterprise/audit-logging.js +2 -2
  94. package/src/enterprise/sso-integration.js +52 -32
  95. package/src/evaluate/context-metrics.js +127 -0
  96. package/src/evaluate/evaluator.js +4 -2
  97. package/src/evaluate/faithfulness.js +117 -0
  98. package/src/evaluate/groundedness.js +60 -0
  99. package/src/evaluate/pipeline-evaluator.js +156 -0
  100. package/src/evaluate/relevance.js +129 -0
  101. package/src/evaluate/scoring.js +27 -0
  102. package/src/graph/entity-extractor.js +183 -0
  103. package/src/graph/graph-index.js +113 -0
  104. package/src/graph/graph-retriever.js +244 -0
  105. package/src/graph/knowledge-graph.js +256 -0
  106. package/src/guardrails/guardrails-pipeline.js +136 -0
  107. package/src/guardrails/post-generation-guard.js +170 -0
  108. package/src/guardrails/pre-retrieval-guard.js +161 -0
  109. package/src/guardrails/retrieval-guard.js +126 -0
  110. package/src/index.js +255 -9
  111. package/src/loader/html-loader.js +4 -15
  112. package/src/loader/markdown-loader.js +4 -15
  113. package/src/mcp/mcp-server.js +199 -0
  114. package/src/mcp/mcp-tool-builder.js +121 -0
  115. package/src/observability/audit-logger.js +0 -2
  116. package/src/observability/event-logger.js +2 -1
  117. package/src/observability/security-monitor.js +0 -2
  118. package/src/observability/telemetry.js +0 -2
  119. package/src/reranker/cascade-reranker.js +79 -0
  120. package/src/reranker/embedding-reranker.js +124 -0
  121. package/src/reranker/llm-reranker.js +6 -3
  122. package/src/reranker/scoring-reranker.js +123 -0
  123. package/src/retrieval/bm25-search.js +157 -0
  124. package/src/retrieval/hybrid-retriever.js +111 -0
  125. package/src/retrieval/rank-fusion.js +62 -0
  126. package/src/security/jwt-config.js +4 -3
  127. package/src/security/jwt-validator.js +2 -1
  128. package/src/security/plugin-sandbox.js +0 -3
  129. package/src/security/plugin-signature-verifier.js +4 -6
  130. package/src/security/validator.js +0 -2
  131. package/src/streaming/embedding-stream.js +197 -0
  132. package/src/streaming/sse-adapter.js +168 -0
  133. package/src/streaming/stream-router.js +210 -0
  134. package/src/streaming/streaming-embedder.js +231 -0
  135. package/src/streaming/websocket-adapter.js +171 -0
  136. package/src/templates/scaffold.js +147 -0
  137. package/src/templates/template-registry.js +272 -0
  138. package/src/utils/contract-validator.js +2 -1
  139. package/src/utils/input-sanitizer.js +80 -99
  140. package/src/utils/performance-monitor.js +0 -3
  141. package/src/utils/plugin-scaffolder.js +9 -16
  142. package/src/utils/rate-limiter.js +0 -1
  143. package/src/utils/retry-policy.js +0 -3
  144. package/src/utils/safe-expression-evaluator.js +0 -3
  145. package/src/utils/secure-logger-config.js +2 -4
  146. package/src/utils/secure-logging.js +0 -3
  147. package/src/utils/structured-logger.js +0 -3
  148. package/src/ai/index-fixed.js +0 -509
  149. package/src/ai/module-template.js +0 -37
  150. package/src/ci/workflow-analyzer.js +0 -71
  151. package/src/cli/plugin-marketplace-commands.js +0 -869
  152. package/src/dev/README.md +0 -126
  153. package/src/dev/examples/hot-reload-usage.js +0 -355
  154. package/src/mocks/openai-embedder.js +0 -119
  155. package/src/mocks/openai-llm.js +0 -77
  156. package/src/mocks/pdf-loader.js +0 -21
  157. package/src/mocks/pinecone-retriever.js +0 -106
  158. package/src/scripts/script-utilities.js +0 -36
  159. package/src/utils/ci/diagnostic-reporter.js +0 -57
  160. package/src/utils/examples/error-formatter-usage.js +0 -351
package/contracts/loader-contract.json CHANGED
@@ -14,13 +14,13 @@
14
14
  "properties": {
15
15
  "load": {
16
16
  "type": "function",
17
- "description": "Load documents from source",
17
+ "description": "Load and parse a document from the given file path",
18
18
  "parameters": {
19
19
  "type": "object",
20
20
  "properties": {
21
21
  "source": {
22
22
  "type": "string",
23
- "description": "Source path or identifier"
23
+ "description": "File path to the document to load"
24
24
  }
25
25
  },
26
26
  "required": ["source"]
@@ -30,11 +30,16 @@
30
30
  "items": {
31
31
  "type": "object",
32
32
  "properties": {
33
- "id": { "type": "string" },
34
- "content": { "type": "string" },
35
- "metadata": { "type": "object" }
33
+ "chunk": {
34
+ "type": "function",
35
+ "description": "Split the document into text chunks",
36
+ "returns": {
37
+ "type": "array",
38
+ "items": { "type": "string" }
39
+ }
40
+ }
36
41
  },
37
- "required": ["id", "content"]
42
+ "required": ["chunk"]
38
43
  }
39
44
  }
40
45
  }
package/contracts/reranker-contract.json CHANGED
@@ -7,60 +7,52 @@
7
7
  "methods": [
8
8
  {
9
9
  "name": "rerank",
10
- "parameters": ["results", "query", "options"],
11
- "returns": "rankedResults"
10
+ "parameters": ["query", "documents"],
11
+ "returns": "documents"
12
12
  }
13
13
  ],
14
14
  "properties": {
15
15
  "rerank": {
16
16
  "type": "function",
17
- "description": "Rerank search results based on relevance to query",
17
+ "description": "Rerank documents based on relevance to query",
18
18
  "parameters": {
19
19
  "type": "object",
20
20
  "properties": {
21
- "results": {
21
+ "query": {
22
+ "type": "string",
23
+ "description": "The search query to rank documents against"
24
+ },
25
+ "documents": {
22
26
  "type": "array",
23
27
  "items": {
24
28
  "type": "object",
25
29
  "properties": {
26
- "id": { "type": "string" },
27
- "content": { "type": "string" },
28
- "score": { "type": "number" }
29
- },
30
- "required": ["id", "content"]
31
- }
32
- },
33
- "query": {
34
- "type": "string",
35
- "description": "The original search query"
30
+ "text": { "type": "string" },
31
+ "content": { "type": "string" }
32
+ }
33
+ },
34
+ "description": "Array of retrieved documents to rerank"
36
35
  },
37
36
  "options": {
38
37
  "type": "object",
39
38
  "properties": {
40
39
  "topK": {
41
40
  "type": "number",
42
- "description": "Number of top results to return"
43
- },
44
- "threshold": {
45
- "type": "number",
46
- "description": "Minimum relevance score threshold"
41
+ "description": "Maximum number of top documents to return"
47
42
  }
48
43
  }
49
44
  }
50
45
  },
51
- "required": ["results", "query"]
46
+ "required": ["query", "documents"]
52
47
  },
53
48
  "returns": {
54
49
  "type": "array",
55
50
  "items": {
56
51
  "type": "object",
57
52
  "properties": {
58
- "id": { "type": "string" },
59
- "content": { "type": "string" },
60
- "score": { "type": "number" },
61
- "relevanceScore": { "type": "number" }
62
- },
63
- "required": ["id", "content", "relevanceScore"]
53
+ "text": { "type": "string" },
54
+ "content": { "type": "string" }
55
+ }
64
56
  }
65
57
  }
66
58
  }