@devilsdev/rag-pipeline-utils 2.1.11 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/.ragrc.schema.json +218 -0
  2. package/README.md +651 -393
  3. package/bin/cli.js +8 -14
  4. package/dist/index.d.ts +237 -0
  5. package/dist/index.mjs +22 -7
  6. package/package.json +17 -13
  7. package/src/ai/adaptive-retrieval.js +191 -0
  8. package/src/ai/federated-learning.js +214 -0
  9. package/src/ai/federation/federated-learning-coordinator.js +345 -0
  10. package/src/ai/federation.js +76 -0
  11. package/src/ai/index-fixed.js +509 -0
  12. package/src/ai/index.js +33 -0
  13. package/src/ai/model-trainer.js +217 -0
  14. package/src/ai/model-training.js +205 -0
  15. package/src/ai/module-template.js +37 -0
  16. package/src/ai/multimodal/multi-modal-processor.js +345 -0
  17. package/src/ai/multimodal-processing.js +268 -0
  18. package/src/ai/multimodal.js +71 -0
  19. package/src/ai/orchestrator.js +70 -0
  20. package/src/ai/retrieval/adaptive-retrieval-engine.js +398 -0
  21. package/src/ai/retrieval-engine.js +69 -0
  22. package/src/ai/training/model-training-orchestrator.js +228 -0
  23. package/src/ci/workflow-analyzer.js +71 -0
  24. package/src/cli/commands/ai-ml.js +967 -0
  25. package/src/cli/commands/docs.js +228 -0
  26. package/src/cli/commands/dx.js +538 -0
  27. package/src/cli/commands/plugin-hub.js +875 -0
  28. package/src/cli/doctor-command.js +639 -0
  29. package/src/cli/enhanced-cli-commands.js +945 -0
  30. package/src/cli/interactive-wizard.js +997 -0
  31. package/src/cli/plugin-marketplace-commands.js +869 -0
  32. package/src/cli.js +17 -0
  33. package/src/config/enhanced-ragrc-schema.js +632 -0
  34. package/src/config/load-config.js +54 -0
  35. package/src/config/load-plugin-config.js +69 -0
  36. package/src/config/normalize-config.js +105 -0
  37. package/src/config/schema-validator.js +315 -0
  38. package/src/config/validate-plugin-schema.js +35 -0
  39. package/src/config/validate-schema.js +113 -0
  40. package/src/contracts/plugin-contracts.js +33 -0
  41. package/src/core/create-pipeline.js +99 -0
  42. package/src/core/dependency-injection.js +242 -0
  43. package/src/core/observability/event-logger.js +241 -0
  44. package/src/core/observability/instrumented-pipeline.js +609 -0
  45. package/src/core/observability/metrics.js +497 -0
  46. package/src/core/observability/tracing.js +781 -0
  47. package/src/core/performance/benchmark.js +516 -0
  48. package/src/core/performance/parallel-processor.js +240 -0
  49. package/src/core/performance/streaming-safeguards.js +611 -0
  50. package/src/core/pipeline-factory.js +60 -0
  51. package/src/core/plugin-contracts.js +89 -0
  52. package/src/core/plugin-marketplace/plugin-metadata.js +485 -0
  53. package/src/core/plugin-marketplace/plugin-publisher.js +574 -0
  54. package/src/core/plugin-marketplace/plugin-registry-format.js +419 -0
  55. package/src/core/plugin-marketplace/version-resolver.js +61 -0
  56. package/src/core/plugin-registry.js +265 -0
  57. package/src/dag/dag-engine.js +954 -0
  58. package/src/dag/dag-node.js +76 -0
  59. package/src/dag/enhanced-dag-engine.js +822 -0
  60. package/src/dx/index.js +40 -0
  61. package/src/dx/integration-templates.js +665 -0
  62. package/src/dx/performance-profiler.js +968 -0
  63. package/src/dx/realtime-debugger.js +755 -0
  64. package/src/dx/visual-pipeline-builder.js +578 -0
  65. package/src/ecosystem/plugin-analytics-dashboard.js +778 -0
  66. package/src/ecosystem/plugin-certification.js +580 -0
  67. package/src/ecosystem/plugin-hub.js +821 -0
  68. package/src/enterprise/audit-logging.js +967 -0
  69. package/src/enterprise/data-governance.js +408 -0
  70. package/src/enterprise/multi-tenancy.js +496 -0
  71. package/src/enterprise/sso-integration.js +903 -0
  72. package/src/evaluate/evaluator.js +52 -0
  73. package/src/evaluate/scoring.js +91 -0
  74. package/src/index.js +67 -0
  75. package/src/ingest.js +59 -0
  76. package/src/loader/csv-loader.js +38 -0
  77. package/src/loader/directory-loader.js +48 -0
  78. package/src/loader/html-loader.js +62 -0
  79. package/src/loader/markdown-loader.js +67 -0
  80. package/src/mocks/openai-embedder.js +119 -0
  81. package/src/mocks/openai-llm.js +77 -0
  82. package/src/mocks/pdf-loader.js +21 -0
  83. package/src/mocks/pinecone-retriever.js +106 -0
  84. package/src/observability/audit-logger.js +758 -0
  85. package/src/observability/dashboard-config.json +675 -0
  86. package/src/observability/event-logger.js +310 -0
  87. package/src/observability/metrics.js +691 -0
  88. package/src/observability/security-monitor.js +830 -0
  89. package/src/observability/slo-monitor.js +319 -0
  90. package/src/observability/telemetry.js +686 -0
  91. package/src/pipeline/pipeline.js +18 -0
  92. package/src/plugins/sample-embedder.js +53 -0
  93. package/src/plugins/sample-llm.js +30 -0
  94. package/src/plugins/sample-loader.js +24 -0
  95. package/src/plugins/sample-retriever.js +84 -0
  96. package/src/query.js +117 -0
  97. package/src/reranker/llm-reranker.js +85 -0
  98. package/src/scripts/script-utilities.js +36 -0
  99. package/src/security/plugin-sandbox.js +586 -0
  100. package/src/security/plugin-signature-verifier.js +414 -0
  101. package/src/security/validator.js +36 -0
  102. package/src/utils/ci/diagnostic-reporter.js +57 -0
  103. package/src/utils/logger.js +211 -0
  104. package/src/utils/performance-monitor.js +561 -0
  105. package/src/utils/plugin-scaffolder.js +447 -0
  106. package/src/utils/retry-policy.js +443 -0
  107. package/src/utils/retry.js +107 -0
  108. package/src/utils/safe-expression-evaluator.js +579 -0
  109. package/src/utils/structured-logger.js +361 -0
  110. package/src/utils/validate-plugin-contract.js +42 -0
package/.ragrc.schema.json ADDED
@@ -0,0 +1,218 @@
1
+ {
2
+ "$schema": "http://json-schema.org/draft-07/schema#",
3
+ "$id": "https://raw.githubusercontent.com/DevilsDev/rag-pipeline-utils/main/.ragrc.schema.json",
4
+ "title": "RAG Pipeline Configuration Schema",
5
+ "description": "Configuration schema for @devilsdev/rag-pipeline-utils",
6
+ "type": "object",
7
+ "required": ["plugins"],
8
+ "properties": {
9
+ "plugins": {
10
+ "type": "object",
11
+ "description": "Plugin specifications for RAG pipeline components",
12
+ "required": ["loader", "embedder", "retriever", "llm"],
13
+ "properties": {
14
+ "loader": {
15
+ "type": "object",
16
+ "description": "Document loader plugins",
17
+ "minProperties": 1,
18
+ "additionalProperties": {
19
+ "oneOf": [
20
+ { "type": "string" },
21
+ {
22
+ "type": "object",
23
+ "required": ["name"],
24
+ "properties": {
25
+ "name": { "type": "string" },
26
+ "version": { "type": "string" },
27
+ "source": {
28
+ "type": "string",
29
+ "enum": ["registry", "local", "git", "npm"]
30
+ },
31
+ "path": { "type": "string" },
32
+ "config": { "type": "object" },
33
+ "enabled": { "type": "boolean" }
34
+ }
35
+ }
36
+ ]
37
+ }
38
+ },
39
+ "embedder": {
40
+ "type": "object",
41
+ "description": "Text embedding plugins",
42
+ "minProperties": 1,
43
+ "additionalProperties": {
44
+ "oneOf": [
45
+ { "type": "string" },
46
+ {
47
+ "type": "object",
48
+ "required": ["name"],
49
+ "properties": {
50
+ "name": { "type": "string" },
51
+ "version": { "type": "string" },
52
+ "source": {
53
+ "type": "string",
54
+ "enum": ["registry", "local", "git", "npm"]
55
+ },
56
+ "path": { "type": "string" },
57
+ "config": { "type": "object" },
58
+ "enabled": { "type": "boolean" }
59
+ }
60
+ }
61
+ ]
62
+ }
63
+ },
64
+ "retriever": {
65
+ "type": "object",
66
+ "description": "Vector retrieval plugins",
67
+ "minProperties": 1,
68
+ "additionalProperties": {
69
+ "oneOf": [
70
+ { "type": "string" },
71
+ {
72
+ "type": "object",
73
+ "required": ["name"],
74
+ "properties": {
75
+ "name": { "type": "string" },
76
+ "version": { "type": "string" },
77
+ "source": {
78
+ "type": "string",
79
+ "enum": ["registry", "local", "git", "npm"]
80
+ },
81
+ "path": { "type": "string" },
82
+ "config": { "type": "object" },
83
+ "enabled": { "type": "boolean" }
84
+ }
85
+ }
86
+ ]
87
+ }
88
+ },
89
+ "llm": {
90
+ "type": "object",
91
+ "description": "Language model plugins",
92
+ "minProperties": 1,
93
+ "additionalProperties": {
94
+ "oneOf": [
95
+ { "type": "string" },
96
+ {
97
+ "type": "object",
98
+ "required": ["name"],
99
+ "properties": {
100
+ "name": { "type": "string" },
101
+ "version": { "type": "string" },
102
+ "source": {
103
+ "type": "string",
104
+ "enum": ["registry", "local", "git", "npm"]
105
+ },
106
+ "path": { "type": "string" },
107
+ "config": { "type": "object" },
108
+ "enabled": { "type": "boolean" }
109
+ }
110
+ }
111
+ ]
112
+ }
113
+ },
114
+ "reranker": {
115
+ "type": "object",
116
+ "description": "Result reranking plugins (optional)",
117
+ "minProperties": 1,
118
+ "additionalProperties": {
119
+ "oneOf": [
120
+ { "type": "string" },
121
+ {
122
+ "type": "object",
123
+ "required": ["name"],
124
+ "properties": {
125
+ "name": { "type": "string" },
126
+ "version": { "type": "string" },
127
+ "source": {
128
+ "type": "string",
129
+ "enum": ["registry", "local", "git", "npm"]
130
+ },
131
+ "path": { "type": "string" },
132
+ "config": { "type": "object" },
133
+ "enabled": { "type": "boolean" }
134
+ }
135
+ }
136
+ ]
137
+ }
138
+ }
139
+ },
140
+ "additionalProperties": false
141
+ },
142
+ "pipeline": {
143
+ "type": "object",
144
+ "description": "Pipeline execution configuration",
145
+ "properties": {
146
+ "stages": {
147
+ "type": "array",
148
+ "description": "Pipeline execution stages",
149
+ "items": {
150
+ "type": "string",
151
+ "enum": ["loader", "embedder", "retriever", "llm", "reranker"]
152
+ }
153
+ },
154
+ "mode": {
155
+ "type": "string",
156
+ "description": "Pipeline execution mode",
157
+ "enum": ["sequential", "parallel", "dag"],
158
+ "default": "sequential"
159
+ }
160
+ }
161
+ },
162
+ "performance": {
163
+ "type": "object",
164
+ "description": "Performance and optimization settings",
165
+ "properties": {
166
+ "parallel": {
167
+ "type": "object",
168
+ "properties": {
169
+ "enabled": { "type": "boolean", "default": false },
170
+ "maxConcurrency": { "type": "number", "default": 5 },
171
+ "batchSize": { "type": "number", "default": 10 }
172
+ }
173
+ },
174
+ "streaming": {
175
+ "type": "object",
176
+ "properties": {
177
+ "enabled": { "type": "boolean", "default": false },
178
+ "maxMemoryMB": { "type": "number", "default": 512 },
179
+ "bufferSize": { "type": "number", "default": 100 }
180
+ }
181
+ }
182
+ }
183
+ },
184
+ "observability": {
185
+ "type": "object",
186
+ "description": "Logging and monitoring configuration",
187
+ "properties": {
188
+ "logging": {
189
+ "type": "object",
190
+ "properties": {
191
+ "level": {
192
+ "type": "string",
193
+ "enum": ["debug", "info", "warn", "error"],
194
+ "default": "info"
195
+ },
196
+ "structured": { "type": "boolean", "default": true }
197
+ }
198
+ },
199
+ "metrics": {
200
+ "type": "object",
201
+ "properties": {
202
+ "enabled": { "type": "boolean", "default": false },
203
+ "exportFormat": {
204
+ "type": "string",
205
+ "enum": ["prometheus", "json", "statsd"],
206
+ "default": "json"
207
+ }
208
+ }
209
+ }
210
+ }
211
+ },
212
+ "namespace": {
213
+ "type": "string",
214
+ "description": "Configuration namespace for isolation"
215
+ }
216
+ },
217
+ "additionalProperties": false
218
+ }