@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +293 -361
  2. package/contracts/loader-contract.json +11 -6
  3. package/contracts/reranker-contract.json +18 -26
  4. package/dist/index.cjs +21190 -47
  5. package/dist/index.d.ts +298 -68
  6. package/dist/index.mjs +9 -76
  7. package/package.json +20 -8
  8. package/src/agentic/agentic-pipeline.js +235 -0
  9. package/src/agentic/iterative-retriever.js +213 -0
  10. package/src/agentic/query-planner.js +208 -0
  11. package/src/agentic/self-critique.js +102 -0
  12. package/src/ai/adaptive-retrieval.js +0 -3
  13. package/src/ai/federated-learning.js +0 -3
  14. package/src/ai/federation.js +1 -6
  15. package/src/ai/index.js +9 -20
  16. package/src/ai/model-training.js +0 -3
  17. package/src/ai/multimodal-processing.js +0 -3
  18. package/src/ai/multimodal.js +1 -6
  19. package/src/ai/orchestrator.js +1 -6
  20. package/src/ai/retrieval-engine.js +1 -6
  21. package/src/chunking/chunking-engine.js +142 -0
  22. package/src/chunking/strategies/fixed-size.js +54 -0
  23. package/src/chunking/strategies/recursive.js +127 -0
  24. package/src/chunking/strategies/semantic.js +128 -0
  25. package/src/chunking/strategies/sentence.js +32 -0
  26. package/src/chunking/strategies/structure-aware.js +138 -0
  27. package/src/citation/citation-tracker.js +161 -0
  28. package/src/citation/hallucination-detector.js +74 -0
  29. package/src/citation/source-mapper.js +95 -0
  30. package/src/cli/commands/ai-ml/adaptive-commands.js +178 -0
  31. package/src/cli/commands/ai-ml/federated-commands.js +207 -0
  32. package/src/cli/commands/ai-ml/index.js +17 -0
  33. package/src/cli/commands/ai-ml/multimodal-commands.js +187 -0
  34. package/src/cli/commands/ai-ml/shared.js +51 -0
  35. package/src/cli/commands/ai-ml/training-commands.js +228 -0
  36. package/src/cli/commands/ai-ml.js +176 -757
  37. package/src/cli/commands/plugin-hub/certification-commands.js +221 -0
  38. package/src/cli/commands/plugin-hub/event-handlers.js +64 -0
  39. package/src/cli/commands/plugin-hub/hub-commands.js +290 -0
  40. package/src/cli/commands/plugin-hub/review-commands.js +78 -0
  41. package/src/cli/commands/plugin-hub.js +106 -782
  42. package/src/cli/diagnostics/config-checker.js +81 -0
  43. package/src/cli/diagnostics/dependency-checker.js +69 -0
  44. package/src/cli/diagnostics/environment-checker.js +45 -0
  45. package/src/cli/diagnostics/performance-checker.js +51 -0
  46. package/src/cli/diagnostics/plugin-checker.js +106 -0
  47. package/src/cli/diagnostics/security-checker.js +61 -0
  48. package/src/cli/doctor-command.js +70 -369
  49. package/src/cli/enhanced-cli-commands.js +162 -706
  50. package/src/cli/handlers/config-handler.js +126 -0
  51. package/src/cli/handlers/diagnostics-handler.js +113 -0
  52. package/src/cli/handlers/ingest-handler.js +42 -0
  53. package/src/cli/handlers/init-handler.js +51 -0
  54. package/src/cli/handlers/query-handler.js +45 -0
  55. package/src/cli/handlers/run-handler.js +181 -0
  56. package/src/cli/interactive-wizard.js +99 -723
  57. package/src/cli/shared/cli-utils.js +89 -0
  58. package/src/cli/wizard/configuration.js +148 -0
  59. package/src/cli/wizard/observability.js +79 -0
  60. package/src/cli/wizard/performance.js +76 -0
  61. package/src/cli/wizard/plugin-selection.js +297 -0
  62. package/src/cli/wizard/preview-save.js +172 -0
  63. package/src/cli/wizard/project-setup.js +66 -0
  64. package/src/config/memory-optimization.js +3 -3
  65. package/src/connectors/anthropic-connector.js +179 -0
  66. package/src/connectors/base-connector.js +94 -0
  67. package/src/connectors/cohere-connector.js +143 -0
  68. package/src/connectors/local-embedder.js +171 -0
  69. package/src/connectors/memory-retriever.js +150 -0
  70. package/src/connectors/ollama-connector.js +205 -0
  71. package/src/connectors/openai-connector.js +173 -0
  72. package/src/core/create-pipeline.js +21 -1
  73. package/src/core/observability/tracing.js +2 -1
  74. package/src/core/pipeline-factory.js +0 -1
  75. package/src/core/plugin-registry.js +14 -14
  76. package/src/cost/cost-calculator.js +146 -0
  77. package/src/cost/cost-tracker.js +260 -0
  78. package/src/cost/token-budget.js +245 -0
  79. package/src/dag/core/execution-scheduler.js +2 -1
  80. package/src/dag/core/topology-validation.js +0 -3
  81. package/src/dag/dag-engine.js +3 -5
  82. package/src/dag/dag-node.js +0 -1
  83. package/src/dag/enhanced-dag-engine.js +171 -99
  84. package/src/dashboard/dashboard-generator.js +642 -0
  85. package/src/dashboard/metrics-aggregator.js +409 -0
  86. package/src/debug/execution-tracer.js +210 -0
  87. package/src/debug/trace-inspector.js +281 -0
  88. package/src/dev/dev-server.js +5 -5
  89. package/src/dev/hot-reload.js +3 -2
  90. package/src/dx/integration-templates.js +0 -3
  91. package/src/ecosystem/plugin-analytics-dashboard.js +2 -2
  92. package/src/ecosystem/plugin-hub.js +2 -2
  93. package/src/enterprise/audit-logging.js +2 -2
  94. package/src/enterprise/sso-integration.js +52 -32
  95. package/src/evaluate/context-metrics.js +127 -0
  96. package/src/evaluate/evaluator.js +4 -2
  97. package/src/evaluate/faithfulness.js +117 -0
  98. package/src/evaluate/groundedness.js +60 -0
  99. package/src/evaluate/pipeline-evaluator.js +156 -0
  100. package/src/evaluate/relevance.js +129 -0
  101. package/src/evaluate/scoring.js +27 -0
  102. package/src/graph/entity-extractor.js +183 -0
  103. package/src/graph/graph-index.js +113 -0
  104. package/src/graph/graph-retriever.js +244 -0
  105. package/src/graph/knowledge-graph.js +256 -0
  106. package/src/guardrails/guardrails-pipeline.js +136 -0
  107. package/src/guardrails/post-generation-guard.js +170 -0
  108. package/src/guardrails/pre-retrieval-guard.js +161 -0
  109. package/src/guardrails/retrieval-guard.js +126 -0
  110. package/src/index.js +255 -9
  111. package/src/loader/html-loader.js +4 -15
  112. package/src/loader/markdown-loader.js +4 -15
  113. package/src/mcp/mcp-server.js +199 -0
  114. package/src/mcp/mcp-tool-builder.js +121 -0
  115. package/src/observability/audit-logger.js +0 -2
  116. package/src/observability/event-logger.js +2 -1
  117. package/src/observability/security-monitor.js +0 -2
  118. package/src/observability/telemetry.js +0 -2
  119. package/src/reranker/cascade-reranker.js +79 -0
  120. package/src/reranker/embedding-reranker.js +124 -0
  121. package/src/reranker/llm-reranker.js +6 -3
  122. package/src/reranker/scoring-reranker.js +123 -0
  123. package/src/retrieval/bm25-search.js +157 -0
  124. package/src/retrieval/hybrid-retriever.js +111 -0
  125. package/src/retrieval/rank-fusion.js +62 -0
  126. package/src/security/jwt-config.js +4 -3
  127. package/src/security/jwt-validator.js +2 -1
  128. package/src/security/plugin-sandbox.js +0 -3
  129. package/src/security/plugin-signature-verifier.js +4 -6
  130. package/src/security/validator.js +0 -2
  131. package/src/streaming/embedding-stream.js +197 -0
  132. package/src/streaming/sse-adapter.js +168 -0
  133. package/src/streaming/stream-router.js +210 -0
  134. package/src/streaming/streaming-embedder.js +231 -0
  135. package/src/streaming/websocket-adapter.js +171 -0
  136. package/src/templates/scaffold.js +147 -0
  137. package/src/templates/template-registry.js +272 -0
  138. package/src/utils/contract-validator.js +2 -1
  139. package/src/utils/input-sanitizer.js +80 -99
  140. package/src/utils/performance-monitor.js +0 -3
  141. package/src/utils/plugin-scaffolder.js +9 -16
  142. package/src/utils/rate-limiter.js +0 -1
  143. package/src/utils/retry-policy.js +0 -3
  144. package/src/utils/safe-expression-evaluator.js +0 -3
  145. package/src/utils/secure-logger-config.js +2 -4
  146. package/src/utils/secure-logging.js +0 -3
  147. package/src/utils/structured-logger.js +0 -3
  148. package/src/ai/index-fixed.js +0 -509
  149. package/src/ai/module-template.js +0 -37
  150. package/src/ci/workflow-analyzer.js +0 -71
  151. package/src/cli/plugin-marketplace-commands.js +0 -869
  152. package/src/dev/README.md +0 -126
  153. package/src/dev/examples/hot-reload-usage.js +0 -355
  154. package/src/mocks/openai-embedder.js +0 -119
  155. package/src/mocks/openai-llm.js +0 -77
  156. package/src/mocks/pdf-loader.js +0 -21
  157. package/src/mocks/pinecone-retriever.js +0 -106
  158. package/src/scripts/script-utilities.js +0 -36
  159. package/src/utils/ci/diagnostic-reporter.js +0 -57
  160. package/src/utils/examples/error-formatter-usage.js +0 -351
@@ -1,40 +1,65 @@
1
1
  /**
2
2
  * Enhanced CLI Commands with Additional Flags and Features
3
3
  * Provides improved developer experience with dry-run, version display, and enhanced help
4
+ *
5
+ * This is the thin orchestrator that delegates to focused handler modules.
4
6
  */
5
7
 
6
- const { Command } = require('commander');
7
- const fs = require('fs/promises');
8
- const path = require('path');
9
- const pkg = require('../../package.json');
10
-
11
- const { logger } = require('../utils/logger.js');
8
+ const { Command } = require("commander");
9
+ const fs = require("fs/promises");
10
+
11
+ const { logger } = require("../utils/logger.js");
12
+
13
+ // Shared utilities
14
+ const {
15
+ createProgressBar,
16
+ parsePositiveInteger,
17
+ generateCompletionScript,
18
+ getVersion,
19
+ getExtendedHelp,
20
+ } = require("./shared/cli-utils.js");
21
+
22
+ // Handler modules
23
+ const { handleRun } = require("./handlers/run-handler.js");
24
+ const { handleInit } = require("./handlers/init-handler.js");
25
+ const {
26
+ handleDoctor,
27
+ handleValidate,
28
+ handleInfo,
29
+ } = require("./handlers/diagnostics-handler.js");
30
+ const {
31
+ handleConfigShow,
32
+ handleConfigSet,
33
+ handleConfigGet,
34
+ } = require("./handlers/config-handler.js");
35
+ const { handleIngest } = require("./handlers/ingest-handler.js");
36
+ const { handleQuery } = require("./handlers/query-handler.js");
12
37
 
13
38
  // Mock dependencies for test compatibility
14
39
  const mockPluginMarketplaceCommands = () => {
15
- const cmd = new Command('plugin');
16
- cmd.description('Plugin marketplace commands');
40
+ const cmd = new Command("plugin");
41
+ cmd.description("Plugin marketplace commands");
17
42
  cmd
18
- .command('search')
19
- .argument('[query]')
43
+ .command("search")
44
+ .argument("[query]")
20
45
  .action(() => {});
21
46
  cmd
22
- .command('install')
23
- .argument('<name>')
47
+ .command("install")
48
+ .argument("<name>")
24
49
  .action(() => {});
25
- cmd.command('publish').action(() => {});
50
+ cmd.command("publish").action(() => {});
26
51
  return cmd;
27
52
  };
28
53
 
29
54
  const mockPipelineDoctor = async (options) => {
30
- console.log('🏥 Running pipeline diagnostics');
55
+ console.log("🏥 Running pipeline diagnostics");
31
56
  if (options.categories) {
32
57
  options.categories.forEach((cat) => {
33
58
  console.log(`${cat.charAt(0).toUpperCase() + cat.slice(1)} Issues:`);
34
- console.log(' ✅ No issues found');
59
+ console.log(" ✅ No issues found");
35
60
  });
36
61
  } else {
37
- console.log('✅ All systems operational');
62
+ console.log("✅ All systems operational");
38
63
  }
39
64
  return {
40
65
  summary: { total: 0, passed: 0, warnings: 0, errors: 0 },
@@ -44,19 +69,19 @@ const mockPipelineDoctor = async (options) => {
44
69
  };
45
70
 
46
71
  const mockValidateSchema = (config) => {
47
- if (!config || typeof config !== 'object') {
72
+ if (!config || typeof config !== "object") {
48
73
  return {
49
74
  valid: false,
50
- errors: [{ message: 'Invalid configuration format' }],
75
+ errors: [{ message: "Invalid configuration format" }],
51
76
  };
52
77
  }
53
- if (config.plugins && typeof config.plugins === 'string') {
54
- return { valid: false, errors: [{ message: 'Invalid plugins format' }] };
78
+ if (config.plugins && typeof config.plugins === "string") {
79
+ return { valid: false, errors: [{ message: "Invalid plugins format" }] };
55
80
  }
56
81
  const isLegacy =
57
82
  config.plugins &&
58
- typeof config.plugins === 'object' &&
59
- Object.values(config.plugins).some((p) => typeof p === 'string');
83
+ typeof config.plugins === "object" &&
84
+ Object.values(config.plugins).some((p) => typeof p === "string");
60
85
  return { valid: true, legacy: isLegacy };
61
86
  };
62
87
 
@@ -70,50 +95,22 @@ class EnhancedCLI {
70
95
  this.setupCommands();
71
96
  }
72
97
 
73
- /**
74
- * Custom parser for positive integers
75
- */
76
- parsePositiveInteger(value) {
77
- const parsed = parseInt(value, 10);
78
- if (isNaN(parsed) || parsed <= 0) {
79
- throw new Error(`Invalid value '${value}'. Expected positive integer.`);
80
- }
81
- return parsed;
82
- }
83
-
84
- /**
85
- * Create a simple progress bar for CLI output
86
- * @param {number} percentage - Progress percentage (0-100)
87
- * @param {number} width - Width of progress bar in characters
88
- * @returns {string} Progress bar string
89
- */
90
- createProgressBar(percentage, width = 20) {
91
- const filled = Math.round((percentage / 100) * width);
92
- const empty = width - filled;
93
- const bar = '█'.repeat(filled) + '░'.repeat(empty);
94
- return `[${bar}]`;
95
- }
96
-
97
98
  /**
98
99
  * Setup global CLI options
99
100
  */
100
101
  setupGlobalOptions() {
101
102
  this.program
102
- .name('rag-pipeline')
103
- .description('Enterprise-grade RAG pipeline toolkit')
104
- .version(
105
- this.getVersion(),
106
- '-v, --version',
107
- 'Display version information',
108
- )
109
- .option('--config <path>', 'Configuration file path', '.ragrc.json')
110
- .option('--verbose', 'Enable verbose output')
111
- .option('--quiet', 'Suppress non-essential output')
112
- .option('--no-color', 'Disable colored output')
113
- .option('--dry-run', 'Show what would be done without executing')
114
- .helpOption('-h, --help', 'Display help information')
103
+ .name("rag-pipeline")
104
+ .description("Enterprise-grade RAG pipeline toolkit")
105
+ .version(getVersion(), "-v, --version", "Display version information")
106
+ .option("--config <path>", "Configuration file path", ".ragrc.json")
107
+ .option("--verbose", "Enable verbose output")
108
+ .option("--quiet", "Suppress non-essential output")
109
+ .option("--no-color", "Disable colored output")
110
+ .option("--dry-run", "Show what would be done without executing")
111
+ .helpOption("-h, --help", "Display help information")
115
112
  .showHelpAfterError()
116
- .addHelpText('after', this.getExtendedHelp());
113
+ .addHelpText("after", getExtendedHelp());
117
114
  }
118
115
 
119
116
  /**
@@ -122,672 +119,183 @@ class EnhancedCLI {
122
119
  setupCommands() {
123
120
  // Interactive initialization command
124
121
  this.program
125
- .command('init')
126
- .description('Initialize RAG pipeline configuration')
127
- .option('--interactive', 'Use interactive wizard', true)
128
- .option('--no-interactive', 'Skip interactive wizard')
129
- .option('--template <name>', 'Use configuration template')
130
- .option('--output <path>', 'Output configuration file', '.ragrc.json')
131
- .option('--force', 'Overwrite existing configuration')
122
+ .command("init")
123
+ .description("Initialize RAG pipeline configuration")
124
+ .option("--interactive", "Use interactive wizard", true)
125
+ .option("--no-interactive", "Skip interactive wizard")
126
+ .option("--template <name>", "Use configuration template")
127
+ .option("--output <path>", "Output configuration file", ".ragrc.json")
128
+ .option("--force", "Overwrite existing configuration")
132
129
  .action(async (options) => {
133
- await this.handleInit(options);
130
+ await handleInit(this.program.opts(), options, (outputPath) =>
131
+ this.createBasicConfig(outputPath),
132
+ );
134
133
  });
135
134
 
136
135
  // Run pipeline command
137
136
  this.program
138
- .command('run')
139
- .description('Execute RAG pipeline operations')
140
- .option('--ingest <file>', 'Ingest document into pipeline')
141
- .option('--query <prompt>', 'Query the pipeline')
142
- .option('--stream', 'Enable streaming output')
137
+ .command("run")
138
+ .description("Execute RAG pipeline operations")
139
+ .option("--ingest <file>", "Ingest document into pipeline")
140
+ .option("--query <prompt>", "Query the pipeline")
141
+ .option("--stream", "Enable streaming output")
143
142
  .option(
144
- '--abort-timeout <ms>',
145
- 'Abort operation after timeout (milliseconds)',
146
- this.parsePositiveInteger,
143
+ "--abort-timeout <ms>",
144
+ "Abort operation after timeout (milliseconds)",
145
+ parsePositiveInteger,
147
146
  )
148
- .option('--parallel', 'Enable parallel processing')
147
+ .option("--parallel", "Enable parallel processing")
149
148
  .action(async (options) => {
150
- await this.handleRun(options);
149
+ await handleRun(this.program.opts(), options);
151
150
  });
152
151
 
153
152
  // Doctor command for diagnostics
154
153
  this.program
155
- .command('doctor')
156
- .description('Diagnose and fix common issues')
157
- .option('--category <categories...>', 'Diagnostic categories to check')
158
- .option('--auto-fix', 'Automatically fix issues where possible')
159
- .option('--report <path>', 'Save diagnostic report to file')
154
+ .command("doctor")
155
+ .description("Diagnose and fix common issues")
156
+ .option("--category <categories...>", "Diagnostic categories to check")
157
+ .option("--auto-fix", "Automatically fix issues where possible")
158
+ .option("--report <path>", "Save diagnostic report to file")
160
159
  .action(async (options) => {
161
- await this.handleDoctor(options);
160
+ await handleDoctor(this.program.opts(), options, mockPipelineDoctor);
162
161
  });
163
162
 
164
163
  // Enhanced ingest command
165
164
  this.program
166
- .command('ingest')
167
- .description('Ingest documents into the pipeline')
168
- .argument('<file>', 'Document file to ingest')
169
- .option('--validate', 'Validate document before ingesting')
170
- .option('--preview', 'Show preview of document chunks')
171
- .option('--parallel', 'Enable parallel processing')
172
- .option('--streaming', 'Enable streaming with memory safeguards')
173
- .option('--trace', 'Enable detailed tracing and event logging')
174
- .option('--stats', 'Show pipeline metrics summary at the end')
165
+ .command("ingest")
166
+ .description("Ingest documents into the pipeline")
167
+ .argument("<file>", "Document file to ingest")
168
+ .option("--validate", "Validate document before ingesting")
169
+ .option("--preview", "Show preview of document chunks")
170
+ .option("--parallel", "Enable parallel processing")
171
+ .option("--streaming", "Enable streaming with memory safeguards")
172
+ .option("--trace", "Enable detailed tracing and event logging")
173
+ .option("--stats", "Show pipeline metrics summary at the end")
175
174
  .option(
176
- '--export-observability <file>',
177
- 'Export observability data to file',
175
+ "--export-observability <file>",
176
+ "Export observability data to file",
178
177
  )
179
178
  .option(
180
- '--max-concurrency <number>',
181
- 'Maximum concurrent operations',
182
- '3',
179
+ "--max-concurrency <number>",
180
+ "Maximum concurrent operations",
181
+ "3",
183
182
  )
184
183
  .option(
185
- '--batch-size <number>',
186
- 'Batch size for parallel processing',
187
- '10',
184
+ "--batch-size <number>",
185
+ "Batch size for parallel processing",
186
+ "10",
188
187
  )
189
- .option('--max-memory <number>', 'Maximum memory usage in MB', '512')
188
+ .option("--max-memory <number>", "Maximum memory usage in MB", "512")
190
189
  .action(async (file, options) => {
191
- await this.handleIngest(file, options);
190
+ await handleIngest(this.program.opts(), file, options);
192
191
  });
193
192
 
194
193
  // Enhanced query command
195
194
  this.program
196
- .command('query')
197
- .description('Query the RAG pipeline')
198
- .argument('<prompt>', 'Query prompt')
199
- .option('--stream', 'Enable streaming response')
200
- .option('--explain', 'Show explanation of query processing')
201
- .option('--parallel', 'Enable parallel processing')
202
- .option('--trace', 'Enable detailed tracing and event logging')
203
- .option('--stats', 'Show pipeline metrics summary at the end')
195
+ .command("query")
196
+ .description("Query the RAG pipeline")
197
+ .argument("<prompt>", "Query prompt")
198
+ .option("--stream", "Enable streaming response")
199
+ .option("--explain", "Show explanation of query processing")
200
+ .option("--parallel", "Enable parallel processing")
201
+ .option("--trace", "Enable detailed tracing and event logging")
202
+ .option("--stats", "Show pipeline metrics summary at the end")
204
203
  .option(
205
- '--export-observability <file>',
206
- 'Export observability data to file',
204
+ "--export-observability <file>",
205
+ "Export observability data to file",
207
206
  )
208
207
  .action(async (prompt, options) => {
209
- await this.handleQuery(prompt, options);
208
+ await handleQuery(this.program.opts(), prompt, options);
210
209
  });
211
210
 
212
211
  // Configuration validation command
213
212
  this.program
214
- .command('validate')
215
- .description('Validate configuration file')
216
- .option('--schema', 'Validate against schema only')
217
- .option('--plugins', 'Validate plugin availability')
218
- .option('--dependencies', 'Check plugin dependencies')
219
- .option('--fix', 'Attempt to fix validation issues')
213
+ .command("validate")
214
+ .description("Validate configuration file")
215
+ .option("--schema", "Validate against schema only")
216
+ .option("--plugins", "Validate plugin availability")
217
+ .option("--dependencies", "Check plugin dependencies")
218
+ .option("--fix", "Attempt to fix validation issues")
220
219
  .action(async (options) => {
221
- await this.handleValidate(options);
220
+ await handleValidate(this.program.opts(), options, mockValidateSchema);
222
221
  });
223
222
 
224
223
  // Plugin marketplace commands
225
224
  this.program.addCommand(mockPluginMarketplaceCommands());
226
225
 
227
226
  // Configuration management commands
228
- const configCmd = new Command('config');
229
- configCmd.description('Configuration management commands');
227
+ const configCmd = new Command("config");
228
+ configCmd.description("Configuration management commands");
230
229
 
231
230
  configCmd
232
- .command('show')
233
- .description('Show current configuration')
234
- .option('--format <format>', 'Output format (json, yaml, table)', 'json')
235
- .option('--section <section>', 'Show specific configuration section')
231
+ .command("show")
232
+ .description("Show current configuration")
233
+ .option("--format <format>", "Output format (json, yaml, table)", "json")
234
+ .option("--section <section>", "Show specific configuration section")
236
235
  .action(async (options) => {
237
- await this.handleConfigShow(options);
236
+ await handleConfigShow(this.program.opts(), options);
238
237
  });
239
238
 
240
239
  configCmd
241
- .command('set')
242
- .description('Set configuration value')
243
- .argument('<key>', 'Configuration key (dot notation supported)')
244
- .argument('<value>', 'Configuration value')
240
+ .command("set")
241
+ .description("Set configuration value")
242
+ .argument("<key>", "Configuration key (dot notation supported)")
243
+ .argument("<value>", "Configuration value")
245
244
  .action(async (key, value, options) => {
246
- await this.handleConfigSet(key, value, options);
245
+ await handleConfigSet(this.program.opts(), key, value);
247
246
  });
248
247
 
249
248
  configCmd
250
- .command('get')
251
- .description('Get configuration value')
252
- .argument('<key>', 'Configuration key (dot notation supported)')
249
+ .command("get")
250
+ .description("Get configuration value")
251
+ .argument("<key>", "Configuration key (dot notation supported)")
253
252
  .action(async (key, options) => {
254
- await this.handleConfigGet(key, options);
253
+ await handleConfigGet(this.program.opts(), key);
255
254
  });
256
255
 
257
256
  this.program.addCommand(configCmd);
258
257
 
259
258
  // System information command
260
259
  this.program
261
- .command('info')
262
- .description('Show system and plugin information')
263
- .option('--plugins', 'Show registered plugin versions')
264
- .option('--system', 'Show system information')
265
- .option('--config', 'Show configuration summary')
260
+ .command("info")
261
+ .description("Show system and plugin information")
262
+ .option("--plugins", "Show registered plugin versions")
263
+ .option("--system", "Show system information")
264
+ .option("--config", "Show configuration summary")
266
265
  .action(async (options) => {
267
- await this.handleInfo(options);
266
+ await handleInfo(this.program.opts(), options);
268
267
  });
269
268
 
270
269
  // Completion command for shell autocompletion
271
270
  this.program
272
- .command('completion')
273
- .description('Generate shell completion scripts')
274
- .argument('[shell]', 'Shell type (bash, zsh, fish)', 'bash')
271
+ .command("completion")
272
+ .description("Generate shell completion scripts")
273
+ .argument("[shell]", "Shell type (bash, zsh, fish)", "bash")
275
274
  .action(async (shell, options) => {
276
- await this.handleCompletion(shell, options);
275
+ const completionScript = generateCompletionScript(shell);
276
+ console.log(completionScript);
277
277
  });
278
278
  }
279
279
 
280
- /**
281
- * Handle run command
282
- */
283
- async handleRun(options) {
284
- const globalOptions = this.program.opts();
285
-
286
- if (globalOptions.dryRun) {
287
- logger.info('🧪 Dry run: Would execute pipeline operation');
288
- if (options.ingest) logger.info(`Would ingest: ${options.ingest}`);
289
- if (options.query)
290
- logger.info(`Would query: ${options.query.substring(0, 100)}...`);
291
- logger.info(`Streaming: ${options.stream ? 'enabled' : 'disabled'}`);
292
- logger.info(`Parallel: ${options.parallel ? 'enabled' : 'disabled'}`);
293
- if (options.abortTimeout)
294
- logger.info(`Abort timeout: ${options.abortTimeout}ms`);
295
- return;
296
- }
297
-
298
- try {
299
- // Load configuration
300
- const { loadRagConfig } = require('../config/load-config.js');
301
- const config = await loadRagConfig(globalOptions.config);
302
-
303
- // Create AbortController for timeout handling
304
- const abortController = new AbortController();
305
- let timeoutId;
306
-
307
- if (options.abortTimeout) {
308
- timeoutId = setTimeout(() => {
309
- logger.warn(
310
- `Operation aborted after ${options.abortTimeout}ms timeout`,
311
- );
312
- abortController.abort();
313
- }, options.abortTimeout);
314
- }
315
-
316
- // Setup SIGINT handling for clean shutdown
317
- const sigintHandler = () => {
318
- logger.info('Received SIGINT, aborting operation...');
319
- abortController.abort();
320
- if (timeoutId) clearTimeout(timeoutId);
321
- };
322
- process.once('SIGINT', sigintHandler);
323
-
324
- try {
325
- // Create pipeline with performance options
326
- const { createRagPipeline } = require('../core/create-pipeline.js');
327
- const pipeline = createRagPipeline(config.pipeline[0], {
328
- useParallelProcessing: options.parallel,
329
- useStreamingSafeguards: options.stream,
330
- useLogging: true,
331
- useRetry: true,
332
- });
333
-
334
- // Execute operation based on options
335
- if (options.ingest) {
336
- logger.info(`Starting ingestion of: ${options.ingest}`);
337
-
338
- if (options.stream) {
339
- let lastProgressTime = Date.now();
340
- for await (const progress of pipeline.ingestStream(
341
- options.ingest,
342
- )) {
343
- if (abortController.signal.aborted) {
344
- logger.info('Ingestion aborted');
345
- break;
346
- }
347
-
348
- const now = Date.now();
349
- if (progress.type === 'chunk_processed') {
350
- // Show progress ticks every 500ms or every 10 chunks
351
- if (
352
- now - lastProgressTime > 500 ||
353
- progress.progress.processed % 10 === 0
354
- ) {
355
- const percentage = Math.round(
356
- (progress.progress.processed / progress.progress.total) *
357
- 100,
358
- );
359
- const progressBar = this.createProgressBar(percentage, 20);
360
- process.stdout.write(
361
- `\r${progressBar} ${percentage}% (${progress.progress.processed}/${progress.progress.total}) `,
362
- );
363
- lastProgressTime = now;
364
- }
365
- } else if (progress.type === 'ingest_complete') {
366
- process.stdout.write('\n');
367
- logger.info(
368
- `✅ Ingestion complete: ${progress.summary.processedChunks}/${progress.summary.totalChunks} chunks processed`,
369
- );
370
- if (progress.summary.failedChunks > 0) {
371
- logger.warn(
372
- `⚠️ ${progress.summary.failedChunks} chunks failed`,
373
- );
374
- }
375
- } else if (progress.type === 'chunk_failed') {
376
- process.stdout.write('✗');
377
- }
378
- }
379
- } else {
380
- // Non-streaming ingestion with progress callback
381
- let currentStage = '';
382
- const progressCallback = (progress) => {
383
- if (progress.stage !== currentStage) {
384
- if (currentStage) process.stdout.write('\n');
385
- currentStage = progress.stage;
386
- logger.info(
387
- `${progress.stage}: ${progress.message || 'Processing...'}`,
388
- );
389
- }
390
- if (
391
- progress.completed !== undefined &&
392
- progress.total !== undefined
393
- ) {
394
- const percentage = Math.round(
395
- (progress.completed / progress.total) * 100,
396
- );
397
- const progressBar = this.createProgressBar(percentage, 20);
398
- process.stdout.write(
399
- `\r${progressBar} ${percentage}% (${progress.completed}/${progress.total}) `,
400
- );
401
- }
402
- };
403
-
404
- const pipelineWithProgress = createRagPipeline(config.pipeline[0], {
405
- useParallelProcessing: options.parallel,
406
- useStreamingSafeguards: false,
407
- useLogging: true,
408
- useRetry: true,
409
- onProgress: progressCallback,
410
- });
411
-
412
- await pipelineWithProgress.ingest(options.ingest);
413
- process.stdout.write('\n');
414
- logger.info('✅ Ingestion completed successfully');
415
- }
416
- }
417
-
418
- if (options.query) {
419
- logger.info(`Executing query: ${options.query.substring(0, 100)}...`);
420
-
421
- if (options.stream) {
422
- process.stdout.write('\nResponse: ');
423
- for await (const token of pipeline.queryStream(options.query)) {
424
- if (abortController.signal.aborted) {
425
- logger.info('\nQuery aborted');
426
- break;
427
- }
428
- process.stdout.write(token);
429
- }
430
- process.stdout.write('\n\n');
431
- } else {
432
- const response = await pipeline.query(options.query);
433
- logger.info('Query completed successfully');
434
- console.log('\nResponse:', response);
435
- }
436
- }
437
-
438
- if (!options.ingest && !options.query) {
439
- logger.error(
440
- 'No operation specified. Use --ingest <file> or --query <prompt>',
441
- );
442
- process.exit(1);
443
- }
444
- } finally {
445
- // Cleanup
446
- process.removeListener('SIGINT', sigintHandler);
447
- if (timeoutId) clearTimeout(timeoutId);
448
- }
449
- } catch (error) {
450
- logger.error('Pipeline operation failed', { error: error.message });
451
- process.exit(1);
452
- }
453
- }
454
-
455
- /**
456
- * Handle init command
457
- */
458
- async handleInit(options) {
459
- try {
460
- const globalOptions = this.program.opts();
461
-
462
- if (globalOptions.dryRun) {
463
- console.log('🧪 Dry run: Would initialize RAG pipeline configuration');
464
- console.log(`Output file: ${options.output}`);
465
- console.log(`Interactive mode: ${!options.noInteractive}`);
466
- return;
467
- }
468
-
469
- // Check if config already exists
470
- if (!options.force) {
471
- try {
472
- await fs.access(options.output);
473
- console.log(
474
- `❌ Configuration file already exists: ${options.output}`,
475
- );
476
- console.log(
477
- 'Use --force to overwrite or choose a different output path.',
478
- );
479
- return;
480
- } catch (error) {
481
- // File doesn't exist, which is good
482
- }
483
- }
484
-
485
- if (!options.noInteractive) {
486
- console.log('🧙‍♂️ Starting interactive configuration wizard...\n');
487
- // Mock interactive wizard
488
- await this.createBasicConfig(options.output);
489
- } else {
490
- await this.createBasicConfig(options.output);
491
- }
492
- } catch (error) {
493
- logger.error('❌ Initialization failed:', error.message);
494
- process.exit(1);
495
- }
496
- }
497
-
498
- /**
499
- * Handle doctor command
500
- */
501
- async handleDoctor(options) {
502
- try {
503
- const globalOptions = this.program.opts();
504
-
505
- const doctorOptions = {
506
- configPath: globalOptions.config,
507
- verbose: globalOptions.verbose,
508
- autoFix: options.autoFix,
509
- categories: options.category,
510
- };
511
-
512
- const report = await mockPipelineDoctor(doctorOptions);
513
-
514
- if (options.report) {
515
- await fs.writeFile(options.report, JSON.stringify(report, null, 2));
516
- console.log(`📋 Diagnostic report saved to: ${options.report}`);
517
- }
518
- } catch (error) {
519
- logger.error('❌ Doctor command failed:', error.message);
520
- process.exit(1);
521
- }
522
- }
523
-
524
- /**
525
- * Handle ingest command
526
- */
527
- async handleIngest(file, options) {
528
- try {
529
- const globalOptions = this.program.opts();
530
-
531
- if (globalOptions.dryRun) {
532
- console.log('🧪 Dry run: Would ingest document');
533
- console.log(`File: ${file}`);
534
- console.log(
535
- `Parallel processing: ${options.parallel ? 'enabled' : 'disabled'}`,
536
- );
537
- console.log(`Streaming: ${options.streaming ? 'enabled' : 'disabled'}`);
538
- console.log(`Tracing: ${options.trace ? 'enabled' : 'disabled'}`);
539
-
540
- if (options.preview) {
541
- console.log('📄 Document Preview:');
542
- }
543
- if (options.validate) {
544
- console.log('File validation passed');
545
- }
546
- return;
547
- }
548
-
549
- console.log(`📄 Ingesting document: ${file}`);
550
- console.log('✅ Document ingested successfully!');
551
- } catch (error) {
552
- logger.error('❌ Document ingestion failed:', error.message);
553
- process.exit(1);
554
- }
555
- }
556
-
557
- /**
558
- * Handle query command
559
- */
560
- async handleQuery(prompt, options) {
561
- try {
562
- const globalOptions = this.program.opts();
563
-
564
- if (globalOptions.dryRun) {
565
- console.log('🧪 Dry run: Would execute query');
566
- console.log(
567
- `Prompt: ${prompt.substring(0, 100)}${prompt.length > 100 ? '...' : ''}`,
568
- );
569
- console.log(`Streaming: ${options.stream ? 'enabled' : 'disabled'}`);
570
-
571
- if (options.explain) {
572
- console.log('🔍 Query Explanation:');
573
- console.log('Processing steps:');
574
- console.log(' 1. Text embedding generation');
575
- console.log(' 2. Vector similarity search');
576
- console.log(' 3. Context retrieval');
577
- console.log(' 4. Language model generation');
578
- }
579
- return;
580
- }
581
-
582
- console.log(
583
- `🤔 Processing query: ${prompt.substring(0, 100)}${prompt.length > 100 ? '...' : ''}`,
584
- );
585
- console.log('\n📝 Response:');
586
- console.log('Mock response to query');
587
- } catch (error) {
588
- logger.error('❌ Query processing failed:', error.message);
589
- process.exit(1);
590
- }
591
- }
592
-
593
- /**
594
- * Handle validate command
595
- */
596
- async handleValidate(options) {
597
- try {
598
- const globalOptions = this.program.opts();
599
- const configPath = globalOptions.config;
600
-
601
- console.log(`🔍 Validating configuration: ${configPath}`);
602
-
603
- // Load and validate configuration
604
- const configContent = await fs.readFile(configPath, 'utf-8');
605
- const config = JSON.parse(configContent);
606
-
607
- const validation = mockValidateSchema(config);
608
-
609
- if (validation.valid) {
610
- console.log('✅ Configuration is valid');
611
-
612
- if (validation.legacy) {
613
- console.log('⚠️ Using legacy format - consider upgrading');
614
- }
615
- } else {
616
- console.log('❌ Configuration validation failed:');
617
- validation.errors?.forEach((error) => {
618
- console.log(` ${error.instancePath || 'root'}: ${error.message}`);
619
- });
620
- }
621
- } catch (error) {
622
- logger.error('❌ Validation failed:', error.message);
623
- process.exit(1);
624
- }
625
- }
626
-
627
- /**
628
- * Handle config show command
629
- */
630
- async handleConfigShow(options) {
631
- try {
632
- const globalOptions = this.program.opts();
633
- const config = JSON.parse(
634
- await fs.readFile(globalOptions.config, 'utf-8'),
635
- );
636
-
637
- let output = config;
638
- if (options.section) {
639
- const sections = options.section.split('.');
640
- for (const section of sections) {
641
- output = output[section];
642
- if (output === undefined) {
643
- console.log(`❌ Section not found: ${options.section}`);
644
- return;
645
- }
646
- }
647
- }
648
-
649
- console.log(JSON.stringify(output, null, 2));
650
- } catch (error) {
651
- logger.error('❌ Failed to show configuration:', error.message);
652
- process.exit(1);
653
- }
654
- }
655
-
656
- /**
657
- * Handle config set command
658
- */
659
- async handleConfigSet(key, value) {
660
- try {
661
- const globalOptions = this.program.opts();
662
- const config = JSON.parse(
663
- await fs.readFile(globalOptions.config, 'utf-8'),
664
- );
665
-
666
- // Set value using dot notation
667
- const keys = key.split('.');
668
- let current = config;
669
-
670
- for (let i = 0; i < keys.length - 1; i++) {
671
- if (!current[keys[i]]) {
672
- current[keys[i]] = {};
673
- }
674
- current = current[keys[i]];
675
- }
676
-
677
- // Parse value
678
- let parsedValue;
679
- try {
680
- parsedValue = JSON.parse(value);
681
- } catch (error) {
682
- parsedValue = value; // Keep as string
683
- }
684
-
685
- current[keys[keys.length - 1]] = parsedValue;
686
-
687
- // Save configuration
688
- await fs.writeFile(globalOptions.config, JSON.stringify(config, null, 2));
689
- console.log('✅ Configuration updated');
690
- } catch (error) {
691
- logger.error('❌ Failed to set configuration:', error.message);
692
- process.exit(1);
693
- }
694
- }
695
-
696
- /**
697
- * Handle config get command
698
- */
699
- async handleConfigGet(key) {
700
- try {
701
- const globalOptions = this.program.opts();
702
- const config = JSON.parse(
703
- await fs.readFile(globalOptions.config, 'utf-8'),
704
- );
705
-
706
- const keys = key.split('.');
707
- let value = config;
708
-
709
- for (const k of keys) {
710
- value = value[k];
711
- if (value === undefined) {
712
- console.log('❌ Key not found');
713
- return;
714
- }
715
- }
716
-
717
- console.log(JSON.stringify(value, null, 2));
718
- } catch (error) {
719
- logger.error('❌ Failed to get configuration:', error.message);
720
- process.exit(1);
721
- }
722
- }
723
-
724
- /**
725
- * Handle info command
726
- */
727
- async handleInfo(options) {
728
- try {
729
- console.log('📊 RAG Pipeline Information\n');
730
-
731
- // System information
732
- if ((!options.plugins && !options.config) || options.system) {
733
- console.log('System:');
734
- console.log(` Node.js: ${process.version}`);
735
- console.log(` Platform: ${process.platform} ${process.arch}`);
736
- console.log(
737
- ` Memory: ${Math.round(process.memoryUsage().heapUsed / 1024 / 1024)}MB used`,
738
- );
739
- console.log('');
740
- }
741
-
742
- // Configuration summary
743
- if ((!options.system && !options.plugins) || options.config) {
744
- const globalOptions = this.program.opts();
745
- try {
746
- const config = JSON.parse(
747
- await fs.readFile(globalOptions.config, 'utf-8'),
748
- );
749
- console.log('Configuration:');
750
- console.log(` File: ${globalOptions.config}`);
751
- console.log(` Format: ${config.plugins ? 'Enhanced' : 'Legacy'}`);
752
- console.log('');
753
- } catch (error) {
754
- console.log('Configuration: Not found or invalid');
755
- console.log('');
756
- }
757
- }
758
- } catch (error) {
759
- logger.error('❌ Failed to get system information:', error.message);
760
- process.exit(1);
761
- }
762
- }
763
-
764
- /**
765
- * Handle completion command
766
- */
767
- async handleCompletion(shell) {
768
- const completionScript = this.generateCompletionScript(shell);
769
- console.log(completionScript);
770
- }
771
-
772
280
  /**
773
281
  * Create basic configuration
774
282
  */
775
283
  async createBasicConfig(outputPath) {
776
- const { validateRagrc } = require('../config/enhanced-ragrc-schema.js');
284
+ const { validateRagrc } = require("../config/enhanced-ragrc-schema.js");
777
285
 
778
286
  const base = {
779
- namespace: 'default',
287
+ namespace: "default",
780
288
  metadata: {
781
- name: 'rag-pipeline',
782
- version: '1.0.0',
783
- description: 'RAG pipeline configuration',
289
+ name: "rag-pipeline",
290
+ version: "1.0.0",
291
+ description: "RAG pipeline configuration",
784
292
  createdAt: new Date().toISOString(),
785
293
  },
786
294
  pipeline: [
787
- { stage: 'loader', name: 'fs-loader', version: '1.0.0' },
788
- { stage: 'embedder', name: 'mock-embedder', version: '1.0.0' },
789
- { stage: 'retriever', name: 'mock-retriever', version: '1.0.0' },
790
- { stage: 'llm', name: 'mock-llm', version: '1.0.0' },
295
+ { stage: "loader", name: "fs-loader", version: "1.0.0" },
296
+ { stage: "embedder", name: "mock-embedder", version: "1.0.0" },
297
+ { stage: "retriever", name: "mock-retriever", version: "1.0.0" },
298
+ { stage: "llm", name: "mock-llm", version: "1.0.0" },
791
299
  ],
792
300
  performance: {
793
301
  parallel: {
@@ -802,7 +310,7 @@ class EnhancedCLI {
802
310
  },
803
311
  observability: {
804
312
  logging: {
805
- level: 'info',
313
+ level: "info",
806
314
  structured: true,
807
315
  },
808
316
  },
@@ -811,9 +319,9 @@ class EnhancedCLI {
811
319
  const result = validateRagrc(base);
812
320
  if (!result.valid) {
813
321
  result.errors?.forEach((error) =>
814
- console.error(` ${error.instancePath || ''}: ${error.message}`),
322
+ console.error(` ${error.instancePath || ""}: ${error.message}`),
815
323
  );
816
- throw new Error('Failed to generate valid configuration');
324
+ throw new Error("Failed to generate valid configuration");
817
325
  }
818
326
 
819
327
  if (outputPath) {
@@ -827,58 +335,6 @@ class EnhancedCLI {
827
335
  return result.normalized;
828
336
  }
829
337
 
830
- /**
831
- * Get version information from package.json
832
- */
833
- getVersion() {
834
- return pkg.version;
835
- }
836
-
837
- /**
838
- * Get extended help text
839
- */
840
- getExtendedHelp() {
841
- return `
842
- Examples:
843
- rag-pipeline init --interactive Initialize with wizard
844
- rag-pipeline doctor --auto-fix Diagnose and fix issues
845
- rag-pipeline plugin search openai Search for plugins
846
- rag-pipeline config show plugins Show plugin configuration
847
-
848
- For more information, visit: https://github.com/DevilsDev/rag-pipeline-utils
849
- `;
850
- }
851
-
852
- /**
853
- * Generate shell completion script
854
- */
855
- generateCompletionScript(shell) {
856
- switch (shell) {
857
- case 'bash':
858
- return `# RAG Pipeline bash completion
859
- _rag_pipeline_completions() {
860
- COMPREPLY=($(compgen -W "init doctor ingest query plugin config info validate completion" -- "\${COMP_WORDS[COMP_CWORD]}"))
861
- }
862
- complete -F _rag_pipeline_completions rag-pipeline`;
863
-
864
- case 'zsh':
865
- return `# RAG Pipeline zsh completion
866
- #compdef rag-pipeline
867
- _rag_pipeline() {
868
- _arguments \\
869
- '1:command:(init doctor ingest query plugin config info validate completion)'
870
- }
871
- _rag_pipeline "$@"`;
872
-
873
- case 'fish':
874
- return `# RAG Pipeline fish completion
875
- complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query plugin config info validate completion'`;
876
-
877
- default:
878
- return `# Completion not available for ${shell}`;
879
- }
880
- }
881
-
882
338
  /**
883
339
  * Run the CLI
884
340
  */
@@ -886,8 +342,8 @@ complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query
886
342
  try {
887
343
  await this.program.parseAsync(argv);
888
344
  } catch (error) {
889
- if (error.code !== 'commander.unknownCommand') {
890
- logger.error('❌ CLI error:', error.message);
345
+ if (error.code !== "commander.unknownCommand") {
346
+ logger.error("❌ CLI error:", error.message);
891
347
  process.exit(1);
892
348
  }
893
349
  throw error;
@@ -913,7 +369,7 @@ const ingest = async () => ({ ok: true });
913
369
  const query = async () => ({ ok: true });
914
370
  const handleError = async () => ({ ok: true });
915
371
  const validateArgs = () => true;
916
- const showHelp = () => 'help text';
372
+ const showHelp = () => "help text";
917
373
 
918
374
  // Extract createBasicConfig from the class for direct export
919
375
  const createBasicConfig = async (outputPath) => {