@devilsdev/rag-pipeline-utils 2.3.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/README.md +293 -361
  2. package/contracts/loader-contract.json +11 -6
  3. package/contracts/reranker-contract.json +18 -26
  4. package/dist/index.cjs +21190 -47
  5. package/dist/index.d.ts +298 -68
  6. package/dist/index.mjs +9 -76
  7. package/package.json +20 -8
  8. package/src/agentic/agentic-pipeline.js +235 -0
  9. package/src/agentic/iterative-retriever.js +213 -0
  10. package/src/agentic/query-planner.js +208 -0
  11. package/src/agentic/self-critique.js +102 -0
  12. package/src/ai/adaptive-retrieval.js +0 -3
  13. package/src/ai/federated-learning.js +0 -3
  14. package/src/ai/federation.js +1 -6
  15. package/src/ai/index.js +9 -20
  16. package/src/ai/model-training.js +0 -3
  17. package/src/ai/multimodal-processing.js +0 -3
  18. package/src/ai/multimodal.js +1 -6
  19. package/src/ai/orchestrator.js +1 -6
  20. package/src/ai/retrieval-engine.js +1 -6
  21. package/src/chunking/chunking-engine.js +142 -0
  22. package/src/chunking/strategies/fixed-size.js +54 -0
  23. package/src/chunking/strategies/recursive.js +127 -0
  24. package/src/chunking/strategies/semantic.js +128 -0
  25. package/src/chunking/strategies/sentence.js +32 -0
  26. package/src/chunking/strategies/structure-aware.js +138 -0
  27. package/src/citation/citation-tracker.js +161 -0
  28. package/src/citation/hallucination-detector.js +74 -0
  29. package/src/citation/source-mapper.js +95 -0
  30. package/src/cli/commands/ai-ml/adaptive-commands.js +178 -0
  31. package/src/cli/commands/ai-ml/federated-commands.js +207 -0
  32. package/src/cli/commands/ai-ml/index.js +17 -0
  33. package/src/cli/commands/ai-ml/multimodal-commands.js +187 -0
  34. package/src/cli/commands/ai-ml/shared.js +51 -0
  35. package/src/cli/commands/ai-ml/training-commands.js +228 -0
  36. package/src/cli/commands/ai-ml.js +176 -757
  37. package/src/cli/commands/plugin-hub/certification-commands.js +221 -0
  38. package/src/cli/commands/plugin-hub/event-handlers.js +64 -0
  39. package/src/cli/commands/plugin-hub/hub-commands.js +290 -0
  40. package/src/cli/commands/plugin-hub/review-commands.js +78 -0
  41. package/src/cli/commands/plugin-hub.js +106 -782
  42. package/src/cli/diagnostics/config-checker.js +81 -0
  43. package/src/cli/diagnostics/dependency-checker.js +69 -0
  44. package/src/cli/diagnostics/environment-checker.js +45 -0
  45. package/src/cli/diagnostics/performance-checker.js +51 -0
  46. package/src/cli/diagnostics/plugin-checker.js +106 -0
  47. package/src/cli/diagnostics/security-checker.js +61 -0
  48. package/src/cli/doctor-command.js +70 -369
  49. package/src/cli/enhanced-cli-commands.js +162 -706
  50. package/src/cli/handlers/config-handler.js +126 -0
  51. package/src/cli/handlers/diagnostics-handler.js +113 -0
  52. package/src/cli/handlers/ingest-handler.js +42 -0
  53. package/src/cli/handlers/init-handler.js +51 -0
  54. package/src/cli/handlers/query-handler.js +45 -0
  55. package/src/cli/handlers/run-handler.js +181 -0
  56. package/src/cli/interactive-wizard.js +99 -723
  57. package/src/cli/shared/cli-utils.js +89 -0
  58. package/src/cli/wizard/configuration.js +148 -0
  59. package/src/cli/wizard/observability.js +79 -0
  60. package/src/cli/wizard/performance.js +76 -0
  61. package/src/cli/wizard/plugin-selection.js +297 -0
  62. package/src/cli/wizard/preview-save.js +172 -0
  63. package/src/cli/wizard/project-setup.js +66 -0
  64. package/src/config/memory-optimization.js +3 -3
  65. package/src/connectors/anthropic-connector.js +179 -0
  66. package/src/connectors/base-connector.js +94 -0
  67. package/src/connectors/cohere-connector.js +143 -0
  68. package/src/connectors/local-embedder.js +171 -0
  69. package/src/connectors/memory-retriever.js +150 -0
  70. package/src/connectors/ollama-connector.js +205 -0
  71. package/src/connectors/openai-connector.js +173 -0
  72. package/src/core/create-pipeline.js +21 -1
  73. package/src/core/observability/tracing.js +2 -1
  74. package/src/core/pipeline-factory.js +0 -1
  75. package/src/core/plugin-registry.js +14 -14
  76. package/src/cost/cost-calculator.js +146 -0
  77. package/src/cost/cost-tracker.js +260 -0
  78. package/src/cost/token-budget.js +245 -0
  79. package/src/dag/core/execution-scheduler.js +2 -1
  80. package/src/dag/core/topology-validation.js +0 -3
  81. package/src/dag/dag-engine.js +3 -5
  82. package/src/dag/dag-node.js +0 -1
  83. package/src/dag/enhanced-dag-engine.js +171 -99
  84. package/src/dashboard/dashboard-generator.js +642 -0
  85. package/src/dashboard/metrics-aggregator.js +409 -0
  86. package/src/debug/execution-tracer.js +210 -0
  87. package/src/debug/trace-inspector.js +281 -0
  88. package/src/dev/dev-server.js +5 -5
  89. package/src/dev/hot-reload.js +3 -2
  90. package/src/dx/integration-templates.js +0 -3
  91. package/src/ecosystem/plugin-analytics-dashboard.js +2 -2
  92. package/src/ecosystem/plugin-hub.js +2 -2
  93. package/src/enterprise/audit-logging.js +2 -2
  94. package/src/enterprise/sso-integration.js +52 -32
  95. package/src/evaluate/context-metrics.js +127 -0
  96. package/src/evaluate/evaluator.js +4 -2
  97. package/src/evaluate/faithfulness.js +117 -0
  98. package/src/evaluate/groundedness.js +60 -0
  99. package/src/evaluate/pipeline-evaluator.js +156 -0
  100. package/src/evaluate/relevance.js +129 -0
  101. package/src/evaluate/scoring.js +27 -0
  102. package/src/graph/entity-extractor.js +183 -0
  103. package/src/graph/graph-index.js +113 -0
  104. package/src/graph/graph-retriever.js +244 -0
  105. package/src/graph/knowledge-graph.js +256 -0
  106. package/src/guardrails/guardrails-pipeline.js +136 -0
  107. package/src/guardrails/post-generation-guard.js +170 -0
  108. package/src/guardrails/pre-retrieval-guard.js +161 -0
  109. package/src/guardrails/retrieval-guard.js +126 -0
  110. package/src/index.js +255 -9
  111. package/src/loader/html-loader.js +4 -15
  112. package/src/loader/markdown-loader.js +4 -15
  113. package/src/mcp/mcp-server.js +199 -0
  114. package/src/mcp/mcp-tool-builder.js +121 -0
  115. package/src/observability/audit-logger.js +0 -2
  116. package/src/observability/event-logger.js +2 -1
  117. package/src/observability/security-monitor.js +0 -2
  118. package/src/observability/telemetry.js +0 -2
  119. package/src/reranker/cascade-reranker.js +79 -0
  120. package/src/reranker/embedding-reranker.js +124 -0
  121. package/src/reranker/llm-reranker.js +6 -3
  122. package/src/reranker/scoring-reranker.js +123 -0
  123. package/src/retrieval/bm25-search.js +157 -0
  124. package/src/retrieval/hybrid-retriever.js +111 -0
  125. package/src/retrieval/rank-fusion.js +62 -0
  126. package/src/security/jwt-config.js +4 -3
  127. package/src/security/jwt-validator.js +2 -1
  128. package/src/security/plugin-sandbox.js +0 -3
  129. package/src/security/plugin-signature-verifier.js +4 -6
  130. package/src/security/validator.js +0 -2
  131. package/src/streaming/embedding-stream.js +197 -0
  132. package/src/streaming/sse-adapter.js +168 -0
  133. package/src/streaming/stream-router.js +210 -0
  134. package/src/streaming/streaming-embedder.js +231 -0
  135. package/src/streaming/websocket-adapter.js +171 -0
  136. package/src/templates/scaffold.js +147 -0
  137. package/src/templates/template-registry.js +272 -0
  138. package/src/utils/contract-validator.js +2 -1
  139. package/src/utils/input-sanitizer.js +80 -99
  140. package/src/utils/performance-monitor.js +0 -3
  141. package/src/utils/plugin-scaffolder.js +9 -16
  142. package/src/utils/rate-limiter.js +0 -1
  143. package/src/utils/retry-policy.js +0 -3
  144. package/src/utils/safe-expression-evaluator.js +0 -3
  145. package/src/utils/secure-logger-config.js +2 -4
  146. package/src/utils/secure-logging.js +0 -3
  147. package/src/utils/structured-logger.js +0 -3
  148. package/src/ai/index-fixed.js +0 -509
  149. package/src/ai/module-template.js +0 -37
  150. package/src/ci/workflow-analyzer.js +0 -71
  151. package/src/cli/plugin-marketplace-commands.js +0 -869
  152. package/src/dev/README.md +0 -126
  153. package/src/dev/examples/hot-reload-usage.js +0 -355
  154. package/src/mocks/openai-embedder.js +0 -119
  155. package/src/mocks/openai-llm.js +0 -77
  156. package/src/mocks/pdf-loader.js +0 -21
  157. package/src/mocks/pinecone-retriever.js +0 -106
  158. package/src/scripts/script-utilities.js +0 -36
  159. package/src/utils/ci/diagnostic-reporter.js +0 -57
  160. package/src/utils/examples/error-formatter-usage.js +0 -351
@@ -1,18 +1,32 @@
1
1
  /**
2
2
  * Interactive CLI Wizard for RAG Pipeline Setup
3
3
  * Provides guided setup for plugin selection, DAG building, and configuration
4
+ *
5
+ * This module is a thin orchestrator that delegates to focused step modules
6
+ * in src/cli/wizard/. Each step receives the wizard's config state and
7
+ * inquirer instance.
4
8
  */
5
9
 
6
- const inquirer = require('inquirer');
7
- const fs = require('fs/promises');
8
- const path = require('path');
9
- const {
10
- validateRagrc,
11
- extractPluginDependencies,
12
- } = require('../config/enhanced-ragrc-schema.js');
10
+ const inquirer = require("inquirer");
11
+ const path = require("path");
13
12
  const {
14
13
  DEFAULT_REGISTRY_URLS,
15
- } = require('../core/plugin-marketplace/plugin-registry-format.js');
14
+ } = require("../core/plugin-marketplace/plugin-registry-format.js");
15
+
16
+ // Step modules
17
+ const { setupProject } = require("./wizard/project-setup.js");
18
+ const { selectPlugins } = require("./wizard/plugin-selection.js");
19
+ const {
20
+ configureSettings,
21
+ configurePipeline,
22
+ } = require("./wizard/configuration.js");
23
+ const { configurePerformance } = require("./wizard/performance.js");
24
+ const { configureObservability } = require("./wizard/observability.js");
25
+ const {
26
+ previewAndSave,
27
+ showConfigSummary,
28
+ runConfigurationTest,
29
+ } = require("./wizard/preview-save.js");
16
30
 
17
31
  /**
18
32
  * Interactive wizard for RAG pipeline setup
@@ -21,7 +35,7 @@ class InteractiveWizard {
21
35
  constructor(_options = {}) {
22
36
  this._options = {
23
37
  registryUrl: _options.registryUrl || DEFAULT_REGISTRY_URLS[0],
24
- outputPath: _options.outputPath || '.ragrc.json',
38
+ outputPath: _options.outputPath || ".ragrc.json",
25
39
  ..._options,
26
40
  };
27
41
  this.registry = null;
@@ -39,13 +53,13 @@ class InteractiveWizard {
39
53
 
40
54
  // Initialize steps for testing interface
41
55
  this.steps = [
42
- 'project-setup',
43
- 'plugin-selection',
44
- 'configuration',
45
- 'pipeline-setup',
46
- 'performance-tuning',
47
- 'observability',
48
- 'preview-save',
56
+ "project-setup",
57
+ "plugin-selection",
58
+ "configuration",
59
+ "pipeline-setup",
60
+ "performance-tuning",
61
+ "observability",
62
+ "preview-save",
49
63
  ];
50
64
  this.currentStep = 0;
51
65
  }
@@ -55,10 +69,10 @@ class InteractiveWizard {
55
69
  * @returns {Promise<object>} Generated configuration
56
70
  */
57
71
  async run() {
58
- console.log('🧙‍♂️ Welcome to the RAG Pipeline Interactive Setup Wizard!\n');
72
+ console.log("🧙‍♂️ Welcome to the RAG Pipeline Interactive Setup Wizard!\n");
59
73
  // eslint-disable-line no-console
60
74
  console.log(
61
- 'This wizard will help you configure your RAG pipeline with the right plugins and settings.\n',
75
+ "This wizard will help you configure your RAG pipeline with the right plugins and settings.\n",
62
76
  );
63
77
  // eslint-disable-line no-console
64
78
 
@@ -84,738 +98,100 @@ class InteractiveWizard {
84
98
  // Step 7: Preview and save
85
99
  await this.previewAndSave();
86
100
 
87
- console.log('\n🎉 RAG Pipeline setup complete!');
101
+ console.log("\n🎉 RAG Pipeline setup complete!");
88
102
  // eslint-disable-line no-console
89
103
  console.log(`Configuration saved to: ${this._options.outputPath}`);
90
104
  // eslint-disable-line no-console
91
105
 
92
106
  return this._config;
93
107
  } catch (error) {
94
- console.error('\n❌ Setup wizard failed:', error.message);
108
+ console.error("\n❌ Setup wizard failed:", error.message);
95
109
  // eslint-disable-line no-console
96
110
  throw error;
97
111
  }
98
112
  }
99
113
 
100
114
  /**
101
- * Setup project metadata
115
+ * Setup project metadata - delegates to project-setup module
102
116
  */
103
117
  async setupProject() {
104
- console.log('📋 Project Setup\n');
105
- // eslint-disable-line no-console
106
-
107
- const answers = await inquirer.prompt([
108
- {
109
- _type: 'input',
110
- name: 'name',
111
- message: 'Project name:',
112
- default: path.basename(process.cwd()),
113
- validate: (input) => input.length > 0 || 'Project name is required',
114
- },
115
- {
116
- _type: 'input',
117
- name: 'description',
118
- message: 'Project description:',
119
- default: 'A RAG pipeline project',
120
- },
121
- {
122
- _type: 'input',
123
- name: 'author',
124
- message: 'Author:',
125
- default: 'Unknown',
126
- },
127
- {
128
- _type: 'list',
129
- name: 'environment',
130
- message: 'Target environment:',
131
- choices: [
132
- { name: 'Development', value: 'development' },
133
- { name: 'Production', value: 'production' },
134
- { name: 'Testing', value: 'testing' },
135
- ],
136
- default: 'development',
137
- },
138
- ]);
139
-
140
- this._config.metadata = {
141
- name: answers.name,
142
- version: '1.0.0',
143
- description: answers.description,
144
- author: answers.author,
145
- environment: answers.environment,
146
- createdAt: new Date().toISOString(),
147
- };
148
-
149
- console.log(`\n✅ Project "${answers.name}" configured\n`);
150
- // eslint-disable-line no-console
118
+ this._config = await setupProject(this._config, inquirer);
151
119
  }
152
120
 
153
121
  /**
154
- * Select plugins for each _type
122
+ * Select plugins for each type - delegates to plugin-selection module
155
123
  */
156
124
  async selectPlugins() {
157
- console.log('🔌 Plugin Selection\n');
158
- // eslint-disable-line no-console
159
-
160
- // Load registry for plugin suggestions
161
- try {
162
- this.registry = await this.loadRegistry();
163
- } catch (error) {
164
- console.warn(
165
- '⚠️ Could not load plugin registry. Using built-in _options.',
166
- );
167
- // eslint-disable-line no-console
168
- }
169
-
170
- const pluginTypes = [
171
- { key: 'loader', name: 'Document Loader', required: true },
172
- { key: 'embedder', name: 'Text Embedder', required: true },
173
- { key: 'retriever', name: 'Vector Retriever', required: true },
174
- { key: 'llm', name: 'Language Model', required: true },
175
- { key: 'reranker', name: 'Result Reranker', required: false },
176
- ];
177
-
178
- for (const pluginType of pluginTypes) {
179
- await this.selectPluginForType(pluginType);
180
- }
181
-
182
- console.log('✅ Plugin selection complete\n');
183
- // eslint-disable-line no-console
184
- }
185
-
186
- /**
187
- * Select plugin for specific _type
188
- * @param {object} pluginType - Plugin _type configuration
189
- */
190
- async selectPluginForType(pluginType) {
191
- const availablePlugins = this.getAvailablePlugins(pluginType.key);
192
-
193
- if (availablePlugins.length === 0 && pluginType.required) {
194
- console.log(
195
- `⚠️ No ${pluginType.name} plugins available. You'll need to configure this manually.`,
196
- );
197
- // eslint-disable-line no-console
198
- return;
199
- }
200
-
201
- const choices = [
202
- ...availablePlugins.map((plugin) => ({
203
- name: `${plugin.name} - ${plugin.description}`,
204
- value: plugin.name,
205
- short: plugin.name,
206
- })),
207
- { name: 'Skip (configure later)', value: null },
208
- { name: 'Custom plugin', value: 'custom' },
209
- ];
210
-
211
- const answer = await inquirer.prompt([
212
- {
213
- _type: 'list',
214
- name: 'plugin',
215
- message: `Select ${pluginType.name}:`,
216
- choices,
217
- when: () => !pluginType.required || availablePlugins.length > 0,
218
- },
219
- ]);
220
-
221
- if (!answer.plugin) {
222
- return; // Skip
223
- }
224
-
225
- if (answer.plugin === 'custom') {
226
- await this.configureCustomPlugin(pluginType.key);
227
- } else {
228
- await this.configureSelectedPlugin(pluginType.key, answer.plugin);
229
- }
230
- }
231
-
232
- /**
233
- * Configure custom plugin
234
- * @param {string} pluginType - Plugin _type
235
- */
236
- async configureCustomPlugin(pluginType) {
237
- const answers = await inquirer.prompt([
238
- {
239
- _type: 'input',
240
- name: 'name',
241
- message: 'Plugin name:',
242
- validate: (input) => input.length > 0 || 'Plugin name is required',
243
- },
244
- {
245
- _type: 'list',
246
- name: 'source',
247
- message: 'Plugin source:',
248
- choices: [
249
- { name: 'Registry', value: 'registry' },
250
- { name: 'Local file', value: 'local' },
251
- { name: 'Git repository', value: 'git' },
252
- { name: 'NPM package', value: 'npm' },
253
- ],
254
- default: 'registry',
255
- },
256
- {
257
- _type: 'input',
258
- name: 'version',
259
- message: 'Version:',
260
- default: 'latest',
261
- when: (answers) =>
262
- answers.source === 'registry' || answers.source === 'npm',
263
- },
264
- {
265
- _type: 'input',
266
- name: 'path',
267
- message: 'Local path:',
268
- when: (answers) => answers.source === 'local',
269
- },
270
- {
271
- _type: 'input',
272
- name: 'url',
273
- message: 'Repository URL:',
274
- when: (answers) => answers.source === 'git',
275
- },
276
- ]);
277
-
278
- const pluginSpec = {
279
- name: answers.name,
280
- source: answers.source,
125
+ const context = {
126
+ registry: this.registry,
127
+ getAvailablePlugins: (type) => this.getAvailablePlugins(type),
128
+ getPluginInfo: (type, name) => this.getPluginInfo(type, name),
129
+ loadRegistry: () => this.loadRegistry(),
281
130
  };
282
-
283
- if (answers.version) pluginSpec.version = answers.version;
284
- if (answers.path) pluginSpec.path = answers.path;
285
- if (answers.url) pluginSpec.url = answers.url;
286
-
287
- this._config.plugins[pluginType][answers.name] = pluginSpec;
131
+ this._config = await selectPlugins(this._config, inquirer, context);
132
+ // Sync registry back from context (loadRegistry may have updated it)
133
+ this.registry = context.registry;
288
134
  }
289
135
 
290
136
  /**
291
- * Configure selected plugin from registry
292
- * @param {string} pluginType - Plugin _type
293
- * @param {string} pluginName - Plugin name
294
- */
295
- async configureSelectedPlugin(pluginType, pluginName) {
296
- const plugin = this.getPluginInfo(pluginType, pluginName);
297
-
298
- if (!plugin) {
299
- this._config.plugins[pluginType][pluginName] = 'latest';
300
- return;
301
- }
302
-
303
- const versions = Object.keys(plugin.versions || {}).slice(0, 5);
304
- const versionChoices = [
305
- { name: 'Latest stable', value: 'latest' },
306
- { name: 'Latest beta', value: 'beta' },
307
- ...versions.map((v) => ({ name: v, value: v })),
308
- { name: 'Custom version', value: 'custom' },
309
- ];
310
-
311
- const versionAnswer = await inquirer.prompt([
312
- {
313
- _type: 'list',
314
- name: 'version',
315
- message: `Select version for ${pluginName}:`,
316
- choices: versionChoices,
317
- default: 'latest',
318
- },
319
- {
320
- _type: 'input',
321
- name: 'customVersion',
322
- message: 'Enter version:',
323
- when: (answers) => answers.version === 'custom',
324
- },
325
- ]);
326
-
327
- const version = versionAnswer.customVersion || versionAnswer.version;
328
-
329
- // Check if plugin needs configuration
330
- const needsConfig =
331
- plugin.metadata?._config &&
332
- Object.keys(plugin.metadata._config.properties || {}).length > 0;
333
-
334
- if (needsConfig) {
335
- const configAnswer = await inquirer.prompt([
336
- {
337
- _type: 'confirm',
338
- name: 'configure',
339
- message: `Configure ${pluginName} settings?`,
340
- default: false,
341
- },
342
- ]);
343
-
344
- if (configAnswer.configure) {
345
- const _config = await this.configurePluginSettings(plugin);
346
- this._config.plugins[pluginType][pluginName] = {
347
- name: pluginName,
348
- version,
349
- _config,
350
- };
351
- } else {
352
- this._config.plugins[pluginType][pluginName] = {
353
- name: pluginName,
354
- version,
355
- };
356
- }
357
- } else {
358
- this._config.plugins[pluginType][pluginName] = version;
359
- }
360
- }
361
-
362
- /**
363
- * Configure plugin-specific settings
364
- * @param {object} plugin - Plugin information
365
- * @returns {Promise<object>} Plugin configuration
366
- */
367
- async configurePluginSettings(plugin) {
368
- const _config = {};
369
- const schema = plugin.metadata?._config?.properties || {};
370
-
371
- console.log(`\n⚙️ Configuring ${plugin.metadata.name}...\n`);
372
- // eslint-disable-line no-console
373
-
374
- for (const [key, property] of Object.entries(schema)) {
375
- const question = {
376
- name: key,
377
- message: `${key}${property.description ? ` (${property.description})` : ''}:`,
378
- };
379
-
380
- if (property._type === 'boolean') {
381
- question._type = 'confirm';
382
- question.default = property.default || false;
383
- } else if (property.enum) {
384
- question._type = 'list';
385
- question.choices = property.enum;
386
- question.default = property.default;
387
- } else if (property._type === 'number') {
388
- question._type = 'number';
389
- question.default = property.default;
390
- } else {
391
- question._type = 'input';
392
- question.default = property.default || '';
393
- }
394
-
395
- const answer = await inquirer.prompt([question]);
396
- if (answer[key] !== '' && answer[key] !== undefined) {
397
- _config[key] = answer[key];
398
- }
399
- }
400
-
401
- return _config;
402
- }
403
-
404
- /**
405
- * Configure general settings
137
+ * Configure general settings - delegates to configuration module
406
138
  */
407
139
  async configureSettings() {
408
- console.log('⚙️ General Settings\n');
409
- // eslint-disable-line no-console
410
-
411
- const answers = await inquirer.prompt([
412
- {
413
- _type: 'confirm',
414
- name: 'enableCaching',
415
- message: 'Enable result caching?',
416
- default: false,
417
- },
418
- {
419
- _type: 'number',
420
- name: 'cacheSize',
421
- message: 'Cache size (number of entries):',
422
- default: 1000,
423
- when: (answers) => answers.enableCaching,
424
- },
425
- {
426
- _type: 'number',
427
- name: 'cacheTtl',
428
- message: 'Cache TTL (seconds):',
429
- default: 3600,
430
- when: (answers) => answers.enableCaching,
431
- },
432
- {
433
- _type: 'number',
434
- name: 'timeout',
435
- message: 'Pipeline timeout (milliseconds):',
436
- default: 30000,
437
- },
438
- ]);
439
-
440
- if (answers.enableCaching) {
441
- this._config.performance = this._config.performance || {};
442
- this._config.performance.caching = {
443
- enabled: true,
444
- maxSize: answers.cacheSize,
445
- ttl: answers.cacheTtl,
446
- };
447
- }
448
-
449
- if (answers.timeout !== 30000) {
450
- this._config.pipeline = this._config.pipeline || {};
451
- this._config.pipeline.timeout = answers.timeout;
452
- }
453
-
454
- console.log('✅ General settings configured\n');
455
- // eslint-disable-line no-console
140
+ this._config = await configureSettings(this._config, inquirer);
456
141
  }
457
142
 
458
143
  /**
459
- * Configure pipeline stages
144
+ * Configure pipeline stages - delegates to configuration module
460
145
  */
461
146
  async configurePipeline() {
462
- console.log('🔄 Pipeline Configuration\n');
463
- // eslint-disable-line no-console
464
-
465
- const availableStages = Object.keys(this._config.plugins).filter(
466
- (_type) => Object.keys(this._config.plugins[_type]).length > 0,
467
- );
468
-
469
- const answers = await inquirer.prompt([
470
- {
471
- _type: 'checkbox',
472
- name: 'stages',
473
- message: 'Select pipeline stages (in order):',
474
- choices: [
475
- {
476
- name: 'Document Loading',
477
- value: 'loader',
478
- checked: availableStages.includes('loader'),
479
- },
480
- {
481
- name: 'Text Embedding',
482
- value: 'embedder',
483
- checked: availableStages.includes('embedder'),
484
- },
485
- {
486
- name: 'Vector Retrieval',
487
- value: 'retriever',
488
- checked: availableStages.includes('retriever'),
489
- },
490
- {
491
- name: 'Language Model',
492
- value: 'llm',
493
- checked: availableStages.includes('llm'),
494
- },
495
- {
496
- name: 'Result Reranking',
497
- value: 'reranker',
498
- checked: availableStages.includes('reranker'),
499
- },
500
- ],
501
- validate: (input) =>
502
- input.length > 0 || 'At least one stage must be selected',
503
- },
504
- {
505
- _type: 'confirm',
506
- name: 'enableRetries',
507
- message: 'Enable automatic retries on failures?',
508
- default: true,
509
- },
510
- {
511
- _type: 'number',
512
- name: 'maxRetries',
513
- message: 'Maximum retry attempts:',
514
- default: 3,
515
- when: (answers) => answers.enableRetries,
516
- },
517
- ]);
518
-
519
- this._config.pipeline = {
520
- stages: answers.stages,
521
- ...this._config.pipeline,
522
- };
523
-
524
- if (answers.enableRetries) {
525
- this._config.pipeline.retries = {
526
- enabled: true,
527
- maxAttempts: answers.maxRetries,
528
- backoff: 'exponential',
529
- };
530
- }
531
-
532
- console.log('✅ Pipeline configuration complete\n');
533
- // eslint-disable-line no-console
147
+ this._config = await configurePipeline(this._config, inquirer);
534
148
  }
535
149
 
536
150
  /**
537
- * Configure performance settings
151
+ * Configure performance settings - delegates to performance module
538
152
  */
539
153
  async configurePerformance() {
540
- console.log('⚡ Performance Settings\n');
541
- // eslint-disable-line no-console
542
-
543
- const answers = await inquirer.prompt([
544
- {
545
- _type: 'confirm',
546
- name: 'enableParallel',
547
- message: 'Enable parallel processing?',
548
- default: false,
549
- },
550
- {
551
- _type: 'number',
552
- name: 'maxConcurrency',
553
- message: 'Maximum concurrent operations:',
554
- default: 3,
555
- when: (answers) => answers.enableParallel,
556
- },
557
- {
558
- _type: 'number',
559
- name: 'batchSize',
560
- message: 'Batch size for parallel processing:',
561
- default: 10,
562
- when: (answers) => answers.enableParallel,
563
- },
564
- {
565
- _type: 'confirm',
566
- name: 'enableStreaming',
567
- message: 'Enable streaming for large documents?',
568
- default: false,
569
- },
570
- {
571
- _type: 'number',
572
- name: 'maxMemoryMB',
573
- message: 'Maximum memory usage (MB):',
574
- default: 512,
575
- when: (answers) => answers.enableStreaming,
576
- },
577
- ]);
578
-
579
- this._config.performance = this._config.performance || {};
580
-
581
- if (answers.enableParallel) {
582
- this._config.performance.parallel = {
583
- enabled: true,
584
- maxConcurrency: answers.maxConcurrency,
585
- batchSize: answers.batchSize,
586
- };
587
- }
588
-
589
- if (answers.enableStreaming) {
590
- this._config.performance.streaming = {
591
- enabled: true,
592
- maxMemoryMB: answers.maxMemoryMB,
593
- bufferSize: 100,
594
- };
595
- }
596
-
597
- console.log('✅ Performance settings configured\n');
598
- // eslint-disable-line no-console
154
+ this._config = await configurePerformance(this._config, inquirer);
599
155
  }
600
156
 
601
157
  /**
602
- * Configure observability settings
158
+ * Configure observability settings - delegates to observability module
603
159
  */
604
160
  async configureObservability() {
605
- console.log('📊 Observability Settings\n');
606
- // eslint-disable-line no-console
607
-
608
- const answers = await inquirer.prompt([
609
- {
610
- _type: 'list',
611
- name: 'logLevel',
612
- message: 'Log level:',
613
- choices: [
614
- { name: 'Debug (verbose)', value: 'debug' },
615
- { name: 'Info (default)', value: 'info' },
616
- { name: 'Warning (minimal)', value: 'warn' },
617
- { name: 'Error (errors only)', value: 'error' },
618
- ],
619
- default: 'info',
620
- },
621
- {
622
- _type: 'confirm',
623
- name: 'enableTracing',
624
- message: 'Enable distributed tracing?',
625
- default: false,
626
- },
627
- {
628
- _type: 'confirm',
629
- name: 'enableMetrics',
630
- message: 'Enable metrics collection?',
631
- default: false,
632
- },
633
- {
634
- _type: 'input',
635
- name: 'exportUrl',
636
- message: 'Metrics/tracing export URL (optional):',
637
- when: (answers) => answers.enableTracing || answers.enableMetrics,
638
- },
639
- ]);
640
-
641
- this._config.observability = {
642
- logging: {
643
- level: answers.logLevel,
644
- structured: true,
645
- events: answers.enableTracing,
646
- },
647
- };
648
-
649
- if (answers.enableTracing) {
650
- this._config.observability.tracing = {
651
- enabled: true,
652
- exportUrl: answers.exportUrl || undefined,
653
- sampleRate: 1,
654
- };
655
- }
656
-
657
- if (answers.enableMetrics) {
658
- this._config.observability.metrics = {
659
- enabled: true,
660
- exportUrl: answers.exportUrl || undefined,
661
- interval: 60000,
662
- };
663
- }
664
-
665
- console.log('✅ Observability settings configured\n');
666
- // eslint-disable-line no-console
161
+ this._config = await configureObservability(this._config, inquirer);
667
162
  }
668
163
 
669
164
  /**
670
- * Preview configuration and save
165
+ * Preview configuration and save - delegates to preview-save module
671
166
  */
672
167
  async previewAndSave() {
673
- console.log('👀 Configuration Preview\n');
674
- // eslint-disable-line no-console
675
-
676
- // Show configuration summary
677
- this.showConfigSummary();
678
-
679
- const answers = await inquirer.prompt([
680
- {
681
- _type: 'confirm',
682
- name: 'save',
683
- message: 'Save this configuration?',
684
- default: true,
685
- },
686
- {
687
- _type: 'input',
688
- name: 'filename',
689
- message: 'Configuration filename:',
690
- default: this._options.outputPath,
691
- when: (answers) => answers.save,
692
- },
693
- {
694
- _type: 'confirm',
695
- name: 'testRun',
696
- message: 'Run a test to validate the configuration?',
697
- default: false,
698
- when: (answers) => answers.save,
699
- },
700
- ]);
701
-
702
- if (!answers.save) {
703
- console.log('Configuration not saved.'); // eslint-disable-line no-console
704
- return;
705
- }
706
-
707
- // Validate and normalize configuration
708
- const validation = validateRagrc(this._config);
709
- if (!validation.valid) {
710
- console.error('❌ Configuration validation failed:'); // eslint-disable-line no-console
711
- validation.errors?.forEach((error) => {
712
- console.error(` ${error.instancePath}: ${error.message}`); // eslint-disable-line no-console
713
- });
714
- throw new Error('Invalid configuration generated');
715
- }
716
-
717
- // Use normalized configuration
718
- this._config = validation.normalized;
719
-
720
- // Save configuration
721
- await fs.writeFile(
722
- answers.filename,
723
- JSON.stringify(this._config, null, 2),
724
- 'utf-8',
725
- );
726
-
727
- this._options.outputPath = answers.filename;
728
-
729
- if (answers.testRun) {
730
- await this.runConfigurationTest();
731
- }
168
+ this._config = await previewAndSave(this._config, inquirer, this._options);
732
169
  }
733
170
 
734
171
  /**
735
- * Show configuration summary
172
+ * Show configuration summary - delegates to preview-save module
736
173
  */
737
174
  showConfigSummary() {
738
- console.log('📋 Configuration Summary:');
739
- // eslint-disable-line no-console
740
- console.log(` Project: ${this._config.metadata?.name || 'Unnamed'}`);
741
- // eslint-disable-line no-console
742
- console.log(
743
- ` Environment: ${this._config.metadata?.environment || 'development'}`,
744
- );
745
- // eslint-disable-line no-console
746
-
747
- console.log('\n🔌 Plugins:');
748
- // eslint-disable-line no-console
749
- for (const [_type, plugins] of Object.entries(this._config.plugins)) {
750
- if (Object.keys(plugins).length > 0) {
751
- console.log(` ${_type}: ${Object.keys(plugins).join(', ')}`);
752
- // eslint-disable-line no-console
753
- }
754
- }
755
-
756
- console.log('\n🔄 Pipeline:');
757
- // eslint-disable-line no-console
758
- console.log(
759
- ` Stages: ${this._config.pipeline?.stages?.join(' → ') || 'default'}`,
760
- );
761
- // eslint-disable-line no-console
762
-
763
- if (this._config.performance?.parallel?.enabled) {
764
- console.log('\n⚡ Performance:');
765
- // eslint-disable-line no-console
766
- console.log(
767
- ` Parallel processing: ${this._config.performance.parallel.maxConcurrency} concurrent`,
768
- );
769
- // eslint-disable-line no-console
770
- }
771
-
772
- if (
773
- this._config.observability?.tracing?.enabled ||
774
- this._config.observability?.metrics?.enabled
775
- ) {
776
- console.log('\n📊 Observability:');
777
- // eslint-disable-line no-console
778
- if (this._config.observability.tracing?.enabled)
779
- console.log(' Tracing: enabled');
780
- // eslint-disable-line no-console
781
- if (this._config.observability.metrics?.enabled)
782
- console.log(' Metrics: enabled');
783
- // eslint-disable-line no-console
784
- }
785
-
786
- console.log('');
787
- // eslint-disable-line no-console
175
+ showConfigSummary(this._config);
788
176
  }
789
177
 
790
178
  /**
791
- * Run configuration test
179
+ * Run configuration test - delegates to preview-save module
792
180
  */
793
181
  async runConfigurationTest() {
794
- console.log('🧪 Running configuration test...\n');
795
- // eslint-disable-line no-console
796
-
797
- try {
798
- // Simulate pipeline creation and basic validation
799
- console.log('✅ Configuration syntax: valid');
800
- // eslint-disable-line no-console
801
- console.log('✅ Plugin dependencies: resolved');
802
- // eslint-disable-line no-console
803
- console.log('✅ Pipeline stages: configured');
804
- // eslint-disable-line no-console
805
-
806
- // Check for potential issues
807
- const dependencies = extractPluginDependencies(this._config);
808
- if (dependencies.length === 0) {
809
- console.warn('⚠️ No plugins configured - pipeline will not function');
810
- // eslint-disable-line no-console
811
- }
182
+ await runConfigurationTest(this._config);
183
+ }
812
184
 
813
- console.log('\n🎉 Configuration test passed!');
814
- // eslint-disable-line no-console
815
- } catch (error) {
816
- console.error('❌ Configuration test failed:', error.message);
817
- // eslint-disable-line no-console
818
- }
185
+ /**
186
+ * Configure plugin-specific settings
187
+ * @param {object} plugin - Plugin information
188
+ * @returns {Promise<object>} Plugin configuration
189
+ */
190
+ async configurePluginSettings(plugin) {
191
+ const {
192
+ configurePluginSettings: configPluginSettings,
193
+ } = require("./wizard/plugin-selection.js");
194
+ return configPluginSettings(plugin, inquirer);
819
195
  }
820
196
 
821
197
  /**
@@ -827,31 +203,31 @@ class InteractiveWizard {
827
203
  // For now, return mock data
828
204
  return {
829
205
  plugins: {
830
- 'file-loader': {
206
+ "file-loader": {
831
207
  metadata: {
832
- name: 'file-loader',
833
- _type: 'loader',
834
- description: 'Load files from filesystem',
208
+ name: "file-loader",
209
+ _type: "loader",
210
+ description: "Load files from filesystem",
835
211
  },
836
- versions: { '1.0.0': {}, '1.1.0': {} },
837
- latest: '1.1.0',
212
+ versions: { "1.0.0": {}, "1.1.0": {} },
213
+ latest: "1.1.0",
838
214
  },
839
- 'openai-embedder': {
215
+ "openai-embedder": {
840
216
  metadata: {
841
- name: 'openai-embedder',
842
- _type: 'embedder',
843
- description: 'OpenAI embeddings',
217
+ name: "openai-embedder",
218
+ _type: "embedder",
219
+ description: "OpenAI embeddings",
844
220
  },
845
- versions: { '2.0.0': {}, '2.1.0': {} },
846
- latest: '2.1.0',
221
+ versions: { "2.0.0": {}, "2.1.0": {} },
222
+ latest: "2.1.0",
847
223
  },
848
224
  },
849
225
  };
850
226
  }
851
227
 
852
228
  /**
853
- * Get available plugins for _type
854
- * @param {string} _type - Plugin _type
229
+ * Get available plugins for type
230
+ * @param {string} _type - Plugin type
855
231
  * @returns {Array<object>} Available plugins
856
232
  */
857
233
  getAvailablePlugins(_type) {
@@ -869,32 +245,32 @@ class InteractiveWizard {
869
245
  }
870
246
 
871
247
  /**
872
- * Get built-in plugin _options
873
- * @param {string} _type - Plugin _type
248
+ * Get built-in plugin options
249
+ * @param {string} _type - Plugin type
874
250
  * @returns {Array<object>} Built-in plugins
875
251
  */
876
252
  getBuiltinPlugins(_type) {
877
253
  const builtins = {
878
254
  loader: [
879
- { name: 'file-loader', description: 'Load files from filesystem' },
880
- { name: 'url-loader', description: 'Load content from URLs' },
255
+ { name: "file-loader", description: "Load files from filesystem" },
256
+ { name: "url-loader", description: "Load content from URLs" },
881
257
  ],
882
258
  embedder: [
883
- { name: 'openai-embedder', description: 'OpenAI embeddings' },
884
- { name: 'local-embedder', description: 'Local embedding model' },
259
+ { name: "openai-embedder", description: "OpenAI embeddings" },
260
+ { name: "local-embedder", description: "Local embedding model" },
885
261
  ],
886
262
  retriever: [
887
- { name: 'vector-retriever', description: 'Vector similarity search' },
888
- { name: 'keyword-retriever', description: 'Keyword-based search' },
263
+ { name: "vector-retriever", description: "Vector similarity search" },
264
+ { name: "keyword-retriever", description: "Keyword-based search" },
889
265
  ],
890
266
  llm: [
891
- { name: 'openai-llm', description: 'OpenAI language models' },
892
- { name: 'local-llm', description: 'Local language model' },
267
+ { name: "openai-llm", description: "OpenAI language models" },
268
+ { name: "local-llm", description: "Local language model" },
893
269
  ],
894
270
  reranker: [
895
271
  {
896
- name: 'similarity-reranker',
897
- description: 'Similarity-based reranking',
272
+ name: "similarity-reranker",
273
+ description: "Similarity-based reranking",
898
274
  },
899
275
  ],
900
276
  };
@@ -904,7 +280,7 @@ class InteractiveWizard {
904
280
 
905
281
  /**
906
282
  * Get plugin information
907
- * @param {string} _type - Plugin _type
283
+ * @param {string} _type - Plugin type
908
284
  * @param {string} name - Plugin name
909
285
  * @returns {object|null} Plugin information
910
286
  */
@@ -954,7 +330,7 @@ class InteractiveWizard {
954
330
  return {
955
331
  success: true,
956
332
  config: this._config,
957
- message: 'Wizard completed successfully',
333
+ message: "Wizard completed successfully",
958
334
  };
959
335
  }
960
336
 
@@ -964,7 +340,7 @@ class InteractiveWizard {
964
340
  * @returns {boolean} Whether input is valid
965
341
  */
966
342
  validateInput(input) {
967
- return typeof input === 'string' && input.trim().length > 0;
343
+ return typeof input === "string" && input.trim().length > 0;
968
344
  }
969
345
 
970
346
  /**