@devilsdev/rag-pipeline-utils 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +293 -369
- package/contracts/loader-contract.json +11 -6
- package/contracts/reranker-contract.json +18 -26
- package/dist/index.cjs +21190 -47
- package/dist/index.d.ts +298 -68
- package/dist/index.mjs +9 -76
- package/package.json +20 -8
- package/src/agentic/agentic-pipeline.js +235 -0
- package/src/agentic/iterative-retriever.js +213 -0
- package/src/agentic/query-planner.js +208 -0
- package/src/agentic/self-critique.js +102 -0
- package/src/ai/adaptive-retrieval.js +0 -3
- package/src/ai/federated-learning.js +0 -3
- package/src/ai/federation.js +1 -6
- package/src/ai/index.js +9 -20
- package/src/ai/model-training.js +0 -3
- package/src/ai/multimodal-processing.js +0 -3
- package/src/ai/multimodal.js +1 -6
- package/src/ai/orchestrator.js +1 -6
- package/src/ai/retrieval-engine.js +1 -6
- package/src/chunking/chunking-engine.js +142 -0
- package/src/chunking/strategies/fixed-size.js +54 -0
- package/src/chunking/strategies/recursive.js +127 -0
- package/src/chunking/strategies/semantic.js +128 -0
- package/src/chunking/strategies/sentence.js +32 -0
- package/src/chunking/strategies/structure-aware.js +138 -0
- package/src/citation/citation-tracker.js +161 -0
- package/src/citation/hallucination-detector.js +74 -0
- package/src/citation/source-mapper.js +95 -0
- package/src/cli/commands/ai-ml/adaptive-commands.js +178 -0
- package/src/cli/commands/ai-ml/federated-commands.js +207 -0
- package/src/cli/commands/ai-ml/index.js +17 -0
- package/src/cli/commands/ai-ml/multimodal-commands.js +187 -0
- package/src/cli/commands/ai-ml/shared.js +51 -0
- package/src/cli/commands/ai-ml/training-commands.js +228 -0
- package/src/cli/commands/ai-ml.js +176 -757
- package/src/cli/commands/plugin-hub/certification-commands.js +221 -0
- package/src/cli/commands/plugin-hub/event-handlers.js +64 -0
- package/src/cli/commands/plugin-hub/hub-commands.js +290 -0
- package/src/cli/commands/plugin-hub/review-commands.js +78 -0
- package/src/cli/commands/plugin-hub.js +106 -782
- package/src/cli/diagnostics/config-checker.js +81 -0
- package/src/cli/diagnostics/dependency-checker.js +69 -0
- package/src/cli/diagnostics/environment-checker.js +45 -0
- package/src/cli/diagnostics/performance-checker.js +51 -0
- package/src/cli/diagnostics/plugin-checker.js +106 -0
- package/src/cli/diagnostics/security-checker.js +61 -0
- package/src/cli/doctor-command.js +70 -369
- package/src/cli/enhanced-cli-commands.js +162 -706
- package/src/cli/handlers/config-handler.js +126 -0
- package/src/cli/handlers/diagnostics-handler.js +113 -0
- package/src/cli/handlers/ingest-handler.js +42 -0
- package/src/cli/handlers/init-handler.js +51 -0
- package/src/cli/handlers/query-handler.js +45 -0
- package/src/cli/handlers/run-handler.js +181 -0
- package/src/cli/interactive-wizard.js +99 -723
- package/src/cli/shared/cli-utils.js +89 -0
- package/src/cli/wizard/configuration.js +148 -0
- package/src/cli/wizard/observability.js +79 -0
- package/src/cli/wizard/performance.js +76 -0
- package/src/cli/wizard/plugin-selection.js +297 -0
- package/src/cli/wizard/preview-save.js +172 -0
- package/src/cli/wizard/project-setup.js +66 -0
- package/src/config/memory-optimization.js +3 -3
- package/src/connectors/anthropic-connector.js +179 -0
- package/src/connectors/base-connector.js +94 -0
- package/src/connectors/cohere-connector.js +143 -0
- package/src/connectors/local-embedder.js +171 -0
- package/src/connectors/memory-retriever.js +150 -0
- package/src/connectors/ollama-connector.js +205 -0
- package/src/connectors/openai-connector.js +173 -0
- package/src/core/create-pipeline.js +21 -1
- package/src/core/observability/tracing.js +2 -1
- package/src/core/pipeline-factory.js +0 -1
- package/src/core/plugin-registry.js +14 -14
- package/src/cost/cost-calculator.js +146 -0
- package/src/cost/cost-tracker.js +260 -0
- package/src/cost/token-budget.js +245 -0
- package/src/dag/core/execution-scheduler.js +2 -1
- package/src/dag/core/topology-validation.js +0 -3
- package/src/dag/dag-engine.js +3 -5
- package/src/dag/dag-node.js +0 -1
- package/src/dag/enhanced-dag-engine.js +171 -99
- package/src/dashboard/dashboard-generator.js +642 -0
- package/src/dashboard/metrics-aggregator.js +409 -0
- package/src/debug/execution-tracer.js +210 -0
- package/src/debug/trace-inspector.js +281 -0
- package/src/dev/dev-server.js +5 -5
- package/src/dev/hot-reload.js +3 -2
- package/src/dx/integration-templates.js +0 -3
- package/src/ecosystem/plugin-analytics-dashboard.js +2 -2
- package/src/ecosystem/plugin-hub.js +2 -2
- package/src/enterprise/audit-logging.js +2 -2
- package/src/enterprise/sso-integration.js +52 -32
- package/src/evaluate/context-metrics.js +127 -0
- package/src/evaluate/evaluator.js +4 -2
- package/src/evaluate/faithfulness.js +117 -0
- package/src/evaluate/groundedness.js +60 -0
- package/src/evaluate/pipeline-evaluator.js +156 -0
- package/src/evaluate/relevance.js +129 -0
- package/src/evaluate/scoring.js +27 -0
- package/src/graph/entity-extractor.js +183 -0
- package/src/graph/graph-index.js +113 -0
- package/src/graph/graph-retriever.js +244 -0
- package/src/graph/knowledge-graph.js +256 -0
- package/src/guardrails/guardrails-pipeline.js +136 -0
- package/src/guardrails/post-generation-guard.js +170 -0
- package/src/guardrails/pre-retrieval-guard.js +161 -0
- package/src/guardrails/retrieval-guard.js +126 -0
- package/src/index.js +255 -9
- package/src/loader/html-loader.js +4 -15
- package/src/loader/markdown-loader.js +4 -15
- package/src/mcp/mcp-server.js +199 -0
- package/src/mcp/mcp-tool-builder.js +121 -0
- package/src/observability/audit-logger.js +0 -2
- package/src/observability/event-logger.js +2 -1
- package/src/observability/security-monitor.js +0 -2
- package/src/observability/telemetry.js +0 -2
- package/src/reranker/cascade-reranker.js +79 -0
- package/src/reranker/embedding-reranker.js +124 -0
- package/src/reranker/llm-reranker.js +6 -3
- package/src/reranker/scoring-reranker.js +123 -0
- package/src/retrieval/bm25-search.js +157 -0
- package/src/retrieval/hybrid-retriever.js +111 -0
- package/src/retrieval/rank-fusion.js +62 -0
- package/src/security/jwt-config.js +4 -3
- package/src/security/jwt-validator.js +2 -1
- package/src/security/plugin-sandbox.js +0 -3
- package/src/security/plugin-signature-verifier.js +4 -6
- package/src/security/validator.js +0 -2
- package/src/streaming/embedding-stream.js +197 -0
- package/src/streaming/sse-adapter.js +168 -0
- package/src/streaming/stream-router.js +210 -0
- package/src/streaming/streaming-embedder.js +231 -0
- package/src/streaming/websocket-adapter.js +171 -0
- package/src/templates/scaffold.js +147 -0
- package/src/templates/template-registry.js +272 -0
- package/src/utils/contract-validator.js +2 -1
- package/src/utils/input-sanitizer.js +80 -99
- package/src/utils/performance-monitor.js +0 -3
- package/src/utils/plugin-scaffolder.js +9 -16
- package/src/utils/rate-limiter.js +0 -1
- package/src/utils/retry-policy.js +0 -3
- package/src/utils/safe-expression-evaluator.js +0 -3
- package/src/utils/secure-logger-config.js +2 -4
- package/src/utils/secure-logging.js +0 -3
- package/src/utils/structured-logger.js +0 -3
- package/src/ai/index-fixed.js +0 -509
- package/src/ai/module-template.js +0 -37
- package/src/ci/workflow-analyzer.js +0 -71
- package/src/cli/plugin-marketplace-commands.js +0 -869
- package/src/dev/README.md +0 -126
- package/src/dev/examples/hot-reload-usage.js +0 -355
- package/src/mocks/openai-embedder.js +0 -119
- package/src/mocks/openai-llm.js +0 -77
- package/src/mocks/pdf-loader.js +0 -21
- package/src/mocks/pinecone-retriever.js +0 -106
- package/src/scripts/script-utilities.js +0 -36
- package/src/utils/ci/diagnostic-reporter.js +0 -57
- package/src/utils/examples/error-formatter-usage.js +0 -351
|
@@ -1,40 +1,65 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Enhanced CLI Commands with Additional Flags and Features
|
|
3
3
|
* Provides improved developer experience with dry-run, version display, and enhanced help
|
|
4
|
+
*
|
|
5
|
+
* This is the thin orchestrator that delegates to focused handler modules.
|
|
4
6
|
*/
|
|
5
7
|
|
|
6
|
-
const { Command } = require(
|
|
7
|
-
const fs = require(
|
|
8
|
-
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
const { Command } = require("commander");
|
|
9
|
+
const fs = require("fs/promises");
|
|
10
|
+
|
|
11
|
+
const { logger } = require("../utils/logger.js");
|
|
12
|
+
|
|
13
|
+
// Shared utilities
|
|
14
|
+
const {
|
|
15
|
+
createProgressBar,
|
|
16
|
+
parsePositiveInteger,
|
|
17
|
+
generateCompletionScript,
|
|
18
|
+
getVersion,
|
|
19
|
+
getExtendedHelp,
|
|
20
|
+
} = require("./shared/cli-utils.js");
|
|
21
|
+
|
|
22
|
+
// Handler modules
|
|
23
|
+
const { handleRun } = require("./handlers/run-handler.js");
|
|
24
|
+
const { handleInit } = require("./handlers/init-handler.js");
|
|
25
|
+
const {
|
|
26
|
+
handleDoctor,
|
|
27
|
+
handleValidate,
|
|
28
|
+
handleInfo,
|
|
29
|
+
} = require("./handlers/diagnostics-handler.js");
|
|
30
|
+
const {
|
|
31
|
+
handleConfigShow,
|
|
32
|
+
handleConfigSet,
|
|
33
|
+
handleConfigGet,
|
|
34
|
+
} = require("./handlers/config-handler.js");
|
|
35
|
+
const { handleIngest } = require("./handlers/ingest-handler.js");
|
|
36
|
+
const { handleQuery } = require("./handlers/query-handler.js");
|
|
12
37
|
|
|
13
38
|
// Mock dependencies for test compatibility
|
|
14
39
|
const mockPluginMarketplaceCommands = () => {
|
|
15
|
-
const cmd = new Command(
|
|
16
|
-
cmd.description(
|
|
40
|
+
const cmd = new Command("plugin");
|
|
41
|
+
cmd.description("Plugin marketplace commands");
|
|
17
42
|
cmd
|
|
18
|
-
.command(
|
|
19
|
-
.argument(
|
|
43
|
+
.command("search")
|
|
44
|
+
.argument("[query]")
|
|
20
45
|
.action(() => {});
|
|
21
46
|
cmd
|
|
22
|
-
.command(
|
|
23
|
-
.argument(
|
|
47
|
+
.command("install")
|
|
48
|
+
.argument("<name>")
|
|
24
49
|
.action(() => {});
|
|
25
|
-
cmd.command(
|
|
50
|
+
cmd.command("publish").action(() => {});
|
|
26
51
|
return cmd;
|
|
27
52
|
};
|
|
28
53
|
|
|
29
54
|
const mockPipelineDoctor = async (options) => {
|
|
30
|
-
console.log(
|
|
55
|
+
console.log("🏥 Running pipeline diagnostics");
|
|
31
56
|
if (options.categories) {
|
|
32
57
|
options.categories.forEach((cat) => {
|
|
33
58
|
console.log(`${cat.charAt(0).toUpperCase() + cat.slice(1)} Issues:`);
|
|
34
|
-
console.log(
|
|
59
|
+
console.log(" ✅ No issues found");
|
|
35
60
|
});
|
|
36
61
|
} else {
|
|
37
|
-
console.log(
|
|
62
|
+
console.log("✅ All systems operational");
|
|
38
63
|
}
|
|
39
64
|
return {
|
|
40
65
|
summary: { total: 0, passed: 0, warnings: 0, errors: 0 },
|
|
@@ -44,19 +69,19 @@ const mockPipelineDoctor = async (options) => {
|
|
|
44
69
|
};
|
|
45
70
|
|
|
46
71
|
const mockValidateSchema = (config) => {
|
|
47
|
-
if (!config || typeof config !==
|
|
72
|
+
if (!config || typeof config !== "object") {
|
|
48
73
|
return {
|
|
49
74
|
valid: false,
|
|
50
|
-
errors: [{ message:
|
|
75
|
+
errors: [{ message: "Invalid configuration format" }],
|
|
51
76
|
};
|
|
52
77
|
}
|
|
53
|
-
if (config.plugins && typeof config.plugins ===
|
|
54
|
-
return { valid: false, errors: [{ message:
|
|
78
|
+
if (config.plugins && typeof config.plugins === "string") {
|
|
79
|
+
return { valid: false, errors: [{ message: "Invalid plugins format" }] };
|
|
55
80
|
}
|
|
56
81
|
const isLegacy =
|
|
57
82
|
config.plugins &&
|
|
58
|
-
typeof config.plugins ===
|
|
59
|
-
Object.values(config.plugins).some((p) => typeof p ===
|
|
83
|
+
typeof config.plugins === "object" &&
|
|
84
|
+
Object.values(config.plugins).some((p) => typeof p === "string");
|
|
60
85
|
return { valid: true, legacy: isLegacy };
|
|
61
86
|
};
|
|
62
87
|
|
|
@@ -70,50 +95,22 @@ class EnhancedCLI {
|
|
|
70
95
|
this.setupCommands();
|
|
71
96
|
}
|
|
72
97
|
|
|
73
|
-
/**
|
|
74
|
-
* Custom parser for positive integers
|
|
75
|
-
*/
|
|
76
|
-
parsePositiveInteger(value) {
|
|
77
|
-
const parsed = parseInt(value, 10);
|
|
78
|
-
if (isNaN(parsed) || parsed <= 0) {
|
|
79
|
-
throw new Error(`Invalid value '${value}'. Expected positive integer.`);
|
|
80
|
-
}
|
|
81
|
-
return parsed;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Create a simple progress bar for CLI output
|
|
86
|
-
* @param {number} percentage - Progress percentage (0-100)
|
|
87
|
-
* @param {number} width - Width of progress bar in characters
|
|
88
|
-
* @returns {string} Progress bar string
|
|
89
|
-
*/
|
|
90
|
-
createProgressBar(percentage, width = 20) {
|
|
91
|
-
const filled = Math.round((percentage / 100) * width);
|
|
92
|
-
const empty = width - filled;
|
|
93
|
-
const bar = '█'.repeat(filled) + '░'.repeat(empty);
|
|
94
|
-
return `[${bar}]`;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
98
|
/**
|
|
98
99
|
* Setup global CLI options
|
|
99
100
|
*/
|
|
100
101
|
setupGlobalOptions() {
|
|
101
102
|
this.program
|
|
102
|
-
.name(
|
|
103
|
-
.description(
|
|
104
|
-
.version(
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
)
|
|
109
|
-
.option(
|
|
110
|
-
.
|
|
111
|
-
.option('--quiet', 'Suppress non-essential output')
|
|
112
|
-
.option('--no-color', 'Disable colored output')
|
|
113
|
-
.option('--dry-run', 'Show what would be done without executing')
|
|
114
|
-
.helpOption('-h, --help', 'Display help information')
|
|
103
|
+
.name("rag-pipeline")
|
|
104
|
+
.description("Enterprise-grade RAG pipeline toolkit")
|
|
105
|
+
.version(getVersion(), "-v, --version", "Display version information")
|
|
106
|
+
.option("--config <path>", "Configuration file path", ".ragrc.json")
|
|
107
|
+
.option("--verbose", "Enable verbose output")
|
|
108
|
+
.option("--quiet", "Suppress non-essential output")
|
|
109
|
+
.option("--no-color", "Disable colored output")
|
|
110
|
+
.option("--dry-run", "Show what would be done without executing")
|
|
111
|
+
.helpOption("-h, --help", "Display help information")
|
|
115
112
|
.showHelpAfterError()
|
|
116
|
-
.addHelpText(
|
|
113
|
+
.addHelpText("after", getExtendedHelp());
|
|
117
114
|
}
|
|
118
115
|
|
|
119
116
|
/**
|
|
@@ -122,672 +119,183 @@ class EnhancedCLI {
|
|
|
122
119
|
setupCommands() {
|
|
123
120
|
// Interactive initialization command
|
|
124
121
|
this.program
|
|
125
|
-
.command(
|
|
126
|
-
.description(
|
|
127
|
-
.option(
|
|
128
|
-
.option(
|
|
129
|
-
.option(
|
|
130
|
-
.option(
|
|
131
|
-
.option(
|
|
122
|
+
.command("init")
|
|
123
|
+
.description("Initialize RAG pipeline configuration")
|
|
124
|
+
.option("--interactive", "Use interactive wizard", true)
|
|
125
|
+
.option("--no-interactive", "Skip interactive wizard")
|
|
126
|
+
.option("--template <name>", "Use configuration template")
|
|
127
|
+
.option("--output <path>", "Output configuration file", ".ragrc.json")
|
|
128
|
+
.option("--force", "Overwrite existing configuration")
|
|
132
129
|
.action(async (options) => {
|
|
133
|
-
await this.
|
|
130
|
+
await handleInit(this.program.opts(), options, (outputPath) =>
|
|
131
|
+
this.createBasicConfig(outputPath),
|
|
132
|
+
);
|
|
134
133
|
});
|
|
135
134
|
|
|
136
135
|
// Run pipeline command
|
|
137
136
|
this.program
|
|
138
|
-
.command(
|
|
139
|
-
.description(
|
|
140
|
-
.option(
|
|
141
|
-
.option(
|
|
142
|
-
.option(
|
|
137
|
+
.command("run")
|
|
138
|
+
.description("Execute RAG pipeline operations")
|
|
139
|
+
.option("--ingest <file>", "Ingest document into pipeline")
|
|
140
|
+
.option("--query <prompt>", "Query the pipeline")
|
|
141
|
+
.option("--stream", "Enable streaming output")
|
|
143
142
|
.option(
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
143
|
+
"--abort-timeout <ms>",
|
|
144
|
+
"Abort operation after timeout (milliseconds)",
|
|
145
|
+
parsePositiveInteger,
|
|
147
146
|
)
|
|
148
|
-
.option(
|
|
147
|
+
.option("--parallel", "Enable parallel processing")
|
|
149
148
|
.action(async (options) => {
|
|
150
|
-
await this.
|
|
149
|
+
await handleRun(this.program.opts(), options);
|
|
151
150
|
});
|
|
152
151
|
|
|
153
152
|
// Doctor command for diagnostics
|
|
154
153
|
this.program
|
|
155
|
-
.command(
|
|
156
|
-
.description(
|
|
157
|
-
.option(
|
|
158
|
-
.option(
|
|
159
|
-
.option(
|
|
154
|
+
.command("doctor")
|
|
155
|
+
.description("Diagnose and fix common issues")
|
|
156
|
+
.option("--category <categories...>", "Diagnostic categories to check")
|
|
157
|
+
.option("--auto-fix", "Automatically fix issues where possible")
|
|
158
|
+
.option("--report <path>", "Save diagnostic report to file")
|
|
160
159
|
.action(async (options) => {
|
|
161
|
-
await this.
|
|
160
|
+
await handleDoctor(this.program.opts(), options, mockPipelineDoctor);
|
|
162
161
|
});
|
|
163
162
|
|
|
164
163
|
// Enhanced ingest command
|
|
165
164
|
this.program
|
|
166
|
-
.command(
|
|
167
|
-
.description(
|
|
168
|
-
.argument(
|
|
169
|
-
.option(
|
|
170
|
-
.option(
|
|
171
|
-
.option(
|
|
172
|
-
.option(
|
|
173
|
-
.option(
|
|
174
|
-
.option(
|
|
165
|
+
.command("ingest")
|
|
166
|
+
.description("Ingest documents into the pipeline")
|
|
167
|
+
.argument("<file>", "Document file to ingest")
|
|
168
|
+
.option("--validate", "Validate document before ingesting")
|
|
169
|
+
.option("--preview", "Show preview of document chunks")
|
|
170
|
+
.option("--parallel", "Enable parallel processing")
|
|
171
|
+
.option("--streaming", "Enable streaming with memory safeguards")
|
|
172
|
+
.option("--trace", "Enable detailed tracing and event logging")
|
|
173
|
+
.option("--stats", "Show pipeline metrics summary at the end")
|
|
175
174
|
.option(
|
|
176
|
-
|
|
177
|
-
|
|
175
|
+
"--export-observability <file>",
|
|
176
|
+
"Export observability data to file",
|
|
178
177
|
)
|
|
179
178
|
.option(
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
179
|
+
"--max-concurrency <number>",
|
|
180
|
+
"Maximum concurrent operations",
|
|
181
|
+
"3",
|
|
183
182
|
)
|
|
184
183
|
.option(
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
184
|
+
"--batch-size <number>",
|
|
185
|
+
"Batch size for parallel processing",
|
|
186
|
+
"10",
|
|
188
187
|
)
|
|
189
|
-
.option(
|
|
188
|
+
.option("--max-memory <number>", "Maximum memory usage in MB", "512")
|
|
190
189
|
.action(async (file, options) => {
|
|
191
|
-
await this.
|
|
190
|
+
await handleIngest(this.program.opts(), file, options);
|
|
192
191
|
});
|
|
193
192
|
|
|
194
193
|
// Enhanced query command
|
|
195
194
|
this.program
|
|
196
|
-
.command(
|
|
197
|
-
.description(
|
|
198
|
-
.argument(
|
|
199
|
-
.option(
|
|
200
|
-
.option(
|
|
201
|
-
.option(
|
|
202
|
-
.option(
|
|
203
|
-
.option(
|
|
195
|
+
.command("query")
|
|
196
|
+
.description("Query the RAG pipeline")
|
|
197
|
+
.argument("<prompt>", "Query prompt")
|
|
198
|
+
.option("--stream", "Enable streaming response")
|
|
199
|
+
.option("--explain", "Show explanation of query processing")
|
|
200
|
+
.option("--parallel", "Enable parallel processing")
|
|
201
|
+
.option("--trace", "Enable detailed tracing and event logging")
|
|
202
|
+
.option("--stats", "Show pipeline metrics summary at the end")
|
|
204
203
|
.option(
|
|
205
|
-
|
|
206
|
-
|
|
204
|
+
"--export-observability <file>",
|
|
205
|
+
"Export observability data to file",
|
|
207
206
|
)
|
|
208
207
|
.action(async (prompt, options) => {
|
|
209
|
-
await this.
|
|
208
|
+
await handleQuery(this.program.opts(), prompt, options);
|
|
210
209
|
});
|
|
211
210
|
|
|
212
211
|
// Configuration validation command
|
|
213
212
|
this.program
|
|
214
|
-
.command(
|
|
215
|
-
.description(
|
|
216
|
-
.option(
|
|
217
|
-
.option(
|
|
218
|
-
.option(
|
|
219
|
-
.option(
|
|
213
|
+
.command("validate")
|
|
214
|
+
.description("Validate configuration file")
|
|
215
|
+
.option("--schema", "Validate against schema only")
|
|
216
|
+
.option("--plugins", "Validate plugin availability")
|
|
217
|
+
.option("--dependencies", "Check plugin dependencies")
|
|
218
|
+
.option("--fix", "Attempt to fix validation issues")
|
|
220
219
|
.action(async (options) => {
|
|
221
|
-
await this.
|
|
220
|
+
await handleValidate(this.program.opts(), options, mockValidateSchema);
|
|
222
221
|
});
|
|
223
222
|
|
|
224
223
|
// Plugin marketplace commands
|
|
225
224
|
this.program.addCommand(mockPluginMarketplaceCommands());
|
|
226
225
|
|
|
227
226
|
// Configuration management commands
|
|
228
|
-
const configCmd = new Command(
|
|
229
|
-
configCmd.description(
|
|
227
|
+
const configCmd = new Command("config");
|
|
228
|
+
configCmd.description("Configuration management commands");
|
|
230
229
|
|
|
231
230
|
configCmd
|
|
232
|
-
.command(
|
|
233
|
-
.description(
|
|
234
|
-
.option(
|
|
235
|
-
.option(
|
|
231
|
+
.command("show")
|
|
232
|
+
.description("Show current configuration")
|
|
233
|
+
.option("--format <format>", "Output format (json, yaml, table)", "json")
|
|
234
|
+
.option("--section <section>", "Show specific configuration section")
|
|
236
235
|
.action(async (options) => {
|
|
237
|
-
await this.
|
|
236
|
+
await handleConfigShow(this.program.opts(), options);
|
|
238
237
|
});
|
|
239
238
|
|
|
240
239
|
configCmd
|
|
241
|
-
.command(
|
|
242
|
-
.description(
|
|
243
|
-
.argument(
|
|
244
|
-
.argument(
|
|
240
|
+
.command("set")
|
|
241
|
+
.description("Set configuration value")
|
|
242
|
+
.argument("<key>", "Configuration key (dot notation supported)")
|
|
243
|
+
.argument("<value>", "Configuration value")
|
|
245
244
|
.action(async (key, value, options) => {
|
|
246
|
-
await this.
|
|
245
|
+
await handleConfigSet(this.program.opts(), key, value);
|
|
247
246
|
});
|
|
248
247
|
|
|
249
248
|
configCmd
|
|
250
|
-
.command(
|
|
251
|
-
.description(
|
|
252
|
-
.argument(
|
|
249
|
+
.command("get")
|
|
250
|
+
.description("Get configuration value")
|
|
251
|
+
.argument("<key>", "Configuration key (dot notation supported)")
|
|
253
252
|
.action(async (key, options) => {
|
|
254
|
-
await this.
|
|
253
|
+
await handleConfigGet(this.program.opts(), key);
|
|
255
254
|
});
|
|
256
255
|
|
|
257
256
|
this.program.addCommand(configCmd);
|
|
258
257
|
|
|
259
258
|
// System information command
|
|
260
259
|
this.program
|
|
261
|
-
.command(
|
|
262
|
-
.description(
|
|
263
|
-
.option(
|
|
264
|
-
.option(
|
|
265
|
-
.option(
|
|
260
|
+
.command("info")
|
|
261
|
+
.description("Show system and plugin information")
|
|
262
|
+
.option("--plugins", "Show registered plugin versions")
|
|
263
|
+
.option("--system", "Show system information")
|
|
264
|
+
.option("--config", "Show configuration summary")
|
|
266
265
|
.action(async (options) => {
|
|
267
|
-
await this.
|
|
266
|
+
await handleInfo(this.program.opts(), options);
|
|
268
267
|
});
|
|
269
268
|
|
|
270
269
|
// Completion command for shell autocompletion
|
|
271
270
|
this.program
|
|
272
|
-
.command(
|
|
273
|
-
.description(
|
|
274
|
-
.argument(
|
|
271
|
+
.command("completion")
|
|
272
|
+
.description("Generate shell completion scripts")
|
|
273
|
+
.argument("[shell]", "Shell type (bash, zsh, fish)", "bash")
|
|
275
274
|
.action(async (shell, options) => {
|
|
276
|
-
|
|
275
|
+
const completionScript = generateCompletionScript(shell);
|
|
276
|
+
console.log(completionScript);
|
|
277
277
|
});
|
|
278
278
|
}
|
|
279
279
|
|
|
280
|
-
/**
|
|
281
|
-
* Handle run command
|
|
282
|
-
*/
|
|
283
|
-
async handleRun(options) {
|
|
284
|
-
const globalOptions = this.program.opts();
|
|
285
|
-
|
|
286
|
-
if (globalOptions.dryRun) {
|
|
287
|
-
logger.info('🧪 Dry run: Would execute pipeline operation');
|
|
288
|
-
if (options.ingest) logger.info(`Would ingest: ${options.ingest}`);
|
|
289
|
-
if (options.query)
|
|
290
|
-
logger.info(`Would query: ${options.query.substring(0, 100)}...`);
|
|
291
|
-
logger.info(`Streaming: ${options.stream ? 'enabled' : 'disabled'}`);
|
|
292
|
-
logger.info(`Parallel: ${options.parallel ? 'enabled' : 'disabled'}`);
|
|
293
|
-
if (options.abortTimeout)
|
|
294
|
-
logger.info(`Abort timeout: ${options.abortTimeout}ms`);
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
try {
|
|
299
|
-
// Load configuration
|
|
300
|
-
const { loadRagConfig } = require('../config/load-config.js');
|
|
301
|
-
const config = await loadRagConfig(globalOptions.config);
|
|
302
|
-
|
|
303
|
-
// Create AbortController for timeout handling
|
|
304
|
-
const abortController = new AbortController();
|
|
305
|
-
let timeoutId;
|
|
306
|
-
|
|
307
|
-
if (options.abortTimeout) {
|
|
308
|
-
timeoutId = setTimeout(() => {
|
|
309
|
-
logger.warn(
|
|
310
|
-
`Operation aborted after ${options.abortTimeout}ms timeout`,
|
|
311
|
-
);
|
|
312
|
-
abortController.abort();
|
|
313
|
-
}, options.abortTimeout);
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
// Setup SIGINT handling for clean shutdown
|
|
317
|
-
const sigintHandler = () => {
|
|
318
|
-
logger.info('Received SIGINT, aborting operation...');
|
|
319
|
-
abortController.abort();
|
|
320
|
-
if (timeoutId) clearTimeout(timeoutId);
|
|
321
|
-
};
|
|
322
|
-
process.once('SIGINT', sigintHandler);
|
|
323
|
-
|
|
324
|
-
try {
|
|
325
|
-
// Create pipeline with performance options
|
|
326
|
-
const { createRagPipeline } = require('../core/create-pipeline.js');
|
|
327
|
-
const pipeline = createRagPipeline(config.pipeline[0], {
|
|
328
|
-
useParallelProcessing: options.parallel,
|
|
329
|
-
useStreamingSafeguards: options.stream,
|
|
330
|
-
useLogging: true,
|
|
331
|
-
useRetry: true,
|
|
332
|
-
});
|
|
333
|
-
|
|
334
|
-
// Execute operation based on options
|
|
335
|
-
if (options.ingest) {
|
|
336
|
-
logger.info(`Starting ingestion of: ${options.ingest}`);
|
|
337
|
-
|
|
338
|
-
if (options.stream) {
|
|
339
|
-
let lastProgressTime = Date.now();
|
|
340
|
-
for await (const progress of pipeline.ingestStream(
|
|
341
|
-
options.ingest,
|
|
342
|
-
)) {
|
|
343
|
-
if (abortController.signal.aborted) {
|
|
344
|
-
logger.info('Ingestion aborted');
|
|
345
|
-
break;
|
|
346
|
-
}
|
|
347
|
-
|
|
348
|
-
const now = Date.now();
|
|
349
|
-
if (progress.type === 'chunk_processed') {
|
|
350
|
-
// Show progress ticks every 500ms or every 10 chunks
|
|
351
|
-
if (
|
|
352
|
-
now - lastProgressTime > 500 ||
|
|
353
|
-
progress.progress.processed % 10 === 0
|
|
354
|
-
) {
|
|
355
|
-
const percentage = Math.round(
|
|
356
|
-
(progress.progress.processed / progress.progress.total) *
|
|
357
|
-
100,
|
|
358
|
-
);
|
|
359
|
-
const progressBar = this.createProgressBar(percentage, 20);
|
|
360
|
-
process.stdout.write(
|
|
361
|
-
`\r${progressBar} ${percentage}% (${progress.progress.processed}/${progress.progress.total}) `,
|
|
362
|
-
);
|
|
363
|
-
lastProgressTime = now;
|
|
364
|
-
}
|
|
365
|
-
} else if (progress.type === 'ingest_complete') {
|
|
366
|
-
process.stdout.write('\n');
|
|
367
|
-
logger.info(
|
|
368
|
-
`✅ Ingestion complete: ${progress.summary.processedChunks}/${progress.summary.totalChunks} chunks processed`,
|
|
369
|
-
);
|
|
370
|
-
if (progress.summary.failedChunks > 0) {
|
|
371
|
-
logger.warn(
|
|
372
|
-
`⚠️ ${progress.summary.failedChunks} chunks failed`,
|
|
373
|
-
);
|
|
374
|
-
}
|
|
375
|
-
} else if (progress.type === 'chunk_failed') {
|
|
376
|
-
process.stdout.write('✗');
|
|
377
|
-
}
|
|
378
|
-
}
|
|
379
|
-
} else {
|
|
380
|
-
// Non-streaming ingestion with progress callback
|
|
381
|
-
let currentStage = '';
|
|
382
|
-
const progressCallback = (progress) => {
|
|
383
|
-
if (progress.stage !== currentStage) {
|
|
384
|
-
if (currentStage) process.stdout.write('\n');
|
|
385
|
-
currentStage = progress.stage;
|
|
386
|
-
logger.info(
|
|
387
|
-
`${progress.stage}: ${progress.message || 'Processing...'}`,
|
|
388
|
-
);
|
|
389
|
-
}
|
|
390
|
-
if (
|
|
391
|
-
progress.completed !== undefined &&
|
|
392
|
-
progress.total !== undefined
|
|
393
|
-
) {
|
|
394
|
-
const percentage = Math.round(
|
|
395
|
-
(progress.completed / progress.total) * 100,
|
|
396
|
-
);
|
|
397
|
-
const progressBar = this.createProgressBar(percentage, 20);
|
|
398
|
-
process.stdout.write(
|
|
399
|
-
`\r${progressBar} ${percentage}% (${progress.completed}/${progress.total}) `,
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
};
|
|
403
|
-
|
|
404
|
-
const pipelineWithProgress = createRagPipeline(config.pipeline[0], {
|
|
405
|
-
useParallelProcessing: options.parallel,
|
|
406
|
-
useStreamingSafeguards: false,
|
|
407
|
-
useLogging: true,
|
|
408
|
-
useRetry: true,
|
|
409
|
-
onProgress: progressCallback,
|
|
410
|
-
});
|
|
411
|
-
|
|
412
|
-
await pipelineWithProgress.ingest(options.ingest);
|
|
413
|
-
process.stdout.write('\n');
|
|
414
|
-
logger.info('✅ Ingestion completed successfully');
|
|
415
|
-
}
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
if (options.query) {
|
|
419
|
-
logger.info(`Executing query: ${options.query.substring(0, 100)}...`);
|
|
420
|
-
|
|
421
|
-
if (options.stream) {
|
|
422
|
-
process.stdout.write('\nResponse: ');
|
|
423
|
-
for await (const token of pipeline.queryStream(options.query)) {
|
|
424
|
-
if (abortController.signal.aborted) {
|
|
425
|
-
logger.info('\nQuery aborted');
|
|
426
|
-
break;
|
|
427
|
-
}
|
|
428
|
-
process.stdout.write(token);
|
|
429
|
-
}
|
|
430
|
-
process.stdout.write('\n\n');
|
|
431
|
-
} else {
|
|
432
|
-
const response = await pipeline.query(options.query);
|
|
433
|
-
logger.info('Query completed successfully');
|
|
434
|
-
console.log('\nResponse:', response);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
if (!options.ingest && !options.query) {
|
|
439
|
-
logger.error(
|
|
440
|
-
'No operation specified. Use --ingest <file> or --query <prompt>',
|
|
441
|
-
);
|
|
442
|
-
process.exit(1);
|
|
443
|
-
}
|
|
444
|
-
} finally {
|
|
445
|
-
// Cleanup
|
|
446
|
-
process.removeListener('SIGINT', sigintHandler);
|
|
447
|
-
if (timeoutId) clearTimeout(timeoutId);
|
|
448
|
-
}
|
|
449
|
-
} catch (error) {
|
|
450
|
-
logger.error('Pipeline operation failed', { error: error.message });
|
|
451
|
-
process.exit(1);
|
|
452
|
-
}
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
/**
|
|
456
|
-
* Handle init command
|
|
457
|
-
*/
|
|
458
|
-
async handleInit(options) {
|
|
459
|
-
try {
|
|
460
|
-
const globalOptions = this.program.opts();
|
|
461
|
-
|
|
462
|
-
if (globalOptions.dryRun) {
|
|
463
|
-
console.log('🧪 Dry run: Would initialize RAG pipeline configuration');
|
|
464
|
-
console.log(`Output file: ${options.output}`);
|
|
465
|
-
console.log(`Interactive mode: ${!options.noInteractive}`);
|
|
466
|
-
return;
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
// Check if config already exists
|
|
470
|
-
if (!options.force) {
|
|
471
|
-
try {
|
|
472
|
-
await fs.access(options.output);
|
|
473
|
-
console.log(
|
|
474
|
-
`❌ Configuration file already exists: ${options.output}`,
|
|
475
|
-
);
|
|
476
|
-
console.log(
|
|
477
|
-
'Use --force to overwrite or choose a different output path.',
|
|
478
|
-
);
|
|
479
|
-
return;
|
|
480
|
-
} catch (error) {
|
|
481
|
-
// File doesn't exist, which is good
|
|
482
|
-
}
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
if (!options.noInteractive) {
|
|
486
|
-
console.log('🧙♂️ Starting interactive configuration wizard...\n');
|
|
487
|
-
// Mock interactive wizard
|
|
488
|
-
await this.createBasicConfig(options.output);
|
|
489
|
-
} else {
|
|
490
|
-
await this.createBasicConfig(options.output);
|
|
491
|
-
}
|
|
492
|
-
} catch (error) {
|
|
493
|
-
logger.error('❌ Initialization failed:', error.message);
|
|
494
|
-
process.exit(1);
|
|
495
|
-
}
|
|
496
|
-
}
|
|
497
|
-
|
|
498
|
-
/**
|
|
499
|
-
* Handle doctor command
|
|
500
|
-
*/
|
|
501
|
-
async handleDoctor(options) {
|
|
502
|
-
try {
|
|
503
|
-
const globalOptions = this.program.opts();
|
|
504
|
-
|
|
505
|
-
const doctorOptions = {
|
|
506
|
-
configPath: globalOptions.config,
|
|
507
|
-
verbose: globalOptions.verbose,
|
|
508
|
-
autoFix: options.autoFix,
|
|
509
|
-
categories: options.category,
|
|
510
|
-
};
|
|
511
|
-
|
|
512
|
-
const report = await mockPipelineDoctor(doctorOptions);
|
|
513
|
-
|
|
514
|
-
if (options.report) {
|
|
515
|
-
await fs.writeFile(options.report, JSON.stringify(report, null, 2));
|
|
516
|
-
console.log(`📋 Diagnostic report saved to: ${options.report}`);
|
|
517
|
-
}
|
|
518
|
-
} catch (error) {
|
|
519
|
-
logger.error('❌ Doctor command failed:', error.message);
|
|
520
|
-
process.exit(1);
|
|
521
|
-
}
|
|
522
|
-
}
|
|
523
|
-
|
|
524
|
-
/**
|
|
525
|
-
* Handle ingest command
|
|
526
|
-
*/
|
|
527
|
-
async handleIngest(file, options) {
|
|
528
|
-
try {
|
|
529
|
-
const globalOptions = this.program.opts();
|
|
530
|
-
|
|
531
|
-
if (globalOptions.dryRun) {
|
|
532
|
-
console.log('🧪 Dry run: Would ingest document');
|
|
533
|
-
console.log(`File: ${file}`);
|
|
534
|
-
console.log(
|
|
535
|
-
`Parallel processing: ${options.parallel ? 'enabled' : 'disabled'}`,
|
|
536
|
-
);
|
|
537
|
-
console.log(`Streaming: ${options.streaming ? 'enabled' : 'disabled'}`);
|
|
538
|
-
console.log(`Tracing: ${options.trace ? 'enabled' : 'disabled'}`);
|
|
539
|
-
|
|
540
|
-
if (options.preview) {
|
|
541
|
-
console.log('📄 Document Preview:');
|
|
542
|
-
}
|
|
543
|
-
if (options.validate) {
|
|
544
|
-
console.log('File validation passed');
|
|
545
|
-
}
|
|
546
|
-
return;
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
console.log(`📄 Ingesting document: ${file}`);
|
|
550
|
-
console.log('✅ Document ingested successfully!');
|
|
551
|
-
} catch (error) {
|
|
552
|
-
logger.error('❌ Document ingestion failed:', error.message);
|
|
553
|
-
process.exit(1);
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
|
|
557
|
-
/**
|
|
558
|
-
* Handle query command
|
|
559
|
-
*/
|
|
560
|
-
async handleQuery(prompt, options) {
|
|
561
|
-
try {
|
|
562
|
-
const globalOptions = this.program.opts();
|
|
563
|
-
|
|
564
|
-
if (globalOptions.dryRun) {
|
|
565
|
-
console.log('🧪 Dry run: Would execute query');
|
|
566
|
-
console.log(
|
|
567
|
-
`Prompt: ${prompt.substring(0, 100)}${prompt.length > 100 ? '...' : ''}`,
|
|
568
|
-
);
|
|
569
|
-
console.log(`Streaming: ${options.stream ? 'enabled' : 'disabled'}`);
|
|
570
|
-
|
|
571
|
-
if (options.explain) {
|
|
572
|
-
console.log('🔍 Query Explanation:');
|
|
573
|
-
console.log('Processing steps:');
|
|
574
|
-
console.log(' 1. Text embedding generation');
|
|
575
|
-
console.log(' 2. Vector similarity search');
|
|
576
|
-
console.log(' 3. Context retrieval');
|
|
577
|
-
console.log(' 4. Language model generation');
|
|
578
|
-
}
|
|
579
|
-
return;
|
|
580
|
-
}
|
|
581
|
-
|
|
582
|
-
console.log(
|
|
583
|
-
`🤔 Processing query: ${prompt.substring(0, 100)}${prompt.length > 100 ? '...' : ''}`,
|
|
584
|
-
);
|
|
585
|
-
console.log('\n📝 Response:');
|
|
586
|
-
console.log('Mock response to query');
|
|
587
|
-
} catch (error) {
|
|
588
|
-
logger.error('❌ Query processing failed:', error.message);
|
|
589
|
-
process.exit(1);
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
|
|
593
|
-
/**
|
|
594
|
-
* Handle validate command
|
|
595
|
-
*/
|
|
596
|
-
async handleValidate(options) {
|
|
597
|
-
try {
|
|
598
|
-
const globalOptions = this.program.opts();
|
|
599
|
-
const configPath = globalOptions.config;
|
|
600
|
-
|
|
601
|
-
console.log(`🔍 Validating configuration: ${configPath}`);
|
|
602
|
-
|
|
603
|
-
// Load and validate configuration
|
|
604
|
-
const configContent = await fs.readFile(configPath, 'utf-8');
|
|
605
|
-
const config = JSON.parse(configContent);
|
|
606
|
-
|
|
607
|
-
const validation = mockValidateSchema(config);
|
|
608
|
-
|
|
609
|
-
if (validation.valid) {
|
|
610
|
-
console.log('✅ Configuration is valid');
|
|
611
|
-
|
|
612
|
-
if (validation.legacy) {
|
|
613
|
-
console.log('⚠️ Using legacy format - consider upgrading');
|
|
614
|
-
}
|
|
615
|
-
} else {
|
|
616
|
-
console.log('❌ Configuration validation failed:');
|
|
617
|
-
validation.errors?.forEach((error) => {
|
|
618
|
-
console.log(` ${error.instancePath || 'root'}: ${error.message}`);
|
|
619
|
-
});
|
|
620
|
-
}
|
|
621
|
-
} catch (error) {
|
|
622
|
-
logger.error('❌ Validation failed:', error.message);
|
|
623
|
-
process.exit(1);
|
|
624
|
-
}
|
|
625
|
-
}
|
|
626
|
-
|
|
627
|
-
/**
|
|
628
|
-
* Handle config show command
|
|
629
|
-
*/
|
|
630
|
-
async handleConfigShow(options) {
|
|
631
|
-
try {
|
|
632
|
-
const globalOptions = this.program.opts();
|
|
633
|
-
const config = JSON.parse(
|
|
634
|
-
await fs.readFile(globalOptions.config, 'utf-8'),
|
|
635
|
-
);
|
|
636
|
-
|
|
637
|
-
let output = config;
|
|
638
|
-
if (options.section) {
|
|
639
|
-
const sections = options.section.split('.');
|
|
640
|
-
for (const section of sections) {
|
|
641
|
-
output = output[section];
|
|
642
|
-
if (output === undefined) {
|
|
643
|
-
console.log(`❌ Section not found: ${options.section}`);
|
|
644
|
-
return;
|
|
645
|
-
}
|
|
646
|
-
}
|
|
647
|
-
}
|
|
648
|
-
|
|
649
|
-
console.log(JSON.stringify(output, null, 2));
|
|
650
|
-
} catch (error) {
|
|
651
|
-
logger.error('❌ Failed to show configuration:', error.message);
|
|
652
|
-
process.exit(1);
|
|
653
|
-
}
|
|
654
|
-
}
|
|
655
|
-
|
|
656
|
-
/**
|
|
657
|
-
* Handle config set command
|
|
658
|
-
*/
|
|
659
|
-
async handleConfigSet(key, value) {
|
|
660
|
-
try {
|
|
661
|
-
const globalOptions = this.program.opts();
|
|
662
|
-
const config = JSON.parse(
|
|
663
|
-
await fs.readFile(globalOptions.config, 'utf-8'),
|
|
664
|
-
);
|
|
665
|
-
|
|
666
|
-
// Set value using dot notation
|
|
667
|
-
const keys = key.split('.');
|
|
668
|
-
let current = config;
|
|
669
|
-
|
|
670
|
-
for (let i = 0; i < keys.length - 1; i++) {
|
|
671
|
-
if (!current[keys[i]]) {
|
|
672
|
-
current[keys[i]] = {};
|
|
673
|
-
}
|
|
674
|
-
current = current[keys[i]];
|
|
675
|
-
}
|
|
676
|
-
|
|
677
|
-
// Parse value
|
|
678
|
-
let parsedValue;
|
|
679
|
-
try {
|
|
680
|
-
parsedValue = JSON.parse(value);
|
|
681
|
-
} catch (error) {
|
|
682
|
-
parsedValue = value; // Keep as string
|
|
683
|
-
}
|
|
684
|
-
|
|
685
|
-
current[keys[keys.length - 1]] = parsedValue;
|
|
686
|
-
|
|
687
|
-
// Save configuration
|
|
688
|
-
await fs.writeFile(globalOptions.config, JSON.stringify(config, null, 2));
|
|
689
|
-
console.log('✅ Configuration updated');
|
|
690
|
-
} catch (error) {
|
|
691
|
-
logger.error('❌ Failed to set configuration:', error.message);
|
|
692
|
-
process.exit(1);
|
|
693
|
-
}
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
/**
|
|
697
|
-
* Handle config get command
|
|
698
|
-
*/
|
|
699
|
-
async handleConfigGet(key) {
|
|
700
|
-
try {
|
|
701
|
-
const globalOptions = this.program.opts();
|
|
702
|
-
const config = JSON.parse(
|
|
703
|
-
await fs.readFile(globalOptions.config, 'utf-8'),
|
|
704
|
-
);
|
|
705
|
-
|
|
706
|
-
const keys = key.split('.');
|
|
707
|
-
let value = config;
|
|
708
|
-
|
|
709
|
-
for (const k of keys) {
|
|
710
|
-
value = value[k];
|
|
711
|
-
if (value === undefined) {
|
|
712
|
-
console.log('❌ Key not found');
|
|
713
|
-
return;
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
console.log(JSON.stringify(value, null, 2));
|
|
718
|
-
} catch (error) {
|
|
719
|
-
logger.error('❌ Failed to get configuration:', error.message);
|
|
720
|
-
process.exit(1);
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
|
|
724
|
-
/**
|
|
725
|
-
* Handle info command
|
|
726
|
-
*/
|
|
727
|
-
async handleInfo(options) {
|
|
728
|
-
try {
|
|
729
|
-
console.log('📊 RAG Pipeline Information\n');
|
|
730
|
-
|
|
731
|
-
// System information
|
|
732
|
-
if ((!options.plugins && !options.config) || options.system) {
|
|
733
|
-
console.log('System:');
|
|
734
|
-
console.log(` Node.js: ${process.version}`);
|
|
735
|
-
console.log(` Platform: ${process.platform} ${process.arch}`);
|
|
736
|
-
console.log(
|
|
737
|
-
` Memory: ${Math.round(process.memoryUsage().heapUsed / 1024 / 1024)}MB used`,
|
|
738
|
-
);
|
|
739
|
-
console.log('');
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
// Configuration summary
|
|
743
|
-
if ((!options.system && !options.plugins) || options.config) {
|
|
744
|
-
const globalOptions = this.program.opts();
|
|
745
|
-
try {
|
|
746
|
-
const config = JSON.parse(
|
|
747
|
-
await fs.readFile(globalOptions.config, 'utf-8'),
|
|
748
|
-
);
|
|
749
|
-
console.log('Configuration:');
|
|
750
|
-
console.log(` File: ${globalOptions.config}`);
|
|
751
|
-
console.log(` Format: ${config.plugins ? 'Enhanced' : 'Legacy'}`);
|
|
752
|
-
console.log('');
|
|
753
|
-
} catch (error) {
|
|
754
|
-
console.log('Configuration: Not found or invalid');
|
|
755
|
-
console.log('');
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
} catch (error) {
|
|
759
|
-
logger.error('❌ Failed to get system information:', error.message);
|
|
760
|
-
process.exit(1);
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
|
|
764
|
-
/**
|
|
765
|
-
* Handle completion command
|
|
766
|
-
*/
|
|
767
|
-
async handleCompletion(shell) {
|
|
768
|
-
const completionScript = this.generateCompletionScript(shell);
|
|
769
|
-
console.log(completionScript);
|
|
770
|
-
}
|
|
771
|
-
|
|
772
280
|
/**
|
|
773
281
|
* Create basic configuration
|
|
774
282
|
*/
|
|
775
283
|
async createBasicConfig(outputPath) {
|
|
776
|
-
const { validateRagrc } = require(
|
|
284
|
+
const { validateRagrc } = require("../config/enhanced-ragrc-schema.js");
|
|
777
285
|
|
|
778
286
|
const base = {
|
|
779
|
-
namespace:
|
|
287
|
+
namespace: "default",
|
|
780
288
|
metadata: {
|
|
781
|
-
name:
|
|
782
|
-
version:
|
|
783
|
-
description:
|
|
289
|
+
name: "rag-pipeline",
|
|
290
|
+
version: "1.0.0",
|
|
291
|
+
description: "RAG pipeline configuration",
|
|
784
292
|
createdAt: new Date().toISOString(),
|
|
785
293
|
},
|
|
786
294
|
pipeline: [
|
|
787
|
-
{ stage:
|
|
788
|
-
{ stage:
|
|
789
|
-
{ stage:
|
|
790
|
-
{ stage:
|
|
295
|
+
{ stage: "loader", name: "fs-loader", version: "1.0.0" },
|
|
296
|
+
{ stage: "embedder", name: "mock-embedder", version: "1.0.0" },
|
|
297
|
+
{ stage: "retriever", name: "mock-retriever", version: "1.0.0" },
|
|
298
|
+
{ stage: "llm", name: "mock-llm", version: "1.0.0" },
|
|
791
299
|
],
|
|
792
300
|
performance: {
|
|
793
301
|
parallel: {
|
|
@@ -802,7 +310,7 @@ class EnhancedCLI {
|
|
|
802
310
|
},
|
|
803
311
|
observability: {
|
|
804
312
|
logging: {
|
|
805
|
-
level:
|
|
313
|
+
level: "info",
|
|
806
314
|
structured: true,
|
|
807
315
|
},
|
|
808
316
|
},
|
|
@@ -811,9 +319,9 @@ class EnhancedCLI {
|
|
|
811
319
|
const result = validateRagrc(base);
|
|
812
320
|
if (!result.valid) {
|
|
813
321
|
result.errors?.forEach((error) =>
|
|
814
|
-
console.error(` ${error.instancePath ||
|
|
322
|
+
console.error(` ${error.instancePath || ""}: ${error.message}`),
|
|
815
323
|
);
|
|
816
|
-
throw new Error(
|
|
324
|
+
throw new Error("Failed to generate valid configuration");
|
|
817
325
|
}
|
|
818
326
|
|
|
819
327
|
if (outputPath) {
|
|
@@ -827,58 +335,6 @@ class EnhancedCLI {
|
|
|
827
335
|
return result.normalized;
|
|
828
336
|
}
|
|
829
337
|
|
|
830
|
-
/**
|
|
831
|
-
* Get version information from package.json
|
|
832
|
-
*/
|
|
833
|
-
getVersion() {
|
|
834
|
-
return pkg.version;
|
|
835
|
-
}
|
|
836
|
-
|
|
837
|
-
/**
|
|
838
|
-
* Get extended help text
|
|
839
|
-
*/
|
|
840
|
-
getExtendedHelp() {
|
|
841
|
-
return `
|
|
842
|
-
Examples:
|
|
843
|
-
rag-pipeline init --interactive Initialize with wizard
|
|
844
|
-
rag-pipeline doctor --auto-fix Diagnose and fix issues
|
|
845
|
-
rag-pipeline plugin search openai Search for plugins
|
|
846
|
-
rag-pipeline config show plugins Show plugin configuration
|
|
847
|
-
|
|
848
|
-
For more information, visit: https://github.com/DevilsDev/rag-pipeline-utils
|
|
849
|
-
`;
|
|
850
|
-
}
|
|
851
|
-
|
|
852
|
-
/**
|
|
853
|
-
* Generate shell completion script
|
|
854
|
-
*/
|
|
855
|
-
generateCompletionScript(shell) {
|
|
856
|
-
switch (shell) {
|
|
857
|
-
case 'bash':
|
|
858
|
-
return `# RAG Pipeline bash completion
|
|
859
|
-
_rag_pipeline_completions() {
|
|
860
|
-
COMPREPLY=($(compgen -W "init doctor ingest query plugin config info validate completion" -- "\${COMP_WORDS[COMP_CWORD]}"))
|
|
861
|
-
}
|
|
862
|
-
complete -F _rag_pipeline_completions rag-pipeline`;
|
|
863
|
-
|
|
864
|
-
case 'zsh':
|
|
865
|
-
return `# RAG Pipeline zsh completion
|
|
866
|
-
#compdef rag-pipeline
|
|
867
|
-
_rag_pipeline() {
|
|
868
|
-
_arguments \\
|
|
869
|
-
'1:command:(init doctor ingest query plugin config info validate completion)'
|
|
870
|
-
}
|
|
871
|
-
_rag_pipeline "$@"`;
|
|
872
|
-
|
|
873
|
-
case 'fish':
|
|
874
|
-
return `# RAG Pipeline fish completion
|
|
875
|
-
complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query plugin config info validate completion'`;
|
|
876
|
-
|
|
877
|
-
default:
|
|
878
|
-
return `# Completion not available for ${shell}`;
|
|
879
|
-
}
|
|
880
|
-
}
|
|
881
|
-
|
|
882
338
|
/**
|
|
883
339
|
* Run the CLI
|
|
884
340
|
*/
|
|
@@ -886,8 +342,8 @@ complete -c rag-pipeline -n '__fish_use_subcommand' -a 'init doctor ingest query
|
|
|
886
342
|
try {
|
|
887
343
|
await this.program.parseAsync(argv);
|
|
888
344
|
} catch (error) {
|
|
889
|
-
if (error.code !==
|
|
890
|
-
logger.error(
|
|
345
|
+
if (error.code !== "commander.unknownCommand") {
|
|
346
|
+
logger.error("❌ CLI error:", error.message);
|
|
891
347
|
process.exit(1);
|
|
892
348
|
}
|
|
893
349
|
throw error;
|
|
@@ -913,7 +369,7 @@ const ingest = async () => ({ ok: true });
|
|
|
913
369
|
const query = async () => ({ ok: true });
|
|
914
370
|
const handleError = async () => ({ ok: true });
|
|
915
371
|
const validateArgs = () => true;
|
|
916
|
-
const showHelp = () =>
|
|
372
|
+
const showHelp = () => "help text";
|
|
917
373
|
|
|
918
374
|
// Extract createBasicConfig from the class for direct export
|
|
919
375
|
const createBasicConfig = async (outputPath) => {
|