@devilsdev/rag-pipeline-utils 2.1.12 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ragrc.schema.json +218 -0
- package/README.md +178 -550
- package/bin/cli.js +8 -14
- package/contracts/reranker-contract.json +69 -0
- package/dist/index.d.ts +237 -0
- package/dist/index.mjs +22 -7
- package/package.json +18 -13
- package/src/ai/adaptive-retrieval.js +191 -0
- package/src/ai/federated-learning.js +214 -0
- package/src/ai/federation/federated-learning-coordinator.js +345 -0
- package/src/ai/federation.js +76 -0
- package/src/ai/index-fixed.js +509 -0
- package/src/ai/index.js +33 -0
- package/src/ai/model-trainer.js +217 -0
- package/src/ai/model-training.js +205 -0
- package/src/ai/module-template.js +37 -0
- package/src/ai/multimodal/multi-modal-processor.js +345 -0
- package/src/ai/multimodal-processing.js +268 -0
- package/src/ai/multimodal.js +71 -0
- package/src/ai/orchestrator.js +70 -0
- package/src/ai/retrieval/adaptive-retrieval-engine.js +398 -0
- package/src/ai/retrieval-engine.js +69 -0
- package/src/ai/training/model-training-orchestrator.js +228 -0
- package/src/ci/workflow-analyzer.js +71 -0
- package/src/cli/commands/ai-ml.js +967 -0
- package/src/cli/commands/docs.js +228 -0
- package/src/cli/commands/dx.js +538 -0
- package/src/cli/commands/plugin-hub.js +875 -0
- package/src/cli/doctor-command.js +639 -0
- package/src/cli/enhanced-cli-commands.js +945 -0
- package/src/cli/interactive-wizard.js +997 -0
- package/src/cli/plugin-marketplace-commands.js +869 -0
- package/src/cli.js +17 -0
- package/src/config/enhanced-ragrc-schema.js +632 -0
- package/src/config/load-config.js +54 -0
- package/src/config/load-plugin-config.js +69 -0
- package/src/config/normalize-config.js +105 -0
- package/src/config/schema-validator.js +315 -0
- package/src/config/validate-plugin-schema.js +35 -0
- package/src/config/validate-schema.js +113 -0
- package/src/contracts/plugin-contracts.js +33 -0
- package/src/core/create-pipeline.js +99 -0
- package/src/core/dependency-injection.js +242 -0
- package/src/core/observability/event-logger.js +241 -0
- package/src/core/observability/instrumented-pipeline.js +609 -0
- package/src/core/observability/metrics.js +497 -0
- package/src/core/observability/tracing.js +781 -0
- package/src/core/performance/benchmark.js +516 -0
- package/src/core/performance/parallel-processor.js +240 -0
- package/src/core/performance/streaming-safeguards.js +620 -0
- package/src/core/pipeline-factory.js +60 -0
- package/src/core/plugin-contracts.js +89 -0
- package/src/core/plugin-marketplace/plugin-metadata.js +485 -0
- package/src/core/plugin-marketplace/plugin-publisher.js +574 -0
- package/src/core/plugin-marketplace/plugin-registry-format.js +419 -0
- package/src/core/plugin-marketplace/version-resolver.js +61 -0
- package/src/core/plugin-registry.js +294 -0
- package/src/dag/dag-engine.js +954 -0
- package/src/dag/dag-node.js +76 -0
- package/src/dag/enhanced-dag-engine.js +822 -0
- package/src/dx/index.js +40 -0
- package/src/dx/integration-templates.js +665 -0
- package/src/dx/performance-profiler.js +968 -0
- package/src/dx/realtime-debugger.js +755 -0
- package/src/dx/visual-pipeline-builder.js +578 -0
- package/src/ecosystem/plugin-analytics-dashboard.js +778 -0
- package/src/ecosystem/plugin-certification.js +580 -0
- package/src/ecosystem/plugin-hub.js +821 -0
- package/src/enterprise/audit-logging.js +967 -0
- package/src/enterprise/data-governance.js +408 -0
- package/src/enterprise/multi-tenancy.js +496 -0
- package/src/enterprise/sso-integration.js +1030 -0
- package/src/evaluate/evaluator.js +52 -0
- package/src/evaluate/scoring.js +91 -0
- package/src/index.js +67 -0
- package/src/ingest.js +59 -0
- package/src/loader/csv-loader.js +38 -0
- package/src/loader/directory-loader.js +48 -0
- package/src/loader/html-loader.js +62 -0
- package/src/loader/markdown-loader.js +67 -0
- package/src/mocks/openai-embedder.js +119 -0
- package/src/mocks/openai-llm.js +77 -0
- package/src/mocks/pdf-loader.js +21 -0
- package/src/mocks/pinecone-retriever.js +106 -0
- package/src/observability/audit-logger.js +758 -0
- package/src/observability/dashboard-config.json +675 -0
- package/src/observability/event-logger.js +310 -0
- package/src/observability/metrics.js +691 -0
- package/src/observability/security-monitor.js +830 -0
- package/src/observability/slo-monitor.js +319 -0
- package/src/observability/telemetry.js +686 -0
- package/src/pipeline/pipeline.js +18 -0
- package/src/plugins/sample-embedder.js +53 -0
- package/src/plugins/sample-llm.js +30 -0
- package/src/plugins/sample-loader.js +24 -0
- package/src/plugins/sample-retriever.js +84 -0
- package/src/query.js +117 -0
- package/src/reranker/llm-reranker.js +111 -0
- package/src/scripts/script-utilities.js +36 -0
- package/src/security/plugin-sandbox.js +586 -0
- package/src/security/plugin-signature-verifier.js +414 -0
- package/src/security/validator.js +36 -0
- package/src/utils/ci/diagnostic-reporter.js +57 -0
- package/src/utils/logger.js +211 -0
- package/src/utils/performance-monitor.js +561 -0
- package/src/utils/plugin-scaffolder.js +447 -0
- package/src/utils/rate-limiter.js +244 -0
- package/src/utils/retry-policy.js +443 -0
- package/src/utils/retry.js +107 -0
- package/src/utils/safe-expression-evaluator.js +579 -0
- package/src/utils/secure-logging.js +338 -0
- package/src/utils/structured-logger.js +388 -0
- package/src/utils/validate-plugin-contract.js +42 -0
package/README.md
CHANGED
|
@@ -1,658 +1,286 @@
|
|
|
1
|
-
#
|
|
1
|
+
# rag-pipeline-utils
|
|
2
|
+
|
|
3
|
+
**Modular toolkit for building production-ready RAG pipelines in Node.js**
|
|
2
4
|
|
|
3
5
|
[](https://github.com/DevilsDev/rag-pipeline-utils/actions)
|
|
4
|
-
[](https://
|
|
6
|
+
[](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils)
|
|
7
|
+
[](https://codecov.io/gh/DevilsDev/rag-pipeline-utils)
|
|
8
|
+
[](https://opensource.org/licenses/GPL-3.0)
|
|
5
9
|
[](https://nodejs.org/)
|
|
6
|
-
[](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE)
|
|
7
10
|
[](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils)
|
|
11
|
+
[](https://github.com/DevilsDev/rag-pipeline-utils/discussions)
|
|
8
12
|
|
|
9
|
-
|
|
13
|
+
---
|
|
10
14
|
|
|
11
|
-
##
|
|
15
|
+
## Why This Exists
|
|
12
16
|
|
|
13
|
-
|
|
14
|
-
npm install @devilsdev/rag-pipeline-utils
|
|
15
|
-
```
|
|
17
|
+
Building Retrieval-Augmented Generation (RAG) systems is harder than it should be. Most solutions lock you into specific vendors, force opinionated architectures, or sacrifice observability for simplicity.
|
|
16
18
|
|
|
17
|
-
**
|
|
19
|
+
**rag-pipeline-utils** takes a different approach: it provides the building blocks—loaders, embedders, retrievers, LLM connectors, and rerankers—that you compose into pipelines that fit your needs. Every component follows clear contracts. Every integration is optional. Every decision is yours.
|
|
20
|
+
|
|
21
|
+
We built this toolkit because we believe infrastructure should be:
|
|
22
|
+
|
|
23
|
+
- **Modular** – swap any component without rewriting your pipeline
|
|
24
|
+
- **Observable** – metrics, tracing, and audit logs built in from day one
|
|
25
|
+
- **Secure** – enterprise-grade defaults, validated inputs, safe JWT handling
|
|
26
|
+
- **Open** – no vendor lock-in, no proprietary APIs, just JavaScript
|
|
27
|
+
|
|
28
|
+
Whether you're prototyping a document Q&A system or scaling a production knowledge base, this toolkit grows with you.
|
|
29
|
+
|
|
30
|
+
---
|
|
18
31
|
|
|
19
32
|
## Quick Start
|
|
20
33
|
|
|
21
|
-
|
|
34
|
+
Here's a complete RAG pipeline in 15 lines:
|
|
22
35
|
|
|
23
36
|
```javascript
|
|
24
37
|
const { createRagPipeline } = require("@devilsdev/rag-pipeline-utils");
|
|
25
38
|
|
|
26
|
-
//
|
|
39
|
+
// Implement your plugins (or use existing ones)
|
|
27
40
|
const pipeline = createRagPipeline({
|
|
28
|
-
loader:
|
|
29
|
-
embedder:
|
|
30
|
-
retriever:
|
|
31
|
-
llm:
|
|
41
|
+
loader: new MyPDFLoader(),
|
|
42
|
+
embedder: new MyOpenAIEmbedder({ apiKey: process.env.OPENAI_API_KEY }),
|
|
43
|
+
retriever: new MyVectorDBRetriever({ url: "http://localhost:8000" }),
|
|
44
|
+
llm: new MyLLMConnector({ model: "gpt-4" }),
|
|
32
45
|
});
|
|
33
46
|
|
|
34
|
-
//
|
|
35
|
-
await pipeline.ingest("./
|
|
36
|
-
const result = await pipeline.query("What is this document about?");
|
|
37
|
-
console.log(result);
|
|
38
|
-
```
|
|
47
|
+
// Ingest documents
|
|
48
|
+
await pipeline.ingest("./company-docs.pdf");
|
|
39
49
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
```json
|
|
45
|
-
{
|
|
46
|
-
"loader": {
|
|
47
|
-
"pdf": "your-pdf-loader-plugin"
|
|
48
|
-
},
|
|
49
|
-
"embedder": {
|
|
50
|
-
"default": "your-embedder-plugin"
|
|
51
|
-
},
|
|
52
|
-
"retriever": {
|
|
53
|
-
"default": "your-vector-db-plugin"
|
|
54
|
-
},
|
|
55
|
-
"llm": {
|
|
56
|
-
"default": "your-llm-plugin"
|
|
57
|
-
}
|
|
58
|
-
}
|
|
50
|
+
// Query with natural language
|
|
51
|
+
const answer = await pipeline.query("What is the vacation policy?");
|
|
52
|
+
console.log(answer.text);
|
|
53
|
+
// Output: "Based on the company handbook, employees receive 20 days of vacation per year..."
|
|
59
54
|
```
|
|
60
55
|
|
|
61
|
-
|
|
56
|
+
**Requirements:** Node.js >= 18.0.0
|
|
62
57
|
|
|
63
|
-
|
|
64
|
-
const {
|
|
65
|
-
loadConfig,
|
|
66
|
-
createRagPipeline,
|
|
67
|
-
} = require("@devilsdev/rag-pipeline-utils");
|
|
58
|
+
Install with:
|
|
68
59
|
|
|
69
|
-
|
|
70
|
-
|
|
60
|
+
```bash
|
|
61
|
+
npm install @devilsdev/rag-pipeline-utils
|
|
71
62
|
```
|
|
72
63
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
### Exported Modules
|
|
64
|
+
---
|
|
76
65
|
|
|
77
|
-
|
|
78
|
-
const {
|
|
79
|
-
// Core pipeline
|
|
80
|
-
createRagPipeline,
|
|
81
|
-
|
|
82
|
-
// Configuration
|
|
83
|
-
loadConfig,
|
|
84
|
-
validateRagrc,
|
|
85
|
-
normalizeConfig,
|
|
86
|
-
|
|
87
|
-
// Plugin system
|
|
88
|
-
pluginRegistry,
|
|
89
|
-
|
|
90
|
-
// DAG workflow engine
|
|
91
|
-
DAGEngine,
|
|
92
|
-
|
|
93
|
-
// Observability
|
|
94
|
-
eventLogger,
|
|
95
|
-
metrics,
|
|
66
|
+
## How It Works
|
|
96
67
|
|
|
97
|
-
|
|
98
|
-
ParallelProcessor,
|
|
68
|
+
**rag-pipeline-utils** orchestrates data through a clear, predictable flow:
|
|
99
69
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
70
|
+
```mermaid
|
|
71
|
+
graph LR
|
|
72
|
+
A[Documents] --> B[Loader]
|
|
73
|
+
B --> C[Embedder]
|
|
74
|
+
C --> D[Vector Store]
|
|
75
|
+
D --> E[Retriever]
|
|
76
|
+
F[User Query] --> C
|
|
77
|
+
F --> E
|
|
78
|
+
E --> G[Reranker]
|
|
79
|
+
G --> H[LLM]
|
|
80
|
+
H --> I[Response]
|
|
103
81
|
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
} = require("@devilsdev/rag-pipeline-utils");
|
|
82
|
+
style B fill:#e1f5e1
|
|
83
|
+
style C fill:#e1f5e1
|
|
84
|
+
style E fill:#e1f5e1
|
|
85
|
+
style G fill:#fff4e1
|
|
86
|
+
style H fill:#e1f5e1
|
|
87
|
+
style I fill:#e3f2fd
|
|
111
88
|
```
|
|
112
89
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
90
|
+
1. **Loader** – Parse documents (PDF, HTML, Markdown, etc.)
|
|
91
|
+
2. **Embedder** – Convert text to vector embeddings
|
|
92
|
+
3. **Vector Store** – Index and persist embeddings
|
|
93
|
+
4. **Retriever** – Find semantically similar content
|
|
94
|
+
5. **Reranker** (optional) – Refine retrieval results
|
|
95
|
+
6. **LLM** – Generate contextual responses
|
|
116
96
|
|
|
117
|
-
|
|
97
|
+
Every component is replaceable. Every step is observable.
|
|
118
98
|
|
|
119
|
-
|
|
120
|
-
- `config.embedder` - Text embedding plugin
|
|
121
|
-
- `config.retriever` - Vector storage and retrieval plugin
|
|
122
|
-
- `config.llm` - Language model plugin
|
|
123
|
-
- `config.reranker` (optional) - Result reranking plugin
|
|
99
|
+
---
|
|
124
100
|
|
|
125
|
-
|
|
101
|
+
## Features
|
|
126
102
|
|
|
127
|
-
###
|
|
103
|
+
### Modular Plugin Architecture
|
|
128
104
|
|
|
129
|
-
|
|
130
|
-
// Load configuration from file
|
|
131
|
-
const config = await loadConfig(".ragrc.json");
|
|
105
|
+
Swap any component without rewriting your pipeline. Use OpenAI embeddings with Pinecone storage, or mix Hugging Face models with PostgreSQL—your choice.
|
|
132
106
|
|
|
133
|
-
|
|
134
|
-
const isValid = validateRagrc(config);
|
|
107
|
+
### Enterprise Observability
|
|
135
108
|
|
|
136
|
-
|
|
137
|
-
const normalized = normalizeConfig(config);
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
### Plugin Registry
|
|
141
|
-
|
|
142
|
-
```javascript
|
|
143
|
-
const { pluginRegistry } = require("@devilsdev/rag-pipeline-utils");
|
|
144
|
-
|
|
145
|
-
// Register a plugin
|
|
146
|
-
pluginRegistry.register("my-plugin", MyPluginClass);
|
|
147
|
-
|
|
148
|
-
// Get a plugin
|
|
149
|
-
const Plugin = pluginRegistry.get("my-plugin");
|
|
150
|
-
|
|
151
|
-
// List all plugins
|
|
152
|
-
const plugins = pluginRegistry.list();
|
|
153
|
-
|
|
154
|
-
// Check if plugin exists
|
|
155
|
-
const exists = pluginRegistry.has("my-plugin");
|
|
156
|
-
```
|
|
109
|
+
Built-in structured logging, Prometheus metrics, OpenTelemetry tracing, and audit logs for compliance-grade transparency.
|
|
157
110
|
|
|
158
111
|
### DAG Workflow Engine
|
|
159
112
|
|
|
160
|
-
|
|
113
|
+
Chain complex multi-step operations with retry logic, parallel execution, and graceful error handling. Perfect for batch ingestion or multi-stage transformations.
|
|
161
114
|
|
|
162
|
-
|
|
163
|
-
const { DAGEngine } = require("@devilsdev/rag-pipeline-utils");
|
|
164
|
-
|
|
165
|
-
const dag = new DAGEngine({
|
|
166
|
-
nodes: [
|
|
167
|
-
{
|
|
168
|
-
id: "load",
|
|
169
|
-
type: "loader",
|
|
170
|
-
config: {
|
|
171
|
-
/* loader config */
|
|
172
|
-
},
|
|
173
|
-
},
|
|
174
|
-
{
|
|
175
|
-
id: "embed",
|
|
176
|
-
type: "embedder",
|
|
177
|
-
dependencies: ["load"],
|
|
178
|
-
config: {
|
|
179
|
-
/* embedder config */
|
|
180
|
-
},
|
|
181
|
-
},
|
|
182
|
-
{
|
|
183
|
-
id: "store",
|
|
184
|
-
type: "retriever",
|
|
185
|
-
dependencies: ["embed"],
|
|
186
|
-
config: {
|
|
187
|
-
/* retriever config */
|
|
188
|
-
},
|
|
189
|
-
},
|
|
190
|
-
],
|
|
191
|
-
});
|
|
192
|
-
|
|
193
|
-
await dag.execute();
|
|
194
|
-
```
|
|
195
|
-
|
|
196
|
-
### Observability
|
|
197
|
-
|
|
198
|
-
```javascript
|
|
199
|
-
const { eventLogger, metrics } = require("@devilsdev/rag-pipeline-utils");
|
|
200
|
-
|
|
201
|
-
// Configure event logging
|
|
202
|
-
eventLogger.configure({
|
|
203
|
-
level: "info",
|
|
204
|
-
format: "json",
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
// Log events
|
|
208
|
-
eventLogger.log("info", "Pipeline started", { pipelineId: "abc123" });
|
|
209
|
-
|
|
210
|
-
// Access metrics
|
|
211
|
-
const pipelineMetrics = metrics.getMetrics();
|
|
212
|
-
console.log(pipelineMetrics);
|
|
213
|
-
```
|
|
214
|
-
|
|
215
|
-
### Parallel Processing
|
|
216
|
-
|
|
217
|
-
```javascript
|
|
218
|
-
const { ParallelProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
219
|
-
|
|
220
|
-
const processor = new ParallelProcessor({
|
|
221
|
-
maxConcurrency: 5,
|
|
222
|
-
});
|
|
223
|
-
|
|
224
|
-
await processor.process(items, async (item) => {
|
|
225
|
-
// Process each item
|
|
226
|
-
return processItem(item);
|
|
227
|
-
});
|
|
228
|
-
```
|
|
229
|
-
|
|
230
|
-
## Plugin Development
|
|
231
|
-
|
|
232
|
-
This package provides a plugin-based architecture. You need to implement plugins for your specific use case.
|
|
233
|
-
|
|
234
|
-
### Plugin Contracts
|
|
235
|
-
|
|
236
|
-
#### Loader Plugin
|
|
237
|
-
|
|
238
|
-
```javascript
|
|
239
|
-
class MyLoader {
|
|
240
|
-
async load(filePath) {
|
|
241
|
-
// Load and parse document
|
|
242
|
-
// Return array of document chunks
|
|
243
|
-
return [
|
|
244
|
-
{
|
|
245
|
-
content: "Document text...",
|
|
246
|
-
metadata: { page: 1, source: filePath },
|
|
247
|
-
},
|
|
248
|
-
];
|
|
249
|
-
}
|
|
250
|
-
}
|
|
251
|
-
```
|
|
252
|
-
|
|
253
|
-
#### Embedder Plugin
|
|
254
|
-
|
|
255
|
-
```javascript
|
|
256
|
-
class MyEmbedder {
|
|
257
|
-
async embed(texts) {
|
|
258
|
-
// Generate embeddings for array of texts
|
|
259
|
-
// Return array of vectors
|
|
260
|
-
return [
|
|
261
|
-
[0.1, 0.2, 0.3, ...], // vector for text[0]
|
|
262
|
-
[0.4, 0.5, 0.6, ...] // vector for text[1]
|
|
263
|
-
];
|
|
264
|
-
}
|
|
265
|
-
|
|
266
|
-
async embedQuery(query) {
|
|
267
|
-
// Generate embedding for single query
|
|
268
|
-
return [0.7, 0.8, 0.9, ...];
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
```
|
|
272
|
-
|
|
273
|
-
#### Retriever Plugin
|
|
274
|
-
|
|
275
|
-
```javascript
|
|
276
|
-
class MyRetriever {
|
|
277
|
-
async store(vectors, metadata) {
|
|
278
|
-
// Store vectors with metadata in your vector database
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
async retrieve(queryVector, options = {}) {
|
|
282
|
-
const { topK = 5, threshold = 0.7 } = options;
|
|
283
|
-
|
|
284
|
-
// Search for similar vectors
|
|
285
|
-
// Return documents with similarity scores
|
|
286
|
-
return [
|
|
287
|
-
{
|
|
288
|
-
content: "Document text...",
|
|
289
|
-
metadata: {
|
|
290
|
-
/* document metadata */
|
|
291
|
-
},
|
|
292
|
-
score: 0.95,
|
|
293
|
-
},
|
|
294
|
-
];
|
|
295
|
-
}
|
|
296
|
-
}
|
|
297
|
-
```
|
|
115
|
+
### Type-Safe Contracts
|
|
298
116
|
|
|
299
|
-
|
|
117
|
+
Complete TypeScript interfaces for all plugin types. Full IDE autocomplete and compile-time validation.
|
|
300
118
|
|
|
301
|
-
|
|
302
|
-
class MyLLM {
|
|
303
|
-
async generate(prompt, options = {}) {
|
|
304
|
-
// Generate text from prompt
|
|
305
|
-
return {
|
|
306
|
-
text: "Generated response...",
|
|
307
|
-
usage: { tokens: 150 },
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
async *stream(prompt, options = {}) {
|
|
312
|
-
// Stream response chunks
|
|
313
|
-
yield "Generated ";
|
|
314
|
-
yield "response ";
|
|
315
|
-
yield "chunks...";
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
```
|
|
119
|
+
### Production-Ready Security
|
|
319
120
|
|
|
320
|
-
|
|
121
|
+
Secure defaults everywhere: validated JSON parsing, JWT algorithm whitelisting, automatic secret redaction in logs, and dependency scanning.
|
|
321
122
|
|
|
322
|
-
|
|
323
|
-
class MyReranker {
|
|
324
|
-
async rerank(query, documents, options = {}) {
|
|
325
|
-
// Rerank documents based on relevance to query
|
|
326
|
-
return documents
|
|
327
|
-
.map((doc) => ({
|
|
328
|
-
...doc,
|
|
329
|
-
score: calculateRelevanceScore(query, doc),
|
|
330
|
-
}))
|
|
331
|
-
.sort((a, b) => b.score - a.score);
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
```
|
|
123
|
+
### Streaming & Batching
|
|
335
124
|
|
|
336
|
-
|
|
125
|
+
Stream LLM responses in real-time. Batch embed thousands of documents efficiently with parallel processing and backpressure control.
|
|
337
126
|
|
|
338
|
-
|
|
339
|
-
# Example environment variables for your plugins
|
|
340
|
-
OPENAI_API_KEY=your_api_key
|
|
341
|
-
PINECONE_API_KEY=your_api_key
|
|
342
|
-
PINECONE_ENVIRONMENT=us-east-1
|
|
343
|
-
CHROMA_URL=http://localhost:8000
|
|
344
|
-
|
|
345
|
-
# Performance settings
|
|
346
|
-
RAG_MAX_CONCURRENCY=5
|
|
347
|
-
RAG_BATCH_SIZE=10
|
|
348
|
-
```
|
|
127
|
+
### Framework Agnostic
|
|
349
128
|
|
|
350
|
-
|
|
129
|
+
Works with Express, Fastify, Next.js, or any Node.js runtime. Deploy to AWS Lambda, Vercel, Docker, or bare metal.
|
|
351
130
|
|
|
352
|
-
|
|
131
|
+
---
|
|
353
132
|
|
|
354
|
-
|
|
355
|
-
const { MultiModalProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
133
|
+
## Join the Community
|
|
356
134
|
|
|
357
|
-
|
|
358
|
-
// Configuration for processing text, images, audio, etc.
|
|
359
|
-
});
|
|
135
|
+
We're building this together.
|
|
360
136
|
|
|
361
|
-
|
|
362
|
-
```
|
|
137
|
+
Whether you're implementing a new retriever, writing documentation, reporting bugs, or sharing ideas—every contribution makes this toolkit better for everyone.
|
|
363
138
|
|
|
364
|
-
|
|
139
|
+
**Getting Started:**
|
|
365
140
|
|
|
366
|
-
|
|
367
|
-
|
|
141
|
+
- 💬 [GitHub Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions) – Ask questions, share use cases, propose features
|
|
142
|
+
- 🐛 [Issue Tracker](https://github.com/DevilsDev/rag-pipeline-utils/issues) – Report bugs or request features
|
|
143
|
+
- 🤝 [Contributing Guide](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/docs/CONTRIBUTING.md) – Learn how to submit pull requests
|
|
144
|
+
- 📖 [Code of Conduct](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/CODE_OF_CONDUCT.md) – Our community standards
|
|
368
145
|
|
|
369
|
-
|
|
370
|
-
// Configuration for adaptive retrieval strategies
|
|
371
|
-
});
|
|
146
|
+
**Good First Issues:** Look for issues tagged `good first issue` to find beginner-friendly contributions.
|
|
372
147
|
|
|
373
|
-
|
|
374
|
-
```
|
|
148
|
+
**Office Hours:** Join our monthly community calls to discuss roadmap priorities and architectural decisions.
|
|
375
149
|
|
|
376
|
-
|
|
150
|
+
---
|
|
377
151
|
|
|
378
|
-
|
|
379
|
-
const { AuditLogger } = require("@devilsdev/rag-pipeline-utils");
|
|
152
|
+
## Security and Quality
|
|
380
153
|
|
|
381
|
-
|
|
382
|
-
destination: "./audit-logs",
|
|
383
|
-
});
|
|
154
|
+
Security isn't optional. This toolkit is built with defense in depth:
|
|
384
155
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
156
|
+
- **Validated Inputs** – All JSON parsed through schema validation (AJV)
|
|
157
|
+
- **Secure Logging** – Automatic redaction of 36 sensitive field patterns (API keys, JWTs, tokens)
|
|
158
|
+
- **JWT Best Practices** – Algorithm whitelisting, issuer/audience validation, expiration enforcement
|
|
159
|
+
- **Dependency Scanning** – Weekly Dependabot updates, CI blocks on critical vulnerabilities
|
|
160
|
+
- **Audit Trail** – Immutable compliance-grade logs for all security-sensitive operations
|
|
161
|
+
- **Plugin Sandboxing** – Third-party plugins run with declared permissions and resource limits
|
|
162
|
+
- **Supply Chain Security** – All GitHub Actions pinned to commit SHAs, SBOM generation available
|
|
391
163
|
|
|
392
|
-
|
|
164
|
+
**Zero Production Vulnerabilities:** `npm audit --production` returns clean on every release.
|
|
393
165
|
|
|
394
|
-
|
|
395
|
-
const { DataGovernance } = require("@devilsdev/rag-pipeline-utils");
|
|
166
|
+
**Test Coverage:** 845+ tests across 61 suites, with 100% coverage on critical paths.
|
|
396
167
|
|
|
397
|
-
|
|
398
|
-
policies: {
|
|
399
|
-
retention: "90d",
|
|
400
|
-
encryption: true,
|
|
401
|
-
},
|
|
402
|
-
});
|
|
168
|
+
**Supported Node Versions:** 18.x, 20.x, 22.x (tested in CI)
|
|
403
169
|
|
|
404
|
-
|
|
405
|
-
```
|
|
170
|
+
---
|
|
406
171
|
|
|
407
|
-
##
|
|
172
|
+
## Installation & Development
|
|
408
173
|
|
|
409
|
-
|
|
174
|
+
### Install
|
|
410
175
|
|
|
411
176
|
```bash
|
|
412
|
-
|
|
413
|
-
npx rag-pipeline init
|
|
414
|
-
|
|
415
|
-
# Run pipeline operations (requires plugin implementation)
|
|
416
|
-
npx rag-pipeline ingest ./documents
|
|
417
|
-
npx rag-pipeline query "Your question"
|
|
418
|
-
|
|
419
|
-
# Run system diagnostics
|
|
420
|
-
npx rag-pipeline doctor
|
|
421
|
-
```
|
|
422
|
-
|
|
423
|
-
## Error Handling
|
|
424
|
-
|
|
425
|
-
```javascript
|
|
426
|
-
try {
|
|
427
|
-
const result = await pipeline.query("What is AI?");
|
|
428
|
-
} catch (error) {
|
|
429
|
-
if (error.code === "PLUGIN_NOT_FOUND") {
|
|
430
|
-
console.error("Required plugin not found:", error.message);
|
|
431
|
-
} else if (error.code === "INVALID_CONFIG") {
|
|
432
|
-
console.error("Invalid configuration:", error.message);
|
|
433
|
-
} else {
|
|
434
|
-
console.error("Pipeline error:", error.message);
|
|
435
|
-
}
|
|
436
|
-
}
|
|
177
|
+
npm install @devilsdev/rag-pipeline-utils
|
|
437
178
|
```
|
|
438
179
|
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
### Document Q&A System
|
|
442
|
-
|
|
443
|
-
```javascript
|
|
444
|
-
// Implement your plugins
|
|
445
|
-
const loader = new PDFLoaderPlugin();
|
|
446
|
-
const embedder = new OpenAIEmbedderPlugin({
|
|
447
|
-
apiKey: process.env.OPENAI_API_KEY,
|
|
448
|
-
});
|
|
449
|
-
const retriever = new ChromaRetrieverPlugin({ url: "http://localhost:8000" });
|
|
450
|
-
const llm = new OpenAILLMPlugin({ model: "gpt-4" });
|
|
180
|
+
### Development Setup
|
|
451
181
|
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
retriever,
|
|
457
|
-
llm,
|
|
458
|
-
});
|
|
182
|
+
```bash
|
|
183
|
+
# Clone the repository
|
|
184
|
+
git clone https://github.com/DevilsDev/rag-pipeline-utils.git
|
|
185
|
+
cd rag-pipeline-utils
|
|
459
186
|
|
|
460
|
-
|
|
461
|
-
|
|
187
|
+
# Install dependencies
|
|
188
|
+
npm install
|
|
462
189
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
console.log(answer);
|
|
466
|
-
```
|
|
190
|
+
# Run tests
|
|
191
|
+
npm test
|
|
467
192
|
|
|
468
|
-
|
|
193
|
+
# Run linter
|
|
194
|
+
npm run lint
|
|
469
195
|
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
class PostgresRetriever {
|
|
473
|
-
constructor(connectionString) {
|
|
474
|
-
this.db = new PostgresClient(connectionString);
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
async retrieve(queryVector, options) {
|
|
478
|
-
const results = await this.db.query(
|
|
479
|
-
"SELECT content, metadata, vector <-> $1 as score FROM embeddings ORDER BY score LIMIT $2",
|
|
480
|
-
[queryVector, options.topK],
|
|
481
|
-
);
|
|
482
|
-
return results.rows;
|
|
483
|
-
}
|
|
484
|
-
}
|
|
196
|
+
# Build distribution
|
|
197
|
+
npm run build
|
|
485
198
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
embedder: yourEmbedder,
|
|
489
|
-
retriever: new PostgresRetriever("postgresql://localhost/kb"),
|
|
490
|
-
llm: yourLLM,
|
|
491
|
-
});
|
|
199
|
+
# Run development tools
|
|
200
|
+
npm run dev
|
|
492
201
|
```
|
|
493
202
|
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
### Batch Processing
|
|
497
|
-
|
|
498
|
-
```javascript
|
|
499
|
-
const { ParallelProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
203
|
+
### Common Commands
|
|
500
204
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
});
|
|
205
|
+
```bash
|
|
206
|
+
npm test # Run test suite
|
|
207
|
+
npm run test:watch # Watch mode for TDD
|
|
208
|
+
npm run lint # Check code style
|
|
209
|
+
npm run lint:fix # Auto-fix linting issues
|
|
210
|
+
npm run security:audit # Scan dependencies
|
|
211
|
+
npm run docs:api # Generate API documentation
|
|
212
|
+
npm run benchmark # Performance benchmarks
|
|
510
213
|
```
|
|
511
214
|
|
|
512
|
-
|
|
215
|
+
---
|
|
513
216
|
|
|
514
|
-
|
|
515
|
-
const { logger, metrics } = require("@devilsdev/rag-pipeline-utils");
|
|
516
|
-
|
|
517
|
-
// Set log level
|
|
518
|
-
logger.setLevel("debug");
|
|
519
|
-
|
|
520
|
-
// Monitor performance
|
|
521
|
-
const stats = metrics.getMetrics();
|
|
522
|
-
console.log("Total queries:", stats.queries?.total || 0);
|
|
523
|
-
console.log("Average latency:", stats.queries?.avgLatency || 0);
|
|
524
|
-
```
|
|
217
|
+
## Roadmap
|
|
525
218
|
|
|
526
|
-
|
|
219
|
+
### v2.3.0 (Q2 2025)
|
|
527
220
|
|
|
528
|
-
|
|
221
|
+
- **Enhanced CLI** – Interactive wizard for pipeline setup, validation commands, and migration tools
|
|
222
|
+
- **Plugin Marketplace** – Curated registry of community-contributed plugins with ratings and verification
|
|
223
|
+
- **Streaming Embeddings** – Real-time embedding generation with backpressure control
|
|
224
|
+
- **Multi-Tenant Support** – Workspace isolation, resource quotas, and tenant-specific configurations
|
|
529
225
|
|
|
530
|
-
|
|
226
|
+
### v2.4.0 (Q3 2025)
|
|
531
227
|
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
this.model = model;
|
|
537
|
-
}
|
|
538
|
-
|
|
539
|
-
async embed(texts) {
|
|
540
|
-
const response = await this.client.embeddings.create({
|
|
541
|
-
model: this.model,
|
|
542
|
-
input: texts,
|
|
543
|
-
});
|
|
544
|
-
return response.data.map((d) => d.embedding);
|
|
545
|
-
}
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
class OpenAILLM {
|
|
549
|
-
constructor({ apiKey, model = "gpt-4" }) {
|
|
550
|
-
this.client = new OpenAI({ apiKey });
|
|
551
|
-
this.model = model;
|
|
552
|
-
}
|
|
553
|
-
|
|
554
|
-
async generate(prompt) {
|
|
555
|
-
const response = await this.client.chat.completions.create({
|
|
556
|
-
model: this.model,
|
|
557
|
-
messages: [{ role: "user", content: prompt }],
|
|
558
|
-
});
|
|
559
|
-
return { text: response.choices[0].message.content };
|
|
560
|
-
}
|
|
561
|
-
}
|
|
562
|
-
```
|
|
228
|
+
- **GraphRAG Support** – Native knowledge graph integration for entity-relationship retrieval
|
|
229
|
+
- **Federated Learning** – Privacy-preserving model training across distributed datasets
|
|
230
|
+
- **Advanced Reranking** – Cross-encoder models and learned ranking functions
|
|
231
|
+
- **Performance Dashboard** – Web UI for monitoring pipeline health and cost metrics
|
|
563
232
|
|
|
564
|
-
###
|
|
233
|
+
### Beyond
|
|
565
234
|
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
this.client = new ChromaClient({ path: url });
|
|
570
|
-
this.collectionName = collection;
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
async store(vectors, metadata) {
|
|
574
|
-
const collection = await this.client.getOrCreateCollection({
|
|
575
|
-
name: this.collectionName,
|
|
576
|
-
});
|
|
577
|
-
|
|
578
|
-
await collection.add({
|
|
579
|
-
embeddings: vectors,
|
|
580
|
-
metadatas: metadata,
|
|
581
|
-
ids: metadata.map((_, i) => `doc_${i}`),
|
|
582
|
-
});
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
async retrieve(queryVector, options) {
|
|
586
|
-
const collection = await this.client.getCollection({
|
|
587
|
-
name: this.collectionName,
|
|
588
|
-
});
|
|
589
|
-
|
|
590
|
-
const results = await collection.query({
|
|
591
|
-
queryEmbeddings: [queryVector],
|
|
592
|
-
nResults: options.topK || 5,
|
|
593
|
-
});
|
|
594
|
-
|
|
595
|
-
return results.metadatas[0].map((meta, i) => ({
|
|
596
|
-
content: results.documents[0][i],
|
|
597
|
-
metadata: meta,
|
|
598
|
-
score: 1 - results.distances[0][i],
|
|
599
|
-
}));
|
|
600
|
-
}
|
|
601
|
-
}
|
|
602
|
-
```
|
|
235
|
+
- Native Rust bindings for performance-critical paths
|
|
236
|
+
- Kubernetes operator for production deployments
|
|
237
|
+
- Real-time collaboration features for team-based pipelines
|
|
603
238
|
|
|
604
|
-
|
|
239
|
+
**Vote on features:** Share your priorities in [GitHub Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions/categories/roadmap).
|
|
605
240
|
|
|
606
|
-
|
|
241
|
+
---
|
|
607
242
|
|
|
608
|
-
|
|
243
|
+
## Contributing
|
|
609
244
|
|
|
610
|
-
|
|
611
|
-
npm install @devilsdev/rag-pipeline-utils
|
|
612
|
-
```
|
|
245
|
+
We welcome contributions of all kinds:
|
|
613
246
|
|
|
614
|
-
**
|
|
247
|
+
- **Code** – Fix bugs, add features, improve performance
|
|
248
|
+
- **Documentation** – Write guides, improve examples, clarify concepts
|
|
249
|
+
- **Testing** – Expand test coverage, add integration tests, report edge cases
|
|
250
|
+
- **Design** – Propose API improvements, suggest architectural patterns
|
|
251
|
+
- **Community** – Answer questions, review pull requests, mentor new contributors
|
|
615
252
|
|
|
616
|
-
|
|
617
|
-
// Make sure to register your plugins
|
|
618
|
-
pluginRegistry.register("my-embedder", MyEmbedderClass);
|
|
619
|
-
```
|
|
253
|
+
**Before You Start:**
|
|
620
254
|
|
|
621
|
-
|
|
255
|
+
1. Read the [Contributing Guide](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/docs/CONTRIBUTING.md)
|
|
256
|
+
2. Check existing [Issues](https://github.com/DevilsDev/rag-pipeline-utils/issues) and [Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions)
|
|
257
|
+
3. Open an issue to discuss major changes before coding
|
|
258
|
+
4. Follow our [Code of Conduct](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/CODE_OF_CONDUCT.md)
|
|
622
259
|
|
|
623
|
-
|
|
624
|
-
// Validate your configuration
|
|
625
|
-
const { validateRagrc } = require("@devilsdev/rag-pipeline-utils");
|
|
626
|
-
const isValid = validateRagrc(config);
|
|
627
|
-
if (!isValid) {
|
|
628
|
-
console.error("Configuration is invalid");
|
|
629
|
-
}
|
|
630
|
-
```
|
|
260
|
+
**Pull Request Checklist:**
|
|
631
261
|
|
|
632
|
-
|
|
262
|
+
- [ ] Tests pass (`npm test`)
|
|
263
|
+
- [ ] Linter passes (`npm run lint`)
|
|
264
|
+
- [ ] Documentation updated
|
|
265
|
+
- [ ] Changelog entry added
|
|
266
|
+
- [ ] Type definitions updated (if applicable)
|
|
633
267
|
|
|
634
|
-
|
|
635
|
-
const { logger } = require("@devilsdev/rag-pipeline-utils");
|
|
268
|
+
---
|
|
636
269
|
|
|
637
|
-
|
|
638
|
-
logger.setLevel("debug");
|
|
639
|
-
```
|
|
270
|
+
## License
|
|
640
271
|
|
|
641
|
-
|
|
272
|
+
**GPL-3.0** – See [LICENSE](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE) for full terms.
|
|
642
273
|
|
|
643
|
-
|
|
644
|
-
- [Issue Tracker](https://github.com/DevilsDev/rag-pipeline-utils/issues)
|
|
645
|
-
- [Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions)
|
|
274
|
+
This is free software: you are free to change and redistribute it under the terms of the GNU General Public License version 3.
|
|
646
275
|
|
|
647
|
-
|
|
276
|
+
---
|
|
648
277
|
|
|
649
|
-
|
|
278
|
+
## What's Next?
|
|
650
279
|
|
|
651
|
-
|
|
280
|
+
⭐ **Star this repository** if you find it useful—it helps others discover the project.
|
|
652
281
|
|
|
653
|
-
|
|
282
|
+
📢 **Share your use case** in [Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions/categories/show-and-tell) to inspire the community.
|
|
654
283
|
|
|
655
|
-
|
|
284
|
+
🤝 **Contribute** your first pull request—check out issues tagged [good first issue](https://github.com/DevilsDev/rag-pipeline-utils/labels/good%20first%20issue).
|
|
656
285
|
|
|
657
|
-
|
|
658
|
-
- Discussions: https://github.com/DevilsDev/rag-pipeline-utils/discussions
|
|
286
|
+
Happy building. 🚀
|