@devilsdev/rag-pipeline-utils 2.1.9 → 2.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +660 -395
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,395 +1,660 @@
|
|
|
1
|
-
# @
|
|
2
|
-
|
|
3
|
-
[](https://github.com/DevilsDev/rag-pipeline-utils/actions)
|
|
4
|
-
[](https://badge.fury.io/js/%40devilsdev%2Frag-pipeline-utils)
|
|
5
|
-
[](https://github.com/DevilsDev/rag-pipeline-utils/actions)
|
|
4
|
+
[](https://badge.fury.io/js/%40devilsdev%2Frag-pipeline-utils)
|
|
5
|
+
[](https://nodejs.org/)
|
|
6
|
+
[](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE)
|
|
7
|
+
[](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils)
|
|
8
|
+
|
|
9
|
+
Enterprise-grade toolkit for building Retrieval-Augmented Generation (RAG) pipelines in Node.js with modular plugin architecture, streaming support, and observability.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @devilsdev/rag-pipeline-utils
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
**Requirements:** Node.js >= 18.0.0
|
|
18
|
+
|
|
19
|
+
## Quick Start
|
|
20
|
+
|
|
21
|
+
### Basic Pipeline
|
|
22
|
+
|
|
23
|
+
```javascript
|
|
24
|
+
const { createRagPipeline } = require("@devilsdev/rag-pipeline-utils");
|
|
25
|
+
|
|
26
|
+
// Initialize pipeline with plugins
|
|
27
|
+
const pipeline = createRagPipeline({
|
|
28
|
+
loader: new PDFLoader(),
|
|
29
|
+
embedder: new OpenAIEmbedder({
|
|
30
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
31
|
+
model: "text-embedding-ada-002",
|
|
32
|
+
}),
|
|
33
|
+
retriever: new ChromaRetriever({
|
|
34
|
+
url: "http://localhost:8000",
|
|
35
|
+
collection: "documents",
|
|
36
|
+
}),
|
|
37
|
+
llm: new OpenAILLM({
|
|
38
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
39
|
+
model: "gpt-4",
|
|
40
|
+
}),
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
// Ingest documents
|
|
44
|
+
await pipeline.ingest("./documents/whitepaper.pdf");
|
|
45
|
+
|
|
46
|
+
// Query
|
|
47
|
+
const result = await pipeline.query("What is the main conclusion?");
|
|
48
|
+
console.log(result.answer);
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Streaming Responses
|
|
52
|
+
|
|
53
|
+
```javascript
|
|
54
|
+
// Stream tokens in real-time
|
|
55
|
+
const stream = await pipeline.query("Explain the methodology", {
|
|
56
|
+
stream: true,
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
for await (const chunk of stream) {
|
|
60
|
+
process.stdout.write(chunk);
|
|
61
|
+
}
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Batch Processing
|
|
65
|
+
|
|
66
|
+
```javascript
|
|
67
|
+
// Process multiple documents concurrently
|
|
68
|
+
const results = await pipeline.ingestBatch(
|
|
69
|
+
["./docs/paper1.pdf", "./docs/paper2.pdf", "./docs/paper3.pdf"],
|
|
70
|
+
{
|
|
71
|
+
parallel: true,
|
|
72
|
+
maxConcurrency: 5,
|
|
73
|
+
},
|
|
74
|
+
);
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Core API
|
|
78
|
+
|
|
79
|
+
### createRagPipeline(config)
|
|
80
|
+
|
|
81
|
+
Creates a pipeline instance with the specified plugins.
|
|
82
|
+
|
|
83
|
+
**Parameters:**
|
|
84
|
+
|
|
85
|
+
- `config.loader` - Document loader plugin
|
|
86
|
+
- `config.embedder` - Text embedding plugin
|
|
87
|
+
- `config.retriever` - Vector storage and retrieval plugin
|
|
88
|
+
- `config.llm` - Language model plugin
|
|
89
|
+
- `config.reranker` (optional) - Result reranking plugin
|
|
90
|
+
|
|
91
|
+
**Returns:** Pipeline instance
|
|
92
|
+
|
|
93
|
+
### Pipeline Methods
|
|
94
|
+
|
|
95
|
+
#### `pipeline.ingest(filePath, options)`
|
|
96
|
+
|
|
97
|
+
Ingest a document into the pipeline.
|
|
98
|
+
|
|
99
|
+
```javascript
|
|
100
|
+
await pipeline.ingest("./document.pdf", {
|
|
101
|
+
chunkSize: 1000,
|
|
102
|
+
chunkOverlap: 200,
|
|
103
|
+
metadata: { source: "research", year: 2024 },
|
|
104
|
+
});
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
#### `pipeline.query(question, options)`
|
|
108
|
+
|
|
109
|
+
Query the pipeline.
|
|
110
|
+
|
|
111
|
+
```javascript
|
|
112
|
+
const result = await pipeline.query("What is RAG?", {
|
|
113
|
+
topK: 5,
|
|
114
|
+
similarityThreshold: 0.7,
|
|
115
|
+
stream: false,
|
|
116
|
+
});
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
**Result structure:**
|
|
120
|
+
|
|
121
|
+
```javascript
|
|
122
|
+
{
|
|
123
|
+
answer: string,
|
|
124
|
+
sources: Array<{
|
|
125
|
+
content: string,
|
|
126
|
+
metadata: object,
|
|
127
|
+
score: number
|
|
128
|
+
}>,
|
|
129
|
+
metrics: {
|
|
130
|
+
retrievalTime: number,
|
|
131
|
+
generationTime: number,
|
|
132
|
+
totalTime: number
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
#### `pipeline.ingestBatch(files, options)`
|
|
138
|
+
|
|
139
|
+
Batch ingest multiple documents.
|
|
140
|
+
|
|
141
|
+
```javascript
|
|
142
|
+
await pipeline.ingestBatch(["file1.pdf", "file2.pdf"], {
|
|
143
|
+
parallel: true,
|
|
144
|
+
maxConcurrency: 3,
|
|
145
|
+
});
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Plugin System
|
|
149
|
+
|
|
150
|
+
### Plugin Contracts
|
|
151
|
+
|
|
152
|
+
All plugins implement standardized interfaces for interoperability.
|
|
153
|
+
|
|
154
|
+
#### Loader Plugin
|
|
155
|
+
|
|
156
|
+
```javascript
|
|
157
|
+
class CustomLoader {
|
|
158
|
+
async load(filePath) {
|
|
159
|
+
// Return array of document chunks
|
|
160
|
+
return [{ content: string, metadata: object }];
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
#### Embedder Plugin
|
|
166
|
+
|
|
167
|
+
```javascript
|
|
168
|
+
class CustomEmbedder {
|
|
169
|
+
async embed(texts) {
|
|
170
|
+
// Return array of vectors
|
|
171
|
+
return [[0.1, 0.2, ...], [0.3, 0.4, ...]];
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async embedQuery(query) {
|
|
175
|
+
// Return single vector
|
|
176
|
+
return [0.5, 0.6, ...];
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
getDimensions() {
|
|
180
|
+
return 1536; // Vector dimensions
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
#### Retriever Plugin
|
|
186
|
+
|
|
187
|
+
```javascript
|
|
188
|
+
class CustomRetriever {
|
|
189
|
+
async store(vectors, metadata) {
|
|
190
|
+
// Store vectors with metadata
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async retrieve(queryVector, options = {}) {
|
|
194
|
+
const { topK = 5, threshold = 0.7 } = options;
|
|
195
|
+
// Return similar documents
|
|
196
|
+
return [{ content: string, metadata: object, score: number }];
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
#### LLM Plugin
|
|
202
|
+
|
|
203
|
+
```javascript
|
|
204
|
+
class CustomLLM {
|
|
205
|
+
async generate(prompt, options = {}) {
|
|
206
|
+
// Return generated text
|
|
207
|
+
return { text: string, usage: object };
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async stream(prompt, options = {}) {
|
|
211
|
+
// Return async iterable
|
|
212
|
+
async function* generate() {
|
|
213
|
+
yield chunk1;
|
|
214
|
+
yield chunk2;
|
|
215
|
+
}
|
|
216
|
+
return generate();
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
#### Reranker Plugin (Optional)
|
|
222
|
+
|
|
223
|
+
```javascript
|
|
224
|
+
class CustomReranker {
|
|
225
|
+
async rerank(query, documents, options = {}) {
|
|
226
|
+
// Return reranked documents
|
|
227
|
+
return documents.map((doc) => ({
|
|
228
|
+
...doc,
|
|
229
|
+
score: newScore,
|
|
230
|
+
}));
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
## Configuration
|
|
236
|
+
|
|
237
|
+
### Environment Variables
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
# OpenAI
|
|
241
|
+
OPENAI_API_KEY=sk-...
|
|
242
|
+
OPENAI_MODEL=gpt-4
|
|
243
|
+
|
|
244
|
+
# Pinecone
|
|
245
|
+
PINECONE_API_KEY=...
|
|
246
|
+
PINECONE_ENVIRONMENT=us-east-1
|
|
247
|
+
|
|
248
|
+
# ChromaDB
|
|
249
|
+
CHROMA_URL=http://localhost:8000
|
|
250
|
+
|
|
251
|
+
# Performance
|
|
252
|
+
RAG_MAX_CONCURRENCY=5
|
|
253
|
+
RAG_BATCH_SIZE=10
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
### Configuration File
|
|
257
|
+
|
|
258
|
+
Create `.ragrc.json`:
|
|
259
|
+
|
|
260
|
+
```json
|
|
261
|
+
{
|
|
262
|
+
"loader": {
|
|
263
|
+
"pdf": "@devilsdev/pdf-loader",
|
|
264
|
+
"markdown": "@devilsdev/markdown-loader"
|
|
265
|
+
},
|
|
266
|
+
"embedder": {
|
|
267
|
+
"openai": "@devilsdev/openai-embedder"
|
|
268
|
+
},
|
|
269
|
+
"retriever": {
|
|
270
|
+
"chroma": "@devilsdev/chroma-retriever"
|
|
271
|
+
},
|
|
272
|
+
"llm": {
|
|
273
|
+
"openai": "@devilsdev/openai-llm"
|
|
274
|
+
},
|
|
275
|
+
"pipeline": {
|
|
276
|
+
"loader": "pdf",
|
|
277
|
+
"embedder": "openai",
|
|
278
|
+
"retriever": "chroma",
|
|
279
|
+
"llm": "openai"
|
|
280
|
+
},
|
|
281
|
+
"performance": {
|
|
282
|
+
"maxConcurrency": 5,
|
|
283
|
+
"enableStreaming": true
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Load configuration:
|
|
289
|
+
|
|
290
|
+
```javascript
|
|
291
|
+
const { loadConfig } = require("@devilsdev/rag-pipeline-utils");
|
|
292
|
+
const config = await loadConfig(".ragrc.json");
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
## Advanced Usage
|
|
296
|
+
|
|
297
|
+
### Custom Pipeline Components
|
|
298
|
+
|
|
299
|
+
```javascript
|
|
300
|
+
const {
|
|
301
|
+
createRagPipeline,
|
|
302
|
+
pluginRegistry,
|
|
303
|
+
} = require("@devilsdev/rag-pipeline-utils");
|
|
304
|
+
|
|
305
|
+
// Register custom plugin
|
|
306
|
+
pluginRegistry.register("my-embedder", MyCustomEmbedder);
|
|
307
|
+
|
|
308
|
+
// Use in pipeline
|
|
309
|
+
const pipeline = createRagPipeline({
|
|
310
|
+
loader: new PDFLoader(),
|
|
311
|
+
embedder: pluginRegistry.get("my-embedder"),
|
|
312
|
+
retriever: new ChromaRetriever({ url: "http://localhost:8000" }),
|
|
313
|
+
llm: new OpenAILLM({ model: "gpt-4" }),
|
|
314
|
+
});
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
### DAG Workflows
|
|
318
|
+
|
|
319
|
+
Build complex multi-step pipelines using the DAG engine:
|
|
320
|
+
|
|
321
|
+
```javascript
|
|
322
|
+
const { DAGEngine } = require("@devilsdev/rag-pipeline-utils");
|
|
323
|
+
|
|
324
|
+
const dag = new DAGEngine({
|
|
325
|
+
nodes: [
|
|
326
|
+
{
|
|
327
|
+
id: "load",
|
|
328
|
+
type: "loader",
|
|
329
|
+
config: { source: "./docs" },
|
|
330
|
+
},
|
|
331
|
+
{
|
|
332
|
+
id: "embed",
|
|
333
|
+
type: "embedder",
|
|
334
|
+
dependencies: ["load"],
|
|
335
|
+
config: { model: "text-embedding-ada-002" },
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
id: "store",
|
|
339
|
+
type: "retriever",
|
|
340
|
+
dependencies: ["embed"],
|
|
341
|
+
config: { collection: "documents" },
|
|
342
|
+
},
|
|
343
|
+
],
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
await dag.execute();
|
|
347
|
+
```
|
|
348
|
+
|
|
349
|
+
### Observability
|
|
350
|
+
|
|
351
|
+
Enable monitoring and tracing:
|
|
352
|
+
|
|
353
|
+
```javascript
|
|
354
|
+
const { eventLogger, metrics } = require("@devilsdev/rag-pipeline-utils");
|
|
355
|
+
|
|
356
|
+
// Configure logging
|
|
357
|
+
eventLogger.configure({
|
|
358
|
+
level: "info",
|
|
359
|
+
format: "json",
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
// Access metrics
|
|
363
|
+
const pipelineMetrics = metrics.getMetrics();
|
|
364
|
+
console.log(pipelineMetrics.queries.total);
|
|
365
|
+
console.log(pipelineMetrics.queries.avgLatency);
|
|
366
|
+
```
|
|
367
|
+
|
|
368
|
+
### Error Handling
|
|
369
|
+
|
|
370
|
+
```javascript
|
|
371
|
+
try {
|
|
372
|
+
const result = await pipeline.query("What is AI?");
|
|
373
|
+
} catch (error) {
|
|
374
|
+
if (error.code === "RETRIEVAL_ERROR") {
|
|
375
|
+
console.error("Vector search failed:", error.message);
|
|
376
|
+
} else if (error.code === "LLM_ERROR") {
|
|
377
|
+
console.error("Generation failed:", error.message);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
## CLI Usage
|
|
383
|
+
|
|
384
|
+
The package includes a CLI for common operations:
|
|
385
|
+
|
|
386
|
+
```bash
|
|
387
|
+
# Initialize configuration
|
|
388
|
+
npx rag-pipeline init
|
|
389
|
+
|
|
390
|
+
# Ingest documents
|
|
391
|
+
npx rag-pipeline ingest ./docs --loader pdf
|
|
392
|
+
|
|
393
|
+
# Query interactively
|
|
394
|
+
npx rag-pipeline query "Your question here"
|
|
395
|
+
|
|
396
|
+
# Run health check
|
|
397
|
+
npx rag-pipeline doctor
|
|
398
|
+
```
|
|
399
|
+
|
|
400
|
+
## Use Cases
|
|
401
|
+
|
|
402
|
+
### Document Q&A Systems
|
|
403
|
+
|
|
404
|
+
Build question-answering systems over proprietary documents:
|
|
405
|
+
|
|
406
|
+
```javascript
|
|
407
|
+
const pipeline = createRagPipeline({
|
|
408
|
+
/* config */
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
// Ingest company documentation
|
|
412
|
+
await pipeline.ingestBatch([
|
|
413
|
+
"./docs/handbook.pdf",
|
|
414
|
+
"./docs/policies.pdf",
|
|
415
|
+
"./docs/procedures.pdf",
|
|
416
|
+
]);
|
|
417
|
+
|
|
418
|
+
// Answer questions
|
|
419
|
+
const answer = await pipeline.query("What is the vacation policy?");
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
### Semantic Search
|
|
423
|
+
|
|
424
|
+
Implement semantic search over large document collections:
|
|
425
|
+
|
|
426
|
+
```javascript
|
|
427
|
+
const results = await pipeline.query("machine learning papers", {
|
|
428
|
+
topK: 10,
|
|
429
|
+
similarityThreshold: 0.75,
|
|
430
|
+
returnSources: true,
|
|
431
|
+
});
|
|
432
|
+
|
|
433
|
+
results.sources.forEach((source) => {
|
|
434
|
+
console.log(`${source.metadata.title} (score: ${source.score})`);
|
|
435
|
+
});
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
### Knowledge Base Integration
|
|
439
|
+
|
|
440
|
+
Integrate with existing knowledge bases:
|
|
441
|
+
|
|
442
|
+
```javascript
|
|
443
|
+
// Custom retriever for your database
|
|
444
|
+
class PostgresRetriever {
|
|
445
|
+
constructor(connectionString) {
|
|
446
|
+
this.db = new PostgresClient(connectionString);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
async store(vectors, metadata) {
|
|
450
|
+
await this.db.query(
|
|
451
|
+
"INSERT INTO embeddings (vector, metadata) VALUES ($1, $2)",
|
|
452
|
+
[vectors, metadata],
|
|
453
|
+
);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async retrieve(queryVector, options) {
|
|
457
|
+
const results = await this.db.query(
|
|
458
|
+
"SELECT * FROM embeddings ORDER BY vector <-> $1 LIMIT $2",
|
|
459
|
+
[queryVector, options.topK],
|
|
460
|
+
);
|
|
461
|
+
return results.rows;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
const pipeline = createRagPipeline({
|
|
466
|
+
loader: new PDFLoader(),
|
|
467
|
+
embedder: new OpenAIEmbedder({ apiKey: process.env.OPENAI_API_KEY }),
|
|
468
|
+
retriever: new PostgresRetriever("postgresql://localhost/kb"),
|
|
469
|
+
llm: new OpenAILLM({ model: "gpt-4" }),
|
|
470
|
+
});
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
### Multi-Source Aggregation
|
|
474
|
+
|
|
475
|
+
Combine data from multiple sources:
|
|
476
|
+
|
|
477
|
+
```javascript
|
|
478
|
+
const sources = [
|
|
479
|
+
{ type: "pdf", path: "./research/*.pdf" },
|
|
480
|
+
{ type: "markdown", path: "./docs/*.md" },
|
|
481
|
+
{ type: "api", url: "https://api.example.com/docs" },
|
|
482
|
+
];
|
|
483
|
+
|
|
484
|
+
for (const source of sources) {
|
|
485
|
+
const loader = getLoaderForType(source.type);
|
|
486
|
+
await pipeline.ingest(source.path || source.url, {
|
|
487
|
+
loader,
|
|
488
|
+
metadata: { source: source.type },
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
## Performance Optimization
|
|
494
|
+
|
|
495
|
+
### Chunking Strategies
|
|
496
|
+
|
|
497
|
+
```javascript
|
|
498
|
+
await pipeline.ingest("./large-document.pdf", {
|
|
499
|
+
chunkSize: 1000, // Characters per chunk
|
|
500
|
+
chunkOverlap: 200, // Overlap between chunks
|
|
501
|
+
chunkStrategy: "sentence", // 'sentence' | 'paragraph' | 'fixed'
|
|
502
|
+
});
|
|
503
|
+
```
|
|
504
|
+
|
|
505
|
+
### Concurrent Processing
|
|
506
|
+
|
|
507
|
+
```javascript
|
|
508
|
+
await pipeline.ingestBatch(files, {
|
|
509
|
+
parallel: true,
|
|
510
|
+
maxConcurrency: 5, // Parallel operations
|
|
511
|
+
batchSize: 10, // Embeddings per batch
|
|
512
|
+
});
|
|
513
|
+
```
|
|
514
|
+
|
|
515
|
+
### Caching
|
|
516
|
+
|
|
517
|
+
```javascript
|
|
518
|
+
const pipeline = createRagPipeline({
|
|
519
|
+
loader: new PDFLoader(),
|
|
520
|
+
embedder: new OpenAIEmbedder({
|
|
521
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
522
|
+
cache: true, // Cache embeddings
|
|
523
|
+
cacheTTL: 3600, // Cache lifetime in seconds
|
|
524
|
+
}),
|
|
525
|
+
retriever: new ChromaRetriever({ url: "http://localhost:8000" }),
|
|
526
|
+
llm: new OpenAILLM({ model: "gpt-4" }),
|
|
527
|
+
});
|
|
528
|
+
```
|
|
529
|
+
|
|
530
|
+
## TypeScript Support
|
|
531
|
+
|
|
532
|
+
Full TypeScript definitions included:
|
|
533
|
+
|
|
534
|
+
```typescript
|
|
535
|
+
import {
|
|
536
|
+
createRagPipeline,
|
|
537
|
+
Pipeline,
|
|
538
|
+
LoaderPlugin,
|
|
539
|
+
EmbedderPlugin,
|
|
540
|
+
RetrieverPlugin,
|
|
541
|
+
LLMPlugin,
|
|
542
|
+
} from "@devilsdev/rag-pipeline-utils";
|
|
543
|
+
|
|
544
|
+
const pipeline: Pipeline = createRagPipeline({
|
|
545
|
+
loader: new PDFLoader(),
|
|
546
|
+
embedder: new OpenAIEmbedder({ apiKey: string }),
|
|
547
|
+
retriever: new ChromaRetriever({ url: string }),
|
|
548
|
+
llm: new OpenAILLM({ model: string }),
|
|
549
|
+
});
|
|
550
|
+
|
|
551
|
+
interface QueryResult {
|
|
552
|
+
answer: string;
|
|
553
|
+
sources: Source[];
|
|
554
|
+
metrics: Metrics;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const result: QueryResult = await pipeline.query("question");
|
|
558
|
+
```
|
|
559
|
+
|
|
560
|
+
## API Reference
|
|
561
|
+
|
|
562
|
+
### Exported Modules
|
|
563
|
+
|
|
564
|
+
```javascript
|
|
565
|
+
const {
|
|
566
|
+
// Core
|
|
567
|
+
createRagPipeline,
|
|
568
|
+
loadConfig,
|
|
569
|
+
|
|
570
|
+
// Plugin System
|
|
571
|
+
pluginRegistry,
|
|
572
|
+
|
|
573
|
+
// Workflow Engine
|
|
574
|
+
DAGEngine,
|
|
575
|
+
|
|
576
|
+
// Observability
|
|
577
|
+
eventLogger,
|
|
578
|
+
metrics,
|
|
579
|
+
|
|
580
|
+
// Utilities
|
|
581
|
+
logger,
|
|
582
|
+
} = require("@devilsdev/rag-pipeline-utils");
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
### Plugin Registry API
|
|
586
|
+
|
|
587
|
+
```javascript
|
|
588
|
+
// Register plugin
|
|
589
|
+
pluginRegistry.register("name", PluginClass);
|
|
590
|
+
|
|
591
|
+
// Get plugin
|
|
592
|
+
const Plugin = pluginRegistry.get("name");
|
|
593
|
+
|
|
594
|
+
// List plugins
|
|
595
|
+
const plugins = pluginRegistry.list();
|
|
596
|
+
|
|
597
|
+
// Check if plugin exists
|
|
598
|
+
const exists = pluginRegistry.has("name");
|
|
599
|
+
```
|
|
600
|
+
|
|
601
|
+
## Troubleshooting
|
|
602
|
+
|
|
603
|
+
### Common Issues
|
|
604
|
+
|
|
605
|
+
**Error: Module not found**
|
|
606
|
+
|
|
607
|
+
```bash
|
|
608
|
+
npm install --save @devilsdev/rag-pipeline-utils
|
|
609
|
+
```
|
|
610
|
+
|
|
611
|
+
**Error: Invalid API key**
|
|
612
|
+
|
|
613
|
+
```javascript
|
|
614
|
+
// Ensure environment variables are set
|
|
615
|
+
require("dotenv").config();
|
|
616
|
+
const embedder = new OpenAIEmbedder({
|
|
617
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
618
|
+
});
|
|
619
|
+
```
|
|
620
|
+
|
|
621
|
+
**Error: Connection refused (ChromaDB)**
|
|
622
|
+
|
|
623
|
+
```bash
|
|
624
|
+
# Start ChromaDB server
|
|
625
|
+
docker run -p 8000:8000 chromadb/chroma
|
|
626
|
+
```
|
|
627
|
+
|
|
628
|
+
### Debug Mode
|
|
629
|
+
|
|
630
|
+
Enable verbose logging:
|
|
631
|
+
|
|
632
|
+
```javascript
|
|
633
|
+
const { logger } = require("@devilsdev/rag-pipeline-utils");
|
|
634
|
+
|
|
635
|
+
logger.setLevel("debug");
|
|
636
|
+
```
|
|
637
|
+
|
|
638
|
+
## Examples
|
|
639
|
+
|
|
640
|
+
See the [examples directory](https://github.com/DevilsDev/rag-pipeline-utils/tree/main/examples) for complete working examples:
|
|
641
|
+
|
|
642
|
+
- Basic RAG pipeline
|
|
643
|
+
- Streaming responses
|
|
644
|
+
- Custom plugins
|
|
645
|
+
- DAG workflows
|
|
646
|
+
- Multi-source ingestion
|
|
647
|
+
|
|
648
|
+
## Contributing
|
|
649
|
+
|
|
650
|
+
Contributions welcome. Please read the [contribution guidelines](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/docs/CONTRIBUTING.md) and [plugin developer guide](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/docs/PLUGIN_DEVELOPER_GUIDE.md).
|
|
651
|
+
|
|
652
|
+
## License
|
|
653
|
+
|
|
654
|
+
GPL-3.0 - See [LICENSE](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE) for details.
|
|
655
|
+
|
|
656
|
+
## Support
|
|
657
|
+
|
|
658
|
+
- Documentation: https://devilsdev.github.io/rag-pipeline-utils/
|
|
659
|
+
- Issues: https://github.com/DevilsDev/rag-pipeline-utils/issues
|
|
660
|
+
- Discussions: https://github.com/DevilsDev/rag-pipeline-utils/discussionsDo
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devilsdev/rag-pipeline-utils",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.11",
|
|
4
4
|
"description": "A comprehensive enterprise-grade toolkit for building RAG (Retrieval-Augmented Generation) pipelines with advanced AI/ML capabilities",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"type": "commonjs",
|