@devilsdev/rag-pipeline-utils 2.1.9 → 2.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +658 -395
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -1,395 +1,658 @@
|
|
|
1
|
-
# @
|
|
2
|
-
|
|
3
|
-
[](https://github.com/DevilsDev/rag-pipeline-utils/actions)
|
|
4
|
-
[](https://badge.fury.io/js/%40devilsdev%2Frag-pipeline-utils)
|
|
5
|
-
[](https://github.com/DevilsDev/rag-pipeline-utils/actions)
|
|
4
|
+
[](https://badge.fury.io/js/%40devilsdev%2Frag-pipeline-utils)
|
|
5
|
+
[](https://nodejs.org/)
|
|
6
|
+
[](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE)
|
|
7
|
+
[](https://www.npmjs.com/package/@devilsdev/rag-pipeline-utils)
|
|
8
|
+
|
|
9
|
+
Enterprise-grade toolkit for building Retrieval-Augmented Generation (RAG) pipelines in Node.js with modular plugin architecture, streaming support, and observability.
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @devilsdev/rag-pipeline-utils
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
**Requirements:** Node.js >= 18.0.0
|
|
18
|
+
|
|
19
|
+
## Quick Start
|
|
20
|
+
|
|
21
|
+
### Basic Pipeline Setup
|
|
22
|
+
|
|
23
|
+
```javascript
|
|
24
|
+
const { createRagPipeline } = require("@devilsdev/rag-pipeline-utils");
|
|
25
|
+
|
|
26
|
+
// Create pipeline with your plugin implementations
|
|
27
|
+
const pipeline = createRagPipeline({
|
|
28
|
+
loader: yourLoaderPlugin,
|
|
29
|
+
embedder: yourEmbedderPlugin,
|
|
30
|
+
retriever: yourRetrieverPlugin,
|
|
31
|
+
llm: yourLLMPlugin,
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
// Use the pipeline
|
|
35
|
+
await pipeline.ingest("./document.pdf");
|
|
36
|
+
const result = await pipeline.query("What is this document about?");
|
|
37
|
+
console.log(result);
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Using Configuration File
|
|
41
|
+
|
|
42
|
+
Create `.ragrc.json`:
|
|
43
|
+
|
|
44
|
+
```json
|
|
45
|
+
{
|
|
46
|
+
"loader": {
|
|
47
|
+
"pdf": "your-pdf-loader-plugin"
|
|
48
|
+
},
|
|
49
|
+
"embedder": {
|
|
50
|
+
"default": "your-embedder-plugin"
|
|
51
|
+
},
|
|
52
|
+
"retriever": {
|
|
53
|
+
"default": "your-vector-db-plugin"
|
|
54
|
+
},
|
|
55
|
+
"llm": {
|
|
56
|
+
"default": "your-llm-plugin"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Load and use configuration:
|
|
62
|
+
|
|
63
|
+
```javascript
|
|
64
|
+
const {
|
|
65
|
+
loadConfig,
|
|
66
|
+
createRagPipeline,
|
|
67
|
+
} = require("@devilsdev/rag-pipeline-utils");
|
|
68
|
+
|
|
69
|
+
const config = await loadConfig(".ragrc.json");
|
|
70
|
+
const pipeline = createRagPipeline(config);
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
## Core API
|
|
74
|
+
|
|
75
|
+
### Exported Modules
|
|
76
|
+
|
|
77
|
+
```javascript
|
|
78
|
+
const {
|
|
79
|
+
// Core pipeline
|
|
80
|
+
createRagPipeline,
|
|
81
|
+
|
|
82
|
+
// Configuration
|
|
83
|
+
loadConfig,
|
|
84
|
+
validateRagrc,
|
|
85
|
+
normalizeConfig,
|
|
86
|
+
|
|
87
|
+
// Plugin system
|
|
88
|
+
pluginRegistry,
|
|
89
|
+
|
|
90
|
+
// DAG workflow engine
|
|
91
|
+
DAGEngine,
|
|
92
|
+
|
|
93
|
+
// Observability
|
|
94
|
+
eventLogger,
|
|
95
|
+
metrics,
|
|
96
|
+
|
|
97
|
+
// Performance
|
|
98
|
+
ParallelProcessor,
|
|
99
|
+
|
|
100
|
+
// AI/ML
|
|
101
|
+
MultiModalProcessor,
|
|
102
|
+
AdaptiveRetrievalEngine,
|
|
103
|
+
|
|
104
|
+
// Enterprise
|
|
105
|
+
AuditLogger,
|
|
106
|
+
DataGovernance,
|
|
107
|
+
|
|
108
|
+
// Utilities
|
|
109
|
+
logger,
|
|
110
|
+
} = require("@devilsdev/rag-pipeline-utils");
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
### createRagPipeline(config)
|
|
114
|
+
|
|
115
|
+
Creates a RAG pipeline instance.
|
|
116
|
+
|
|
117
|
+
**Parameters:**
|
|
118
|
+
|
|
119
|
+
- `config.loader` - Document loader plugin
|
|
120
|
+
- `config.embedder` - Text embedding plugin
|
|
121
|
+
- `config.retriever` - Vector storage and retrieval plugin
|
|
122
|
+
- `config.llm` - Language model plugin
|
|
123
|
+
- `config.reranker` (optional) - Result reranking plugin
|
|
124
|
+
|
|
125
|
+
**Returns:** Pipeline instance with `ingest()` and `query()` methods
|
|
126
|
+
|
|
127
|
+
### Configuration Management
|
|
128
|
+
|
|
129
|
+
```javascript
|
|
130
|
+
// Load configuration from file
|
|
131
|
+
const config = await loadConfig(".ragrc.json");
|
|
132
|
+
|
|
133
|
+
// Validate configuration
|
|
134
|
+
const isValid = validateRagrc(config);
|
|
135
|
+
|
|
136
|
+
// Normalize configuration
|
|
137
|
+
const normalized = normalizeConfig(config);
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
### Plugin Registry
|
|
141
|
+
|
|
142
|
+
```javascript
|
|
143
|
+
const { pluginRegistry } = require("@devilsdev/rag-pipeline-utils");
|
|
144
|
+
|
|
145
|
+
// Register a plugin
|
|
146
|
+
pluginRegistry.register("my-plugin", MyPluginClass);
|
|
147
|
+
|
|
148
|
+
// Get a plugin
|
|
149
|
+
const Plugin = pluginRegistry.get("my-plugin");
|
|
150
|
+
|
|
151
|
+
// List all plugins
|
|
152
|
+
const plugins = pluginRegistry.list();
|
|
153
|
+
|
|
154
|
+
// Check if plugin exists
|
|
155
|
+
const exists = pluginRegistry.has("my-plugin");
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
### DAG Workflow Engine
|
|
159
|
+
|
|
160
|
+
Build complex multi-step workflows:
|
|
161
|
+
|
|
162
|
+
```javascript
|
|
163
|
+
const { DAGEngine } = require("@devilsdev/rag-pipeline-utils");
|
|
164
|
+
|
|
165
|
+
const dag = new DAGEngine({
|
|
166
|
+
nodes: [
|
|
167
|
+
{
|
|
168
|
+
id: "load",
|
|
169
|
+
type: "loader",
|
|
170
|
+
config: {
|
|
171
|
+
/* loader config */
|
|
172
|
+
},
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
id: "embed",
|
|
176
|
+
type: "embedder",
|
|
177
|
+
dependencies: ["load"],
|
|
178
|
+
config: {
|
|
179
|
+
/* embedder config */
|
|
180
|
+
},
|
|
181
|
+
},
|
|
182
|
+
{
|
|
183
|
+
id: "store",
|
|
184
|
+
type: "retriever",
|
|
185
|
+
dependencies: ["embed"],
|
|
186
|
+
config: {
|
|
187
|
+
/* retriever config */
|
|
188
|
+
},
|
|
189
|
+
},
|
|
190
|
+
],
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
await dag.execute();
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
### Observability
|
|
197
|
+
|
|
198
|
+
```javascript
|
|
199
|
+
const { eventLogger, metrics } = require("@devilsdev/rag-pipeline-utils");
|
|
200
|
+
|
|
201
|
+
// Configure event logging
|
|
202
|
+
eventLogger.configure({
|
|
203
|
+
level: "info",
|
|
204
|
+
format: "json",
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
// Log events
|
|
208
|
+
eventLogger.log("info", "Pipeline started", { pipelineId: "abc123" });
|
|
209
|
+
|
|
210
|
+
// Access metrics
|
|
211
|
+
const pipelineMetrics = metrics.getMetrics();
|
|
212
|
+
console.log(pipelineMetrics);
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
### Parallel Processing
|
|
216
|
+
|
|
217
|
+
```javascript
|
|
218
|
+
const { ParallelProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
219
|
+
|
|
220
|
+
const processor = new ParallelProcessor({
|
|
221
|
+
maxConcurrency: 5,
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
await processor.process(items, async (item) => {
|
|
225
|
+
// Process each item
|
|
226
|
+
return processItem(item);
|
|
227
|
+
});
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
## Plugin Development
|
|
231
|
+
|
|
232
|
+
This package provides a plugin-based architecture. You need to implement plugins for your specific use case.
|
|
233
|
+
|
|
234
|
+
### Plugin Contracts
|
|
235
|
+
|
|
236
|
+
#### Loader Plugin
|
|
237
|
+
|
|
238
|
+
```javascript
|
|
239
|
+
class MyLoader {
|
|
240
|
+
async load(filePath) {
|
|
241
|
+
// Load and parse document
|
|
242
|
+
// Return array of document chunks
|
|
243
|
+
return [
|
|
244
|
+
{
|
|
245
|
+
content: "Document text...",
|
|
246
|
+
metadata: { page: 1, source: filePath },
|
|
247
|
+
},
|
|
248
|
+
];
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
#### Embedder Plugin
|
|
254
|
+
|
|
255
|
+
```javascript
|
|
256
|
+
class MyEmbedder {
|
|
257
|
+
async embed(texts) {
|
|
258
|
+
// Generate embeddings for array of texts
|
|
259
|
+
// Return array of vectors
|
|
260
|
+
return [
|
|
261
|
+
[0.1, 0.2, 0.3, ...], // vector for text[0]
|
|
262
|
+
[0.4, 0.5, 0.6, ...] // vector for text[1]
|
|
263
|
+
];
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
async embedQuery(query) {
|
|
267
|
+
// Generate embedding for single query
|
|
268
|
+
return [0.7, 0.8, 0.9, ...];
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
#### Retriever Plugin
|
|
274
|
+
|
|
275
|
+
```javascript
|
|
276
|
+
class MyRetriever {
|
|
277
|
+
async store(vectors, metadata) {
|
|
278
|
+
// Store vectors with metadata in your vector database
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
async retrieve(queryVector, options = {}) {
|
|
282
|
+
const { topK = 5, threshold = 0.7 } = options;
|
|
283
|
+
|
|
284
|
+
// Search for similar vectors
|
|
285
|
+
// Return documents with similarity scores
|
|
286
|
+
return [
|
|
287
|
+
{
|
|
288
|
+
content: "Document text...",
|
|
289
|
+
metadata: {
|
|
290
|
+
/* document metadata */
|
|
291
|
+
},
|
|
292
|
+
score: 0.95,
|
|
293
|
+
},
|
|
294
|
+
];
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
#### LLM Plugin
|
|
300
|
+
|
|
301
|
+
```javascript
|
|
302
|
+
class MyLLM {
|
|
303
|
+
async generate(prompt, options = {}) {
|
|
304
|
+
// Generate text from prompt
|
|
305
|
+
return {
|
|
306
|
+
text: "Generated response...",
|
|
307
|
+
usage: { tokens: 150 },
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
async *stream(prompt, options = {}) {
|
|
312
|
+
// Stream response chunks
|
|
313
|
+
yield "Generated ";
|
|
314
|
+
yield "response ";
|
|
315
|
+
yield "chunks...";
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
```
|
|
319
|
+
|
|
320
|
+
#### Reranker Plugin (Optional)
|
|
321
|
+
|
|
322
|
+
```javascript
|
|
323
|
+
class MyReranker {
|
|
324
|
+
async rerank(query, documents, options = {}) {
|
|
325
|
+
// Rerank documents based on relevance to query
|
|
326
|
+
return documents
|
|
327
|
+
.map((doc) => ({
|
|
328
|
+
...doc,
|
|
329
|
+
score: calculateRelevanceScore(query, doc),
|
|
330
|
+
}))
|
|
331
|
+
.sort((a, b) => b.score - a.score);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
## Environment Configuration
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
# Example environment variables for your plugins
|
|
340
|
+
OPENAI_API_KEY=your_api_key
|
|
341
|
+
PINECONE_API_KEY=your_api_key
|
|
342
|
+
PINECONE_ENVIRONMENT=us-east-1
|
|
343
|
+
CHROMA_URL=http://localhost:8000
|
|
344
|
+
|
|
345
|
+
# Performance settings
|
|
346
|
+
RAG_MAX_CONCURRENCY=5
|
|
347
|
+
RAG_BATCH_SIZE=10
|
|
348
|
+
```
|
|
349
|
+
|
|
350
|
+
## Advanced Features
|
|
351
|
+
|
|
352
|
+
### Multi-Modal Processing
|
|
353
|
+
|
|
354
|
+
```javascript
|
|
355
|
+
const { MultiModalProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
356
|
+
|
|
357
|
+
const processor = new MultiModalProcessor({
|
|
358
|
+
// Configuration for processing text, images, audio, etc.
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
await processor.process(multiModalData);
|
|
362
|
+
```
|
|
363
|
+
|
|
364
|
+
### Adaptive Retrieval
|
|
365
|
+
|
|
366
|
+
```javascript
|
|
367
|
+
const { AdaptiveRetrievalEngine } = require("@devilsdev/rag-pipeline-utils");
|
|
368
|
+
|
|
369
|
+
const retriever = new AdaptiveRetrievalEngine({
|
|
370
|
+
// Configuration for adaptive retrieval strategies
|
|
371
|
+
});
|
|
372
|
+
|
|
373
|
+
const results = await retriever.retrieve(query, context);
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
### Enterprise Audit Logging
|
|
377
|
+
|
|
378
|
+
```javascript
|
|
379
|
+
const { AuditLogger } = require("@devilsdev/rag-pipeline-utils");
|
|
380
|
+
|
|
381
|
+
const auditLogger = new AuditLogger({
|
|
382
|
+
destination: "./audit-logs",
|
|
383
|
+
});
|
|
384
|
+
|
|
385
|
+
auditLogger.log("query", {
|
|
386
|
+
user: "user@example.com",
|
|
387
|
+
query: "sensitive query",
|
|
388
|
+
timestamp: new Date(),
|
|
389
|
+
});
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
### Data Governance
|
|
393
|
+
|
|
394
|
+
```javascript
|
|
395
|
+
const { DataGovernance } = require("@devilsdev/rag-pipeline-utils");
|
|
396
|
+
|
|
397
|
+
const governance = new DataGovernance({
|
|
398
|
+
policies: {
|
|
399
|
+
retention: "90d",
|
|
400
|
+
encryption: true,
|
|
401
|
+
},
|
|
402
|
+
});
|
|
403
|
+
|
|
404
|
+
await governance.enforce(data);
|
|
405
|
+
```
|
|
406
|
+
|
|
407
|
+
## CLI Usage
|
|
408
|
+
|
|
409
|
+
The package includes a CLI tool for pipeline operations:
|
|
410
|
+
|
|
411
|
+
```bash
|
|
412
|
+
# Initialize configuration
|
|
413
|
+
npx rag-pipeline init
|
|
414
|
+
|
|
415
|
+
# Run pipeline operations (requires plugin implementation)
|
|
416
|
+
npx rag-pipeline ingest ./documents
|
|
417
|
+
npx rag-pipeline query "Your question"
|
|
418
|
+
|
|
419
|
+
# Run system diagnostics
|
|
420
|
+
npx rag-pipeline doctor
|
|
421
|
+
```
|
|
422
|
+
|
|
423
|
+
## Error Handling
|
|
424
|
+
|
|
425
|
+
```javascript
|
|
426
|
+
try {
|
|
427
|
+
const result = await pipeline.query("What is AI?");
|
|
428
|
+
} catch (error) {
|
|
429
|
+
if (error.code === "PLUGIN_NOT_FOUND") {
|
|
430
|
+
console.error("Required plugin not found:", error.message);
|
|
431
|
+
} else if (error.code === "INVALID_CONFIG") {
|
|
432
|
+
console.error("Invalid configuration:", error.message);
|
|
433
|
+
} else {
|
|
434
|
+
console.error("Pipeline error:", error.message);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
## Use Cases
|
|
440
|
+
|
|
441
|
+
### Document Q&A System
|
|
442
|
+
|
|
443
|
+
```javascript
|
|
444
|
+
// Implement your plugins
|
|
445
|
+
const loader = new PDFLoaderPlugin();
|
|
446
|
+
const embedder = new OpenAIEmbedderPlugin({
|
|
447
|
+
apiKey: process.env.OPENAI_API_KEY,
|
|
448
|
+
});
|
|
449
|
+
const retriever = new ChromaRetrieverPlugin({ url: "http://localhost:8000" });
|
|
450
|
+
const llm = new OpenAILLMPlugin({ model: "gpt-4" });
|
|
451
|
+
|
|
452
|
+
// Create pipeline
|
|
453
|
+
const pipeline = createRagPipeline({
|
|
454
|
+
loader,
|
|
455
|
+
embedder,
|
|
456
|
+
retriever,
|
|
457
|
+
llm,
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
// Ingest documents
|
|
461
|
+
await pipeline.ingest("./company-handbook.pdf");
|
|
462
|
+
|
|
463
|
+
// Query
|
|
464
|
+
const answer = await pipeline.query("What is the vacation policy?");
|
|
465
|
+
console.log(answer);
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
### Knowledge Base Search
|
|
469
|
+
|
|
470
|
+
```javascript
|
|
471
|
+
// Use your custom database retriever
|
|
472
|
+
class PostgresRetriever {
|
|
473
|
+
constructor(connectionString) {
|
|
474
|
+
this.db = new PostgresClient(connectionString);
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
async retrieve(queryVector, options) {
|
|
478
|
+
const results = await this.db.query(
|
|
479
|
+
"SELECT content, metadata, vector <-> $1 as score FROM embeddings ORDER BY score LIMIT $2",
|
|
480
|
+
[queryVector, options.topK],
|
|
481
|
+
);
|
|
482
|
+
return results.rows;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
const pipeline = createRagPipeline({
|
|
487
|
+
loader: yourLoader,
|
|
488
|
+
embedder: yourEmbedder,
|
|
489
|
+
retriever: new PostgresRetriever("postgresql://localhost/kb"),
|
|
490
|
+
llm: yourLLM,
|
|
491
|
+
});
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
## Performance Tips
|
|
495
|
+
|
|
496
|
+
### Batch Processing
|
|
497
|
+
|
|
498
|
+
```javascript
|
|
499
|
+
const { ParallelProcessor } = require("@devilsdev/rag-pipeline-utils");
|
|
500
|
+
|
|
501
|
+
const processor = new ParallelProcessor({
|
|
502
|
+
maxConcurrency: 5,
|
|
503
|
+
batchSize: 10,
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
// Process documents in parallel
|
|
507
|
+
await processor.process(documents, async (doc) => {
|
|
508
|
+
return await pipeline.ingest(doc);
|
|
509
|
+
});
|
|
510
|
+
```
|
|
511
|
+
|
|
512
|
+
### Logging and Monitoring
|
|
513
|
+
|
|
514
|
+
```javascript
|
|
515
|
+
const { logger, metrics } = require("@devilsdev/rag-pipeline-utils");
|
|
516
|
+
|
|
517
|
+
// Set log level
|
|
518
|
+
logger.setLevel("debug");
|
|
519
|
+
|
|
520
|
+
// Monitor performance
|
|
521
|
+
const stats = metrics.getMetrics();
|
|
522
|
+
console.log("Total queries:", stats.queries?.total || 0);
|
|
523
|
+
console.log("Average latency:", stats.queries?.avgLatency || 0);
|
|
524
|
+
```
|
|
525
|
+
|
|
526
|
+
## Common Integration Examples
|
|
527
|
+
|
|
528
|
+
### With OpenAI
|
|
529
|
+
|
|
530
|
+
You need to implement OpenAI plugin wrappers:
|
|
531
|
+
|
|
532
|
+
```javascript
|
|
533
|
+
class OpenAIEmbedder {
|
|
534
|
+
constructor({ apiKey, model = "text-embedding-ada-002" }) {
|
|
535
|
+
this.client = new OpenAI({ apiKey });
|
|
536
|
+
this.model = model;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
async embed(texts) {
|
|
540
|
+
const response = await this.client.embeddings.create({
|
|
541
|
+
model: this.model,
|
|
542
|
+
input: texts,
|
|
543
|
+
});
|
|
544
|
+
return response.data.map((d) => d.embedding);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
class OpenAILLM {
|
|
549
|
+
constructor({ apiKey, model = "gpt-4" }) {
|
|
550
|
+
this.client = new OpenAI({ apiKey });
|
|
551
|
+
this.model = model;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
async generate(prompt) {
|
|
555
|
+
const response = await this.client.chat.completions.create({
|
|
556
|
+
model: this.model,
|
|
557
|
+
messages: [{ role: "user", content: prompt }],
|
|
558
|
+
});
|
|
559
|
+
return { text: response.choices[0].message.content };
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
```
|
|
563
|
+
|
|
564
|
+
### With ChromaDB
|
|
565
|
+
|
|
566
|
+
```javascript
|
|
567
|
+
class ChromaRetriever {
|
|
568
|
+
constructor({ url, collection }) {
|
|
569
|
+
this.client = new ChromaClient({ path: url });
|
|
570
|
+
this.collectionName = collection;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
async store(vectors, metadata) {
|
|
574
|
+
const collection = await this.client.getOrCreateCollection({
|
|
575
|
+
name: this.collectionName,
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
await collection.add({
|
|
579
|
+
embeddings: vectors,
|
|
580
|
+
metadatas: metadata,
|
|
581
|
+
ids: metadata.map((_, i) => `doc_${i}`),
|
|
582
|
+
});
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
async retrieve(queryVector, options) {
|
|
586
|
+
const collection = await this.client.getCollection({
|
|
587
|
+
name: this.collectionName,
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
const results = await collection.query({
|
|
591
|
+
queryEmbeddings: [queryVector],
|
|
592
|
+
nResults: options.topK || 5,
|
|
593
|
+
});
|
|
594
|
+
|
|
595
|
+
return results.metadatas[0].map((meta, i) => ({
|
|
596
|
+
content: results.documents[0][i],
|
|
597
|
+
metadata: meta,
|
|
598
|
+
score: 1 - results.distances[0][i],
|
|
599
|
+
}));
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
```
|
|
603
|
+
|
|
604
|
+
## Troubleshooting
|
|
605
|
+
|
|
606
|
+
### Common Issues
|
|
607
|
+
|
|
608
|
+
**Error: Cannot find module**
|
|
609
|
+
|
|
610
|
+
```bash
|
|
611
|
+
npm install @devilsdev/rag-pipeline-utils
|
|
612
|
+
```
|
|
613
|
+
|
|
614
|
+
**Error: Plugin not registered**
|
|
615
|
+
|
|
616
|
+
```javascript
|
|
617
|
+
// Make sure to register your plugins
|
|
618
|
+
pluginRegistry.register("my-embedder", MyEmbedderClass);
|
|
619
|
+
```
|
|
620
|
+
|
|
621
|
+
**Error: Invalid configuration**
|
|
622
|
+
|
|
623
|
+
```javascript
|
|
624
|
+
// Validate your configuration
|
|
625
|
+
const { validateRagrc } = require("@devilsdev/rag-pipeline-utils");
|
|
626
|
+
const isValid = validateRagrc(config);
|
|
627
|
+
if (!isValid) {
|
|
628
|
+
console.error("Configuration is invalid");
|
|
629
|
+
}
|
|
630
|
+
```
|
|
631
|
+
|
|
632
|
+
### Debug Mode
|
|
633
|
+
|
|
634
|
+
```javascript
|
|
635
|
+
const { logger } = require("@devilsdev/rag-pipeline-utils");
|
|
636
|
+
|
|
637
|
+
// Enable debug logging
|
|
638
|
+
logger.setLevel("debug");
|
|
639
|
+
```
|
|
640
|
+
|
|
641
|
+
## Documentation
|
|
642
|
+
|
|
643
|
+
- [GitHub Repository](https://github.com/DevilsDev/rag-pipeline-utils)
|
|
644
|
+
- [Issue Tracker](https://github.com/DevilsDev/rag-pipeline-utils/issues)
|
|
645
|
+
- [Discussions](https://github.com/DevilsDev/rag-pipeline-utils/discussions)
|
|
646
|
+
|
|
647
|
+
## Contributing
|
|
648
|
+
|
|
649
|
+
Contributions are welcome. Please see the [contribution guidelines](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/docs/CONTRIBUTING.md).
|
|
650
|
+
|
|
651
|
+
## License
|
|
652
|
+
|
|
653
|
+
GPL-3.0 - See [LICENSE](https://github.com/DevilsDev/rag-pipeline-utils/blob/main/LICENSE) for details.
|
|
654
|
+
|
|
655
|
+
## Support
|
|
656
|
+
|
|
657
|
+
- Issues: https://github.com/DevilsDev/rag-pipeline-utils/issues
|
|
658
|
+
- Discussions: https://github.com/DevilsDev/rag-pipeline-utils/discussions
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@devilsdev/rag-pipeline-utils",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.12",
|
|
4
4
|
"description": "A comprehensive enterprise-grade toolkit for building RAG (Retrieval-Augmented Generation) pipelines with advanced AI/ML capabilities",
|
|
5
5
|
"main": "dist/index.cjs",
|
|
6
6
|
"type": "commonjs",
|