@msbayindir/context-rag 1.0.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +464 -0
- package/dist/bin/cli.cjs +210 -0
- package/dist/bin/cli.cjs.map +1 -0
- package/dist/bin/cli.d.cts +1 -0
- package/dist/bin/cli.d.ts +1 -0
- package/dist/bin/cli.js +187 -0
- package/dist/bin/cli.js.map +1 -0
- package/dist/index.cjs +2877 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +812 -0
- package/dist/index.d.ts +812 -0
- package/dist/index.js +2842 -0
- package/dist/index.js.map +1 -0
- package/package.json +91 -0
- package/prisma/schema.prisma +225 -0
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,812 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Enhancement Types
|
|
3
|
+
*
|
|
4
|
+
* Hierarchical architecture for different RAG enhancement approaches.
|
|
5
|
+
* Supports Anthropic Contextual Retrieval, future Google Grounding, and custom handlers.
|
|
6
|
+
*/
|
|
7
|
+
interface ChunkData {
|
|
8
|
+
content: string;
|
|
9
|
+
searchContent: string;
|
|
10
|
+
displayContent: string;
|
|
11
|
+
chunkType: string;
|
|
12
|
+
page: number;
|
|
13
|
+
parentHeading?: string;
|
|
14
|
+
}
|
|
15
|
+
interface DocumentContext {
|
|
16
|
+
documentType?: string;
|
|
17
|
+
filename: string;
|
|
18
|
+
pageCount: number;
|
|
19
|
+
/** For LLM strategy: uploaded file URI (Gemini Files API) */
|
|
20
|
+
fileUri?: string;
|
|
21
|
+
/** For LLM strategy: full document text for context (Anthropic-style, max 15k chars) */
|
|
22
|
+
fullDocumentText?: string;
|
|
23
|
+
}
|
|
24
|
+
interface NoEnhancementConfig {
|
|
25
|
+
approach: 'none';
|
|
26
|
+
}
|
|
27
|
+
type AnthropicStrategy = 'none' | 'simple' | 'llm';
|
|
28
|
+
interface AnthropicContextualConfig {
|
|
29
|
+
approach: 'anthropic_contextual';
|
|
30
|
+
strategy: AnthropicStrategy;
|
|
31
|
+
/** For 'simple': template pattern
|
|
32
|
+
* Available placeholders: {documentType}, {chunkType}, {page}, {parentHeading}
|
|
33
|
+
* Example: "[{documentType}] [{chunkType}] Page {page}"
|
|
34
|
+
*/
|
|
35
|
+
template?: string;
|
|
36
|
+
/** For 'llm': context generation prompt */
|
|
37
|
+
contextPrompt?: string;
|
|
38
|
+
/** For 'llm': max tokens for generated context (default: 100) */
|
|
39
|
+
maxContextTokens?: number;
|
|
40
|
+
/** Chunk types to skip context generation for (default: ['HEADING', 'IMAGE_REF']) */
|
|
41
|
+
skipChunkTypes?: string[];
|
|
42
|
+
/** Max concurrent LLM calls (default: 5) */
|
|
43
|
+
concurrencyLimit?: number;
|
|
44
|
+
/** Enable caching to avoid duplicate API calls (default: true) */
|
|
45
|
+
enableCache?: boolean;
|
|
46
|
+
}
|
|
47
|
+
type GoogleGroundingStrategy = 'search' | 'factcheck';
|
|
48
|
+
interface GoogleGroundingConfig {
|
|
49
|
+
approach: 'google_grounding';
|
|
50
|
+
strategy: GoogleGroundingStrategy;
|
|
51
|
+
}
|
|
52
|
+
interface EnhancementContext {
|
|
53
|
+
chunk: ChunkData;
|
|
54
|
+
doc: DocumentContext;
|
|
55
|
+
}
|
|
56
|
+
type CustomEnhancementHandler = (ctx: EnhancementContext) => Promise<string>;
|
|
57
|
+
interface CustomEnhancementConfig {
|
|
58
|
+
approach: 'custom';
|
|
59
|
+
handler: CustomEnhancementHandler;
|
|
60
|
+
/** Chunk types to skip (optional) */
|
|
61
|
+
skipChunkTypes?: string[];
|
|
62
|
+
}
|
|
63
|
+
type RagEnhancementConfig = NoEnhancementConfig | AnthropicContextualConfig | GoogleGroundingConfig | CustomEnhancementConfig;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Generic Prisma client type - allows any Prisma client instance
|
|
67
|
+
* This avoids requiring the user to generate Prisma client before using the library
|
|
68
|
+
*/
|
|
69
|
+
type PrismaClientLike = any;
|
|
70
|
+
/**
|
|
71
|
+
* Batch processing configuration
|
|
72
|
+
*/
|
|
73
|
+
interface BatchConfig {
|
|
74
|
+
/** Number of pages per batch (default: 15) */
|
|
75
|
+
pagesPerBatch: number;
|
|
76
|
+
/** Maximum concurrent batch processing (default: 3) */
|
|
77
|
+
maxConcurrency: number;
|
|
78
|
+
/** Maximum retry attempts for failed batches (default: 3) */
|
|
79
|
+
maxRetries: number;
|
|
80
|
+
/** Initial retry delay in milliseconds (default: 1000) */
|
|
81
|
+
retryDelayMs: number;
|
|
82
|
+
/** Backoff multiplier for exponential retry (default: 2) */
|
|
83
|
+
backoffMultiplier: number;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Chunk configuration
|
|
87
|
+
*/
|
|
88
|
+
interface ChunkConfig {
|
|
89
|
+
/** Maximum tokens per chunk (default: 500) */
|
|
90
|
+
maxTokens: number;
|
|
91
|
+
/** Overlap tokens between chunks (default: 50) */
|
|
92
|
+
overlapTokens: number;
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Rate limiting configuration
|
|
96
|
+
*/
|
|
97
|
+
interface RateLimitConfig {
|
|
98
|
+
/** Requests per minute limit (default: 60) */
|
|
99
|
+
requestsPerMinute: number;
|
|
100
|
+
/** Enable adaptive rate limiting (default: true) */
|
|
101
|
+
adaptive: boolean;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Logging configuration
|
|
105
|
+
*/
|
|
106
|
+
interface LogConfig {
|
|
107
|
+
/** Log level */
|
|
108
|
+
level: 'debug' | 'info' | 'warn' | 'error';
|
|
109
|
+
/** Enable structured JSON logging (default: true) */
|
|
110
|
+
structured: boolean;
|
|
111
|
+
/** Custom logger function */
|
|
112
|
+
customLogger?: (level: string, message: string, meta?: Record<string, unknown>) => void;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Generation configuration for Gemini API
|
|
116
|
+
*/
|
|
117
|
+
interface GenerationConfig {
|
|
118
|
+
/** Temperature for generation (0-2, default: 0.3) */
|
|
119
|
+
temperature: number;
|
|
120
|
+
/** Maximum output tokens (default: 8192) */
|
|
121
|
+
maxOutputTokens: number;
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Main Context-RAG configuration
|
|
125
|
+
*/
|
|
126
|
+
interface ContextRAGConfig {
|
|
127
|
+
/** Prisma client instance */
|
|
128
|
+
prisma: PrismaClientLike;
|
|
129
|
+
/** Gemini API key */
|
|
130
|
+
geminiApiKey: string;
|
|
131
|
+
/** Gemini model to use (default: 'gemini-1.5-pro') */
|
|
132
|
+
model?: 'gemini-1.5-pro' | 'gemini-1.5-flash' | 'gemini-2.0-flash-exp' | 'gemini-pro' | 'gemini-2.5-pro' | 'gemini-3-pro-preview' | 'gemini-3-flash-preview';
|
|
133
|
+
/** Embedding model (default: 'text-embedding-004') */
|
|
134
|
+
embeddingModel?: string;
|
|
135
|
+
/** Generation configuration (temperature, maxOutputTokens) */
|
|
136
|
+
generationConfig?: Partial<GenerationConfig>;
|
|
137
|
+
/** Batch processing configuration */
|
|
138
|
+
batchConfig?: Partial<BatchConfig>;
|
|
139
|
+
/** Chunk configuration */
|
|
140
|
+
chunkConfig?: Partial<ChunkConfig>;
|
|
141
|
+
/** Rate limiting configuration */
|
|
142
|
+
rateLimitConfig?: Partial<RateLimitConfig>;
|
|
143
|
+
/** Logging configuration */
|
|
144
|
+
logging?: Partial<LogConfig>;
|
|
145
|
+
/** RAG Enhancement configuration (Contextual Retrieval, etc.) */
|
|
146
|
+
ragEnhancement?: RagEnhancementConfig;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Internal resolved configuration with all defaults applied
|
|
150
|
+
*/
|
|
151
|
+
interface ResolvedConfig {
|
|
152
|
+
prisma: PrismaClientLike;
|
|
153
|
+
geminiApiKey: string;
|
|
154
|
+
model: string;
|
|
155
|
+
embeddingModel: string;
|
|
156
|
+
generationConfig: GenerationConfig;
|
|
157
|
+
batchConfig: BatchConfig;
|
|
158
|
+
chunkConfig: ChunkConfig;
|
|
159
|
+
rateLimitConfig: RateLimitConfig;
|
|
160
|
+
logging: LogConfig;
|
|
161
|
+
ragEnhancement?: RagEnhancementConfig;
|
|
162
|
+
}
|
|
163
|
+
type ContextRAGOptions = Omit<ContextRAGConfig, 'prisma'>;
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Chunk type enumeration
|
|
167
|
+
* Defines the type of content within a chunk
|
|
168
|
+
*/
|
|
169
|
+
declare const ChunkTypeEnum: {
|
|
170
|
+
readonly TEXT: "TEXT";
|
|
171
|
+
readonly TABLE: "TABLE";
|
|
172
|
+
readonly LIST: "LIST";
|
|
173
|
+
readonly CODE: "CODE";
|
|
174
|
+
readonly HEADING: "HEADING";
|
|
175
|
+
readonly IMAGE_REF: "IMAGE_REF";
|
|
176
|
+
readonly QUOTE: "QUOTE";
|
|
177
|
+
readonly QUESTION: "QUESTION";
|
|
178
|
+
readonly MIXED: "MIXED";
|
|
179
|
+
};
|
|
180
|
+
type ChunkTypeEnumType = (typeof ChunkTypeEnum)[keyof typeof ChunkTypeEnum];
|
|
181
|
+
/**
|
|
182
|
+
* Batch processing status enumeration
|
|
183
|
+
*/
|
|
184
|
+
declare const BatchStatusEnum: {
|
|
185
|
+
readonly PENDING: "PENDING";
|
|
186
|
+
readonly PROCESSING: "PROCESSING";
|
|
187
|
+
readonly RETRYING: "RETRYING";
|
|
188
|
+
readonly COMPLETED: "COMPLETED";
|
|
189
|
+
readonly FAILED: "FAILED";
|
|
190
|
+
};
|
|
191
|
+
type BatchStatusEnumType = (typeof BatchStatusEnum)[keyof typeof BatchStatusEnum];
|
|
192
|
+
/**
|
|
193
|
+
* Document processing status enumeration
|
|
194
|
+
*/
|
|
195
|
+
declare const DocumentStatusEnum: {
|
|
196
|
+
readonly PENDING: "PENDING";
|
|
197
|
+
readonly DISCOVERING: "DISCOVERING";
|
|
198
|
+
readonly AWAITING_APPROVAL: "AWAITING_APPROVAL";
|
|
199
|
+
readonly PROCESSING: "PROCESSING";
|
|
200
|
+
readonly COMPLETED: "COMPLETED";
|
|
201
|
+
readonly FAILED: "FAILED";
|
|
202
|
+
readonly PARTIAL: "PARTIAL";
|
|
203
|
+
};
|
|
204
|
+
type DocumentStatusEnumType = (typeof DocumentStatusEnum)[keyof typeof DocumentStatusEnum];
|
|
205
|
+
/**
|
|
206
|
+
* Confidence level enumeration
|
|
207
|
+
*/
|
|
208
|
+
declare const ConfidenceLevelEnum: {
|
|
209
|
+
readonly HIGH: "HIGH";
|
|
210
|
+
readonly MEDIUM: "MEDIUM";
|
|
211
|
+
readonly LOW: "LOW";
|
|
212
|
+
};
|
|
213
|
+
type ConfidenceLevelEnumType = (typeof ConfidenceLevelEnum)[keyof typeof ConfidenceLevelEnum];
|
|
214
|
+
/**
|
|
215
|
+
* Search mode enumeration
|
|
216
|
+
*/
|
|
217
|
+
declare const SearchModeEnum: {
|
|
218
|
+
readonly SEMANTIC: "semantic";
|
|
219
|
+
readonly KEYWORD: "keyword";
|
|
220
|
+
readonly HYBRID: "hybrid";
|
|
221
|
+
};
|
|
222
|
+
type SearchModeEnumType = (typeof SearchModeEnum)[keyof typeof SearchModeEnum];
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Chunk type alias for external use
|
|
226
|
+
*/
|
|
227
|
+
type ChunkType = ChunkTypeEnumType;
|
|
228
|
+
/**
|
|
229
|
+
* Confidence metadata for chunks
|
|
230
|
+
*/
|
|
231
|
+
interface ConfidenceMetadata {
|
|
232
|
+
/** Numeric confidence score (0.0 - 1.0) */
|
|
233
|
+
score: number;
|
|
234
|
+
/** Categorical confidence level */
|
|
235
|
+
category: ConfidenceLevelEnumType;
|
|
236
|
+
/** Individual factors contributing to confidence */
|
|
237
|
+
factors?: {
|
|
238
|
+
textClarity?: number;
|
|
239
|
+
structureRecognition?: number;
|
|
240
|
+
tableAccuracy?: number;
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* Token usage tracking
|
|
245
|
+
*/
|
|
246
|
+
interface TokenUsage {
|
|
247
|
+
input: number;
|
|
248
|
+
output: number;
|
|
249
|
+
total: number;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Chunk metadata stored in JSON column
|
|
253
|
+
*/
|
|
254
|
+
interface ChunkMetadata {
|
|
255
|
+
/** Source page number (single page) */
|
|
256
|
+
page?: number;
|
|
257
|
+
/** Source page range */
|
|
258
|
+
pageRange?: {
|
|
259
|
+
start: number;
|
|
260
|
+
end: number;
|
|
261
|
+
};
|
|
262
|
+
/** Chunk type */
|
|
263
|
+
type: ChunkType;
|
|
264
|
+
/** Confidence information */
|
|
265
|
+
confidence: ConfidenceMetadata;
|
|
266
|
+
/** Token usage for this chunk */
|
|
267
|
+
tokens?: TokenUsage;
|
|
268
|
+
/** Processing duration in milliseconds */
|
|
269
|
+
processingDurationMs?: number;
|
|
270
|
+
/** Section or heading this chunk belongs to */
|
|
271
|
+
section?: string;
|
|
272
|
+
/** Keywords extracted from content */
|
|
273
|
+
keywords?: string[];
|
|
274
|
+
/** Whether parsed with structured SECTION markers */
|
|
275
|
+
parsedWithStructuredMarkers?: boolean;
|
|
276
|
+
/** Custom metadata added by user */
|
|
277
|
+
custom?: Record<string, unknown>;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* Chunking strategy configuration
|
|
281
|
+
*/
|
|
282
|
+
interface ChunkStrategy {
|
|
283
|
+
/** Maximum tokens per chunk */
|
|
284
|
+
maxTokens: number;
|
|
285
|
+
/** Overlap tokens between chunks */
|
|
286
|
+
overlapTokens: number;
|
|
287
|
+
/** How to split the document */
|
|
288
|
+
splitBy: 'page' | 'section' | 'paragraph' | 'semantic';
|
|
289
|
+
/** Preserve table integrity (don't split tables) */
|
|
290
|
+
preserveTables?: boolean;
|
|
291
|
+
/** Preserve list integrity */
|
|
292
|
+
preserveLists?: boolean;
|
|
293
|
+
/** Extract headings as separate chunks */
|
|
294
|
+
extractHeadings?: boolean;
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Vector chunk data structure
|
|
298
|
+
*/
|
|
299
|
+
interface VectorChunk {
|
|
300
|
+
id: string;
|
|
301
|
+
promptConfigId: string;
|
|
302
|
+
documentId: string;
|
|
303
|
+
chunkIndex: number;
|
|
304
|
+
chunkType: ChunkType;
|
|
305
|
+
/** Content optimized for vector search */
|
|
306
|
+
searchContent: string;
|
|
307
|
+
/** Rich Markdown content for display */
|
|
308
|
+
displayContent: string;
|
|
309
|
+
sourcePageStart: number;
|
|
310
|
+
sourcePageEnd: number;
|
|
311
|
+
confidenceScore: number;
|
|
312
|
+
metadata: ChunkMetadata;
|
|
313
|
+
createdAt: Date;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
/**
|
|
317
|
+
* Discovery options
|
|
318
|
+
*/
|
|
319
|
+
interface DiscoveryOptions {
|
|
320
|
+
/** PDF file as buffer or file path */
|
|
321
|
+
file: Buffer | string;
|
|
322
|
+
/** Hint about expected document type */
|
|
323
|
+
documentTypeHint?: string;
|
|
324
|
+
/** Generate sample output for preview */
|
|
325
|
+
generateSample?: boolean;
|
|
326
|
+
/** Number of sample pages to analyze (default: 5) */
|
|
327
|
+
samplePages?: number;
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Detected document element
|
|
331
|
+
*/
|
|
332
|
+
interface DetectedElement {
|
|
333
|
+
type: 'table' | 'list' | 'code' | 'image' | 'chart' | 'form' | 'heading';
|
|
334
|
+
count: number;
|
|
335
|
+
/** Example locations (page numbers) */
|
|
336
|
+
examples?: number[];
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Discovery result from AI analysis
|
|
340
|
+
*/
|
|
341
|
+
interface DiscoveryResult {
|
|
342
|
+
/** Unique ID for this discovery session */
|
|
343
|
+
id: string;
|
|
344
|
+
/** Detected document type */
|
|
345
|
+
documentType: string;
|
|
346
|
+
/** Human-readable document type name */
|
|
347
|
+
documentTypeName: string;
|
|
348
|
+
/** Detected structural elements */
|
|
349
|
+
detectedElements: DetectedElement[];
|
|
350
|
+
/** Document-specific extraction instructions (NEW) */
|
|
351
|
+
specialInstructions: string[];
|
|
352
|
+
/** Example formats for consistent extraction (NEW) */
|
|
353
|
+
exampleFormats?: Record<string, string>;
|
|
354
|
+
/** AI-generated system prompt for ingestion (DEPRECATED - use specialInstructions) */
|
|
355
|
+
suggestedPrompt?: string;
|
|
356
|
+
/** Suggested chunking strategy */
|
|
357
|
+
suggestedChunkStrategy: ChunkStrategy;
|
|
358
|
+
/** AI confidence in this analysis (0.0 - 1.0) */
|
|
359
|
+
confidence: number;
|
|
360
|
+
/** Sample output if requested */
|
|
361
|
+
sampleOutput?: string;
|
|
362
|
+
/** Analysis reasoning */
|
|
363
|
+
reasoning: string;
|
|
364
|
+
/** Total pages in document */
|
|
365
|
+
pageCount: number;
|
|
366
|
+
/** File hash for idempotency */
|
|
367
|
+
fileHash: string;
|
|
368
|
+
/** Timestamps */
|
|
369
|
+
createdAt: Date;
|
|
370
|
+
expiresAt: Date;
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Strategy approval options
|
|
374
|
+
*/
|
|
375
|
+
interface ApproveStrategyOptions {
|
|
376
|
+
/** Override the suggested prompt */
|
|
377
|
+
systemPrompt?: string;
|
|
378
|
+
/** Override document type */
|
|
379
|
+
documentType?: string;
|
|
380
|
+
/** Override chunk strategy */
|
|
381
|
+
chunkStrategy?: Partial<ChunkStrategy>;
|
|
382
|
+
/** Custom name for this config */
|
|
383
|
+
name?: string;
|
|
384
|
+
/** Change log / reason for modifications */
|
|
385
|
+
changeLog?: string;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Batch status during ingestion
|
|
390
|
+
*/
|
|
391
|
+
interface BatchStatus {
|
|
392
|
+
/** Current batch index (1-based) */
|
|
393
|
+
current: number;
|
|
394
|
+
/** Total number of batches */
|
|
395
|
+
total: number;
|
|
396
|
+
/** Batch status */
|
|
397
|
+
status: BatchStatusEnumType;
|
|
398
|
+
/** Page range being processed */
|
|
399
|
+
pageRange: {
|
|
400
|
+
start: number;
|
|
401
|
+
end: number;
|
|
402
|
+
};
|
|
403
|
+
/** Retry count if retrying */
|
|
404
|
+
retryCount?: number;
|
|
405
|
+
/** Error message if failed */
|
|
406
|
+
error?: string;
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* Progress callback type
|
|
410
|
+
*/
|
|
411
|
+
type ProgressCallback = (status: BatchStatus) => void;
|
|
412
|
+
/**
|
|
413
|
+
* Ingestion options
|
|
414
|
+
*/
|
|
415
|
+
interface IngestOptions {
|
|
416
|
+
/** PDF file as buffer or file path */
|
|
417
|
+
file: Buffer | string;
|
|
418
|
+
/** Document type (must match a PromptConfig) */
|
|
419
|
+
documentType?: string;
|
|
420
|
+
/** Specific prompt config ID to use */
|
|
421
|
+
promptConfigId?: string;
|
|
422
|
+
/** Custom prompt override (creates temporary config) */
|
|
423
|
+
customPrompt?: string;
|
|
424
|
+
/** Custom filename (if file is buffer) */
|
|
425
|
+
filename?: string;
|
|
426
|
+
/** Experiment identifier for A/B testing different models
|
|
427
|
+
* e.g., "exp_flash_v1", "exp_pro_v2"
|
|
428
|
+
* Allows same PDF to be processed multiple times with different configs
|
|
429
|
+
*/
|
|
430
|
+
experimentId?: string;
|
|
431
|
+
/** Progress callback */
|
|
432
|
+
onProgress?: ProgressCallback;
|
|
433
|
+
/** Skip if document already exists (based on hash + experimentId) */
|
|
434
|
+
skipExisting?: boolean;
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Individual batch result
|
|
438
|
+
*/
|
|
439
|
+
interface BatchResult {
|
|
440
|
+
batchIndex: number;
|
|
441
|
+
status: BatchStatusEnumType;
|
|
442
|
+
chunksCreated: number;
|
|
443
|
+
tokenUsage: TokenUsage;
|
|
444
|
+
processingMs: number;
|
|
445
|
+
retryCount: number;
|
|
446
|
+
error?: string;
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* Ingestion result
|
|
450
|
+
*/
|
|
451
|
+
interface IngestResult {
|
|
452
|
+
/** Document ID */
|
|
453
|
+
documentId: string;
|
|
454
|
+
/** Final status */
|
|
455
|
+
status: DocumentStatusEnumType;
|
|
456
|
+
/** Total chunks created */
|
|
457
|
+
chunkCount: number;
|
|
458
|
+
/** Total batches processed */
|
|
459
|
+
batchCount: number;
|
|
460
|
+
/** Failed batch count */
|
|
461
|
+
failedBatchCount: number;
|
|
462
|
+
/** Total token usage */
|
|
463
|
+
tokenUsage: TokenUsage;
|
|
464
|
+
/** Total processing time in milliseconds */
|
|
465
|
+
processingMs: number;
|
|
466
|
+
/** Per-batch results */
|
|
467
|
+
batches: BatchResult[];
|
|
468
|
+
/** Warning messages (e.g., partial failures) */
|
|
469
|
+
warnings?: string[];
|
|
470
|
+
}
|
|
471
|
+
/**
|
|
472
|
+
* Document status query result
|
|
473
|
+
*/
|
|
474
|
+
interface DocumentStatus {
|
|
475
|
+
id: string;
|
|
476
|
+
filename: string;
|
|
477
|
+
status: DocumentStatusEnumType;
|
|
478
|
+
documentType?: string;
|
|
479
|
+
pageCount: number;
|
|
480
|
+
progress: {
|
|
481
|
+
totalBatches: number;
|
|
482
|
+
completedBatches: number;
|
|
483
|
+
failedBatches: number;
|
|
484
|
+
percentage: number;
|
|
485
|
+
};
|
|
486
|
+
tokenUsage?: TokenUsage;
|
|
487
|
+
processingMs?: number;
|
|
488
|
+
error?: string;
|
|
489
|
+
createdAt: Date;
|
|
490
|
+
completedAt?: Date;
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Retry options for failed batches
|
|
494
|
+
*/
|
|
495
|
+
interface RetryOptions {
|
|
496
|
+
/** Only retry batches that failed with specific errors */
|
|
497
|
+
errorFilter?: string[];
|
|
498
|
+
/** Override max retries for this retry attempt */
|
|
499
|
+
maxRetries?: number;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Search mode type alias
|
|
504
|
+
*/
|
|
505
|
+
type SearchMode = SearchModeEnumType;
|
|
506
|
+
/**
|
|
507
|
+
* Search filters
|
|
508
|
+
*/
|
|
509
|
+
interface SearchFilters {
|
|
510
|
+
/** Filter by document types */
|
|
511
|
+
documentTypes?: string[];
|
|
512
|
+
/** Filter by chunk types */
|
|
513
|
+
chunkTypes?: ChunkType[];
|
|
514
|
+
/** Minimum confidence score (0.0 - 1.0) */
|
|
515
|
+
minConfidence?: number;
|
|
516
|
+
/** Filter by specific document IDs */
|
|
517
|
+
documentIds?: string[];
|
|
518
|
+
/** Filter by page range */
|
|
519
|
+
pageRange?: {
|
|
520
|
+
start: number;
|
|
521
|
+
end: number;
|
|
522
|
+
};
|
|
523
|
+
/** Filter by prompt config IDs */
|
|
524
|
+
promptConfigIds?: string[];
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Search options
|
|
528
|
+
*/
|
|
529
|
+
interface SearchOptions {
|
|
530
|
+
/** The search query */
|
|
531
|
+
query: string;
|
|
532
|
+
/** Search mode: semantic, keyword, or hybrid (default: hybrid) */
|
|
533
|
+
mode?: SearchMode;
|
|
534
|
+
/** Maximum number of results (default: 10) */
|
|
535
|
+
limit?: number;
|
|
536
|
+
/** Minimum similarity score threshold (0.0 - 1.0) */
|
|
537
|
+
minScore?: number;
|
|
538
|
+
/** Search filters */
|
|
539
|
+
filters?: SearchFilters;
|
|
540
|
+
/** Include explanation of why results matched */
|
|
541
|
+
includeExplanation?: boolean;
|
|
542
|
+
/** Boost results containing specific chunk types */
|
|
543
|
+
typeBoost?: Partial<Record<ChunkType, number>>;
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* Search result explanation
|
|
547
|
+
*/
|
|
548
|
+
interface SearchExplanation {
|
|
549
|
+
/** How the result matched */
|
|
550
|
+
matchType: 'semantic' | 'keyword' | 'both';
|
|
551
|
+
/** Terms that matched (for keyword search) */
|
|
552
|
+
matchedTerms?: string[];
|
|
553
|
+
/** Whether intent boosting was applied */
|
|
554
|
+
intentBoost?: boolean;
|
|
555
|
+
/** Boost reason if applied */
|
|
556
|
+
boostReason?: string;
|
|
557
|
+
/** Raw scores before normalization */
|
|
558
|
+
rawScores?: {
|
|
559
|
+
semantic?: number;
|
|
560
|
+
keyword?: number;
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
/**
|
|
564
|
+
* Individual search result
|
|
565
|
+
*/
|
|
566
|
+
interface SearchResult {
|
|
567
|
+
/** The matched chunk */
|
|
568
|
+
chunk: VectorChunk;
|
|
569
|
+
/** Relevance score (0.0 - 1.0) */
|
|
570
|
+
score: number;
|
|
571
|
+
/** Explanation of match (if requested) */
|
|
572
|
+
explanation?: SearchExplanation;
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Search response metadata
|
|
576
|
+
*/
|
|
577
|
+
interface SearchMetadata {
|
|
578
|
+
/** Total results found (before limit) */
|
|
579
|
+
totalFound: number;
|
|
580
|
+
/** Processing time in milliseconds */
|
|
581
|
+
processingTimeMs: number;
|
|
582
|
+
/** Search mode used */
|
|
583
|
+
searchMode: SearchMode;
|
|
584
|
+
/** Query embedding token usage */
|
|
585
|
+
embeddingTokens?: number;
|
|
586
|
+
}
|
|
587
|
+
/**
|
|
588
|
+
* Full search response
|
|
589
|
+
*/
|
|
590
|
+
interface SearchResponse {
|
|
591
|
+
results: SearchResult[];
|
|
592
|
+
metadata: SearchMetadata;
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* Prompt configuration stored in database
|
|
597
|
+
*/
|
|
598
|
+
interface PromptConfig {
|
|
599
|
+
id: string;
|
|
600
|
+
/** Document type identifier (e.g., 'Medical', 'Legal') */
|
|
601
|
+
documentType: string;
|
|
602
|
+
/** Human-readable name */
|
|
603
|
+
name: string;
|
|
604
|
+
/** System prompt for AI processing */
|
|
605
|
+
systemPrompt: string;
|
|
606
|
+
/** Chunking strategy configuration */
|
|
607
|
+
chunkStrategy: ChunkStrategy;
|
|
608
|
+
/** Version number for this document type */
|
|
609
|
+
version: number;
|
|
610
|
+
/** Whether this config is active */
|
|
611
|
+
isActive: boolean;
|
|
612
|
+
/** Whether this is the default for the document type */
|
|
613
|
+
isDefault: boolean;
|
|
614
|
+
/** Who created this config: 'discovery' | 'manual' | user ID */
|
|
615
|
+
createdBy?: string;
|
|
616
|
+
/** Change log / reason for this version */
|
|
617
|
+
changeLog?: string;
|
|
618
|
+
createdAt: Date;
|
|
619
|
+
updatedAt: Date;
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* Input for creating a new prompt config
|
|
623
|
+
*/
|
|
624
|
+
interface CreatePromptConfig {
|
|
625
|
+
/** Document type identifier */
|
|
626
|
+
documentType: string;
|
|
627
|
+
/** Human-readable name */
|
|
628
|
+
name: string;
|
|
629
|
+
/** System prompt for AI processing */
|
|
630
|
+
systemPrompt: string;
|
|
631
|
+
/** Optional chunk strategy (defaults applied if not provided) */
|
|
632
|
+
chunkStrategy?: Partial<ChunkStrategy>;
|
|
633
|
+
/** Set as default for this document type */
|
|
634
|
+
setAsDefault?: boolean;
|
|
635
|
+
/** Change log / reason */
|
|
636
|
+
changeLog?: string;
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* Input for updating a prompt config
|
|
640
|
+
*/
|
|
641
|
+
interface UpdatePromptConfig {
|
|
642
|
+
/** New system prompt */
|
|
643
|
+
systemPrompt?: string;
|
|
644
|
+
/** New chunk strategy */
|
|
645
|
+
chunkStrategy?: Partial<ChunkStrategy>;
|
|
646
|
+
/** New name */
|
|
647
|
+
name?: string;
|
|
648
|
+
/** Change log / reason for update */
|
|
649
|
+
changeLog?: string;
|
|
650
|
+
}
|
|
651
|
+
/**
|
|
652
|
+
* Prompt config query filters
|
|
653
|
+
*/
|
|
654
|
+
interface PromptConfigFilters {
|
|
655
|
+
/** Filter by document type */
|
|
656
|
+
documentType?: string;
|
|
657
|
+
/** Only active configs */
|
|
658
|
+
activeOnly?: boolean;
|
|
659
|
+
/** Only default configs */
|
|
660
|
+
defaultOnly?: boolean;
|
|
661
|
+
/** Created by filter */
|
|
662
|
+
createdBy?: string;
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
/**
|
|
666
|
+
* Main Context-RAG engine class
|
|
667
|
+
*
|
|
668
|
+
* @example
|
|
669
|
+
* ```typescript
|
|
670
|
+
* import { ContextRAG } from 'context-rag';
|
|
671
|
+
* import { PrismaClient } from '@prisma/client';
|
|
672
|
+
*
|
|
673
|
+
* const prisma = new PrismaClient();
|
|
674
|
+
* const rag = new ContextRAG({
|
|
675
|
+
* prisma,
|
|
676
|
+
* geminiApiKey: process.env.GEMINI_API_KEY!,
|
|
677
|
+
* });
|
|
678
|
+
*
|
|
679
|
+
* // Ingest a document
|
|
680
|
+
* const result = await rag.ingest({ file: pdfBuffer });
|
|
681
|
+
*
|
|
682
|
+
* // Search
|
|
683
|
+
* const results = await rag.search({ query: 'your query' });
|
|
684
|
+
* ```
|
|
685
|
+
*/
|
|
686
|
+
declare class ContextRAG {
|
|
687
|
+
private readonly config;
|
|
688
|
+
private readonly logger;
|
|
689
|
+
private readonly rateLimiter;
|
|
690
|
+
private readonly ingestionEngine;
|
|
691
|
+
private readonly retrievalEngine;
|
|
692
|
+
private readonly discoveryEngine;
|
|
693
|
+
private readonly promptConfigRepo;
|
|
694
|
+
private readonly documentRepo;
|
|
695
|
+
private readonly chunkRepo;
|
|
696
|
+
constructor(userConfig: ContextRAGConfig);
|
|
697
|
+
/**
|
|
698
|
+
* Resolve user config with defaults
|
|
699
|
+
*/
|
|
700
|
+
private resolveConfig;
|
|
701
|
+
/**
|
|
702
|
+
* Get the resolved configuration
|
|
703
|
+
*/
|
|
704
|
+
getConfig(): ResolvedConfig;
|
|
705
|
+
/**
|
|
706
|
+
* Analyze a document and get AI-suggested processing strategy
|
|
707
|
+
*/
|
|
708
|
+
discover(options: DiscoveryOptions): Promise<DiscoveryResult>;
|
|
709
|
+
/**
|
|
710
|
+
* Approve a discovery strategy and create a prompt config
|
|
711
|
+
*/
|
|
712
|
+
approveStrategy(strategyId: string, overrides?: ApproveStrategyOptions): Promise<PromptConfig>;
|
|
713
|
+
/**
|
|
714
|
+
* Create a custom prompt configuration
|
|
715
|
+
*/
|
|
716
|
+
createPromptConfig(config: CreatePromptConfig): Promise<PromptConfig>;
|
|
717
|
+
/**
|
|
718
|
+
* Get prompt configurations
|
|
719
|
+
*/
|
|
720
|
+
getPromptConfigs(filters?: PromptConfigFilters): Promise<PromptConfig[]>;
|
|
721
|
+
/**
|
|
722
|
+
* Update a prompt configuration (creates new version)
|
|
723
|
+
*/
|
|
724
|
+
updatePromptConfig(id: string, updates: UpdatePromptConfig): Promise<PromptConfig>;
|
|
725
|
+
/**
|
|
726
|
+
* Activate a specific prompt config version
|
|
727
|
+
*/
|
|
728
|
+
activatePromptConfig(id: string): Promise<void>;
|
|
729
|
+
/**
|
|
730
|
+
* Ingest a document into the RAG system
|
|
731
|
+
*/
|
|
732
|
+
ingest(options: IngestOptions): Promise<IngestResult>;
|
|
733
|
+
/**
|
|
734
|
+
* Get the status of a document processing job
|
|
735
|
+
*/
|
|
736
|
+
getDocumentStatus(documentId: string): Promise<DocumentStatus>;
|
|
737
|
+
/**
|
|
738
|
+
* Retry failed batches for a document
|
|
739
|
+
*/
|
|
740
|
+
retryFailedBatches(documentId: string, _options?: RetryOptions): Promise<IngestResult>;
|
|
741
|
+
/**
|
|
742
|
+
* Search for relevant content
|
|
743
|
+
*/
|
|
744
|
+
search(options: SearchOptions): Promise<SearchResult[]>;
|
|
745
|
+
/**
|
|
746
|
+
* Search with full metadata response
|
|
747
|
+
*/
|
|
748
|
+
searchWithMetadata(options: SearchOptions): Promise<SearchResponse>;
|
|
749
|
+
/**
|
|
750
|
+
* Delete a document and all its chunks
|
|
751
|
+
*/
|
|
752
|
+
deleteDocument(documentId: string): Promise<void>;
|
|
753
|
+
/**
|
|
754
|
+
* Get system statistics
|
|
755
|
+
*/
|
|
756
|
+
getStats(): Promise<{
|
|
757
|
+
totalDocuments: number;
|
|
758
|
+
totalChunks: number;
|
|
759
|
+
promptConfigs: number;
|
|
760
|
+
storageBytes: number;
|
|
761
|
+
}>;
|
|
762
|
+
/**
|
|
763
|
+
* Health check
|
|
764
|
+
*/
|
|
765
|
+
healthCheck(): Promise<{
|
|
766
|
+
status: 'healthy' | 'degraded' | 'unhealthy';
|
|
767
|
+
database: boolean;
|
|
768
|
+
pgvector: boolean;
|
|
769
|
+
}>;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/**
|
|
773
|
+
* Base error class for Context-RAG
|
|
774
|
+
*/
|
|
775
|
+
declare class ContextRAGError extends Error {
|
|
776
|
+
readonly code: string;
|
|
777
|
+
readonly details?: Record<string, unknown>;
|
|
778
|
+
constructor(message: string, code: string, details?: Record<string, unknown>);
|
|
779
|
+
toJSON(): Record<string, unknown>;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Configuration-related errors
|
|
783
|
+
*/
|
|
784
|
+
declare class ConfigurationError extends ContextRAGError {
|
|
785
|
+
constructor(message: string, details?: Record<string, unknown>);
|
|
786
|
+
}
|
|
787
|
+
/**
|
|
788
|
+
* Ingestion-related errors
|
|
789
|
+
*/
|
|
790
|
+
declare class IngestionError extends ContextRAGError {
|
|
791
|
+
readonly batchIndex?: number;
|
|
792
|
+
readonly retryable: boolean;
|
|
793
|
+
constructor(message: string, options?: {
|
|
794
|
+
batchIndex?: number;
|
|
795
|
+
retryable?: boolean;
|
|
796
|
+
details?: Record<string, unknown>;
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
/**
|
|
800
|
+
* Search-related errors
|
|
801
|
+
*/
|
|
802
|
+
declare class SearchError extends ContextRAGError {
|
|
803
|
+
constructor(message: string, details?: Record<string, unknown>);
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Discovery-related errors
|
|
807
|
+
*/
|
|
808
|
+
declare class DiscoveryError extends ContextRAGError {
|
|
809
|
+
constructor(message: string, details?: Record<string, unknown>);
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
export { type BatchStatus, BatchStatusEnum, type ChunkMetadata, type ChunkStrategy, type ChunkType, ChunkTypeEnum, ConfigurationError, ContextRAG, type ContextRAGConfig, ContextRAGError, type ContextRAGOptions, type CreatePromptConfig, DiscoveryError, type DiscoveryOptions, type DiscoveryResult, type DocumentStatus, DocumentStatusEnum, type IngestOptions, type IngestResult, IngestionError, type PromptConfig, SearchError, type SearchFilters, type SearchMode, type SearchOptions, type SearchResult, type VectorChunk };
|