@denizsvnc/ai-rag-engine 0.0.0-stage → 2.2.1

package/dist/services/document-ingestion.service.d.ts

@@ -0,0 +1,116 @@
+import type { IIngestionBackend } from '../interfaces/ingestion.interface.js';
+import type { IVectorStoreAdapter } from '../interfaces/vector.interface.js';
+import { SMLEngine } from '../core/sml-engine.js';
+import { type PIIPolicy } from '../core/security/pii-guard.js';
+import type { IJobQueue, Job, JobStatus } from '../core/queue/job-queue.interface.js';
+export interface DocumentUploadPayload {
+    documentId?: string | undefined;
+    idempotencyKey?: string | undefined;
+    encoding?: 'utf8' | 'base64' | undefined;
+    filename: string;
+    content: string | Buffer | NodeJS.ReadableStream;
+    mimeType?: string | undefined;
+    tenantId?: string | undefined;
+    userId?: string | undefined;
+}
+export interface StreamIngestionOptions {
+    mimeType?: string | undefined;
+    tenantId?: string | undefined;
+    userId?: string | undefined;
+    onProgress?: ((chunksCount: number) => void) | undefined;
+}
+export interface IngestionJobResult {
+    filename: string;
+    chunksCount: number;
+    piiEntitiesMasked: number;
+    documentIds: string[];
+    processedAt: string;
+}
+export interface DocumentIngestionConfig {
+    backend?: IIngestionBackend | undefined;
+    vectorStore?: IVectorStoreAdapter | undefined;
+    smlEngine?: SMLEngine | undefined;
+    jobQueue?: IJobQueue | undefined;
+    chunkSize?: number | undefined;
+    chunkOverlap?: number | undefined;
+    piiPolicy?: PIIPolicy | undefined;
+}
+export declare class DocumentIngestionService {
+    private workerReady;
+    private backend?;
+    private vectorStore?;
+    private smlEngine?;
+    private jobQueue;
+    private chunkSize;
+    private chunkOverlap;
+    private piiPolicy?;
+    constructor(config?: DocumentIngestionConfig);
+    /**
+     * Inspects magic bytes and file extension to block dangerous executables and scripts.
+     */
+    static validateFileSafety(filename: string, buffer: Buffer): {
+        safe: boolean;
+        reason?: string;
+    };
+    /**
+     * Inspects magic bytes from an active stream non-destructively by peeking the initial 4KB chunk.
+     * Prevents loading multi-gigabyte files into memory just to check safety headers.
+     */
+    static validateStreamSafety(filename: string, stream: NodeJS.ReadableStream): Promise<{
+        safe: boolean;
+        reason?: string | undefined;
+        safeStream: NodeJS.ReadableStream;
+    }>;
+    /**
+     * Enterprise Lexical Document Text Extractor.
+     * Decodes and normalizes textual content across PDF streams, Markdown, CSV, JSON, and plaintext files.
+     *
+     * PDF Specification Compliance:
+     * - Parses text execution state matrices within BT (Begin Text) ... ET (End Text) blocks.
+     * - Evaluates 'Tj', '\'', and '"' text-showing operators with PDF octal and hexadecimal escape decoding.
+     * - Parses 'TJ' array kerning sequences, preserving natural word spacing between glyph segments.
+     * - Evaluates positional newline operators ('T*', 'Td', 'TD', 'Tm') to maintain structural paragraph integrity.
+     * - Performs binary artifact suppression and multi-lingual UTF-8 normalization (Turkish, European, and CJK diacritics).
+     */
+    static extractText(filename: string, buffer: Buffer): string;
+    /**
+     * Enterprise High-Efficiency PDF Lexical Extractor.
+     * Parses PDF content streams without heavy external C++/WASM dependencies.
+     */
+    static extractPdfText(rawContent: string): string;
+    static chunkText(text: string, chunkSize?: number, overlap?: number): string[];
+    /**
+     * Streaming chunk generator.
+     * Ingests chunks of text or buffers incrementally with natural backpressure,
+     * maintaining a constant O(1) memory footprint regardless of file size.
+     */
+    static chunkStream(textStream: AsyncIterable<string | Buffer>, chunkSize?: number, overlap?: number): AsyncGenerator<string, void, unknown>;
+    /**
+     * Directly streams and chunks large documents without intermediate memory bloat.
+     * Sanitizes PII and inserts documents into the vector store with automatic backpressure.
+     */
+    ingestStreamDirect(filename: string, stream: NodeJS.ReadableStream, options?: StreamIngestionOptions): Promise<IngestionJobResult>;
+    /**
+     * Submits a stream for asynchronous background queue processing.
+     * Uses backpressure-regulated disk spooling to temporary storage, preventing V8 heap exhaustion.
+     */
+    submitStream(filename: string, stream: NodeJS.ReadableStream, options?: StreamIngestionOptions): Promise<{
+        jobId: string;
+        status: JobStatus;
+    }>;
+    /**
+     * Submits a document for background asynchronous ingestion.
+     * Supports String, Buffer, or ReadableStream payloads.
+     */
+    submitDocument(payload: DocumentUploadPayload): Promise<{
+        jobId: string;
+        status: JobStatus;
+    }>;
+    /**
+     * Retrieves the status and progress of an ingestion job.
+     */
+    getJobStatus(jobId: string, tenantId?: string): Promise<Job | null>;
+    retryJob(jobId: string, tenantId: string): Promise<void>;
+    private initWorker;
+    private toBuffer;
+}

package/dist/utils/http.d.ts

@@ -0,0 +1,2 @@
+/** Prevent provider calls from holding an HTTP request indefinitely. */
+export declare function providerTimeoutSignal(): AbortSignal;

package/dist/utils/math.d.ts

@@ -0,0 +1,3 @@
+export declare function dotProduct(vecA: number[], vecB: number[]): number;
+export declare function magnitude(vec: number[]): number;
+export declare function cosineSimilarity(vecA: number[], vecB: number[]): number;

package/dist/worker.d.ts

@@ -0,0 +1 @@
+import 'dotenv/config';