@devilsdev/rag-pipeline-utils 2.6.1 → 2.7.0

package/dist/types/ecosystem/plugin-installer.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @param {object} params
+ * @param {string} params.source - Registry id, path, .tgz, .js or "npm:<spec>"
+ * @param {string} params.configPath - .ragrc.json to register the plugin in
+ * @param {string} [params.stage] - Overrides package.json "ragPipeline.stage"
+ * @param {string} [params.name] - Overrides package.json "ragPipeline.name"
+ * @param {boolean} [params.force] - Replace an installed plugin with the same stage/name
+ * @param {boolean} [params.allowRisky] - Install despite high-risk scan findings
+ * @param {object} [params.hub] - PluginHub, required for registry ids
+ * @param {string} [params.version] - Registry version (default "latest")
+ * @param {function} [params.onProgress] - Called with a stage label
+ * @returns {Promise<object>} Installation summary
+ */
+export function installPlugin(params: {
+    source: string;
+    configPath: string;
+    stage?: string;
+    name?: string;
+    force?: boolean;
+    allowRisky?: boolean;
+    hub?: object;
+    version?: string;
+    onProgress?: Function;
+}): Promise<object>;
+/**
+ * Plugins registered under `plugins` in a config, with the installed
+ * package's version when it lives in .rag/plugins.
+ *
+ * @param {string} configPath
+ * @returns {Array<{stage: string, name: string, entry: string, version: string|null, installed: boolean, inPipeline: boolean, exists: boolean}>}
+ */
+export function listInstalledPlugins(configPath: string): Array<{
+    stage: string;
+    name: string;
+    entry: string;
+    version: string | null;
+    installed: boolean;
+    inPipeline: boolean;
+    exists: boolean;
+}>;

package/dist/types/ecosystem/plugin-probe.d.ts

@@ -0,0 +1,21 @@
+/**
+ * @param {string} modulePath - Plugin entry file
+ * @param {string} stage - loader | embedder | retriever | llm | reranker
+ * @param {{timeoutMs?: number}} [options]
+ * @returns {Promise<{ok: boolean, missing: string[], methods: string[], error?: string}>}
+ */
+export function probeContract(modulePath: string, stage: string, options?: {
+    timeoutMs?: number;
+}): Promise<{
+    ok: boolean;
+    missing: string[];
+    methods: string[];
+    error?: string;
+}>;
+export namespace STAGE_METHODS {
+    let loader: string[];
+    let embedder: string[];
+    let retriever: string[];
+    let llm: string[];
+    let reranker: string[];
+}

package/dist/types/ecosystem/plugin-scanner.d.ts

@@ -0,0 +1,30 @@
+/**
+ * @param {string} pluginPath - Plugin directory or single .js file
+ * @returns {{
+ *   findings: Array<{severity: string, rule: string, message: string, file: string, line?: number}>,
+ *   counts: {high: number, medium: number, low: number},
+ *   filesScanned: number
+ * }}
+ */
+export function scanPlugin(pluginPath: string): {
+    findings: Array<{
+        severity: string;
+        rule: string;
+        message: string;
+        file: string;
+        line?: number;
+    }>;
+    counts: {
+        high: number;
+        medium: number;
+        low: number;
+    };
+    filesScanned: number;
+};
+/** severity: high blocks installation unless explicitly allowed */
+export const RULES: {
+    id: string;
+    severity: string;
+    pattern: RegExp;
+    message: string;
+}[];

package/dist/types/ecosystem/tar-extract.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Extract a gzipped tarball into destDir.
+ *
+ * @param {Buffer} archive - .tgz contents
+ * @param {string} destDir - Target directory (created if missing)
+ * @param {object} [options]
+ * @param {number} [options.strip=0] - Leading path components to drop (npm packs use "package/")
+ * @param {number} [options.maxBytes=50MB] - Refuse archives that expand beyond this
+ * @returns {{files: string[], skipped: string[]}}
+ */
+export function extractTarGz(archive: Buffer, destDir: string, options?: {
+    strip?: number;
+    maxBytes?: number;
+}): {
+    files: string[];
+    skipped: string[];
+};
+/** Separator-agnostic "child is root or inside it" (paths already resolved). */
+export function isWithin(root: any, child: any): any;
+/**
+ * Path of `child` relative to `root` with "/" separators, or null when it is
+ * not inside root. Separator-agnostic like isWithin.
+ */
+export function relativeWithin(root: any, child: any): any;

package/dist/types/enterprise/audit-logging.d.ts

@@ -0,0 +1,217 @@
+/**
+ * Records authentication, data access, data change and other audit events
+ * in hash-chained logs, with querying, compliance reports and an integrity
+ * check.
+ */
+export class AuditLogger extends EventEmitter<[never]> {
+    constructor(_options?: {});
+    _config: {
+        logDir: any;
+        retention: {
+            days: any;
+            maxSizeMB: any;
+            compressionEnabled: boolean;
+        };
+        compliance: {
+            standards: any;
+            immutable: boolean;
+            encryption: boolean;
+            digitalSigning: boolean;
+        };
+        realtime: {
+            enabled: boolean;
+            batchSize: any;
+            flushInterval: any;
+            webhookUrl: any;
+        };
+        categories: {
+            authentication: {
+                level: string;
+                retention: number;
+            };
+            authorization: {
+                level: string;
+                retention: number;
+            };
+            dataAccess: {
+                level: string;
+                retention: number;
+            };
+            dataModification: {
+                level: string;
+                retention: number;
+            };
+            systemChanges: {
+                level: string;
+                retention: number;
+            };
+            userActivity: {
+                level: string;
+                retention: number;
+            };
+            apiAccess: {
+                level: string;
+                retention: number;
+            };
+            pluginActivity: {
+                level: string;
+                retention: number;
+            };
+            complianceEvents: {
+                level: string;
+                retention: number;
+            };
+        };
+    };
+    logBuffer: any[];
+    encryptionKey: string | Buffer<ArrayBufferLike>;
+    signingKey: any;
+    sequenceNumber: number;
+    /**
+     * Initialize audit logging system
+     */
+    _initializeAuditSystem(): Promise<void>;
+    flushTimer: NodeJS.Timeout;
+    /**
+     * Log authentication events
+     */
+    logAuthentication(event: any): Promise<any>;
+    /**
+     * Log authorization events
+     */
+    logAuthorization(event: any): Promise<any>;
+    /**
+     * Log data access events
+     */
+    logDataAccess(event: any): Promise<any>;
+    /**
+     * Log data modification events
+     */
+    logDataModification(event: any): Promise<any>;
+    /**
+     * Log system configuration changes
+     */
+    logSystemChanges(event: any): Promise<any>;
+    /**
+     * Log API access events
+     */
+    logAPIAccess(event: any): Promise<any>;
+    /**
+     * Log compliance events
+     */
+    logComplianceEvent(event: any): Promise<any>;
+    /**
+     * Write audit log with integrity protection
+     */
+    _writeAuditLog(auditEvent: any): Promise<any>;
+    /**
+     * Flush logs to persistent storage
+     */
+    _flushLogs(): Promise<void>;
+    /**
+     * Query audit logs
+     */
+    queryLogs(query: any): Promise<{
+        results: any[];
+        total: number;
+        hasMore: boolean;
+    }>;
+    /**
+     * Generate compliance report
+     */
+    generateComplianceReport(_options: any): Promise<{
+        metadata: {
+            _tenantId: any;
+            standard: any;
+            period: {
+                startDate: any;
+                endDate: any;
+            };
+            generatedAt: string;
+            totalEvents: number;
+        };
+        summary: {
+            authenticationEvents: number;
+            authorizationEvents: number;
+            dataAccessEvents: number;
+            dataModificationEvents: number;
+            systemChangeEvents: number;
+            complianceEvents: number;
+            criticalEvents: number;
+            failedEvents: number;
+        };
+        compliance: {
+            requirements: any[];
+            violations: any[];
+            recommendations: any[];
+        };
+        evidence: any[];
+    }>;
+    /**
+     * Verify log integrity
+     */
+    verifyIntegrity(_options?: {}): Promise<{
+        verified: boolean;
+        totalLogs: number;
+        verifiedLogs: number;
+        tamperedLogs: any[];
+        missingLogs: any[];
+        integrityBreaks: any[];
+    }>;
+    _calculateIntegrityHash(auditEvent: any): Promise<string>;
+    _encryptSensitiveFields(auditEvent: any): Promise<any>;
+    _decryptAuditEvent(auditEvent: any): Promise<any>;
+    _calculateSeverity(_category: any, _action: any, result: any): any;
+    _sanitizeQuery(query: any): any;
+    _sanitizeValue(value: any): any;
+    _sanitizeParameters(params: any): any;
+    _hashApiKey(apiKey: any): string;
+    /**
+     * Resolve the field-encryption key.
+     *
+     * Order: the `encryptionKey` option, then the RAG_AUDIT_ENCRYPTION_KEY
+     * environment variable (used as a passphrase, like a string option). A
+     * random per-process key is never generated: records written with it
+     * could not be decrypted after a restart.
+     *
+     * If encryption is enabled but no key is configured, encryption is
+     * disabled for this instance with a loud warning (an `encryption_disabled`
+     * event is also emitted on the next tick). Pass `requireEncryption: true`
+     * to throw instead.
+     *
+     * @param {Object} _options - Constructor options
+     * @returns {Buffer|string|null} Key material, or null when unavailable
+     * @private
+     */
+    private _resolveEncryptionKey;
+    /**
+     * AES-256 needs exactly 32 key bytes. A 32-byte Buffer is used as-is; any
+     * other key (e.g. a passphrase string) is reduced with SHA-256.
+     */
+    _getCipherKey(): Buffer<ArrayBufferLike>;
+    _createDecipher(encryptedField: any): crypto.DecipherGCM;
+    _generateSigningKey(): Buffer<ArrayBufferLike>;
+    _initializeIntegrityChain(): Promise<void>;
+    _updateIntegrityChain(logs: any): Promise<void>;
+    _signAuditEvent(auditEvent: any): Promise<string>;
+    _verifySignature(auditEvent: any): Promise<boolean>;
+    _getLogFilesInRange(startDate: any, endDate: any): Promise<string[]>;
+    _readLogFile(filepath: any): Promise<any[]>;
+    _matchesQuery(log: any, query: any): boolean;
+    _checkCompliance(log: any, standard: any): {
+        requirement: any;
+        violation: boolean;
+        evidence: any;
+    };
+    _sendToWebhook(logs: any): Promise<void>;
+    /**
+     * Cleanup old logs based on retention policy
+     */
+    cleanupOldLogs(): Promise<{
+        filesRemoved: number;
+        bytesFreed: number;
+        errors: any[];
+    }>;
+}
+import { EventEmitter } from "events";
+import crypto = require("crypto");

package/dist/types/evaluate/compare.d.ts

@@ -0,0 +1,140 @@
+/**
+ * Runs every case of a dataset through a pipeline and scores it.
+ *
+ * Each case runs with citations and evaluation on, and its expected answer
+ * as the reference. A case that fails is recorded with its error, not
+ * thrown. Throws a DatasetError (`code: 'RAG_INVALID_DATASET'`, `errors`
+ * from validateDataset) when the dataset is invalid.
+ *
+ * @param {Array<Object>|{cases: Array<Object>}} dataset - See validateDataset
+ * @param {{ run: Function }} pipeline - A createRagPipeline() pipeline
+ * @param {Object} [options]
+ * @param {number} [options.concurrency=1] - Cases run at the same time
+ * @param {number} [options.timeout] - Per-case timeout in milliseconds
+ * @param {AbortSignal} [options.signal] - Stops the run; unstarted cases are not run
+ * @param {Object} [options.runOptions] - Extra `run()` options for every case
+ * @param {(caseResult: Object, index: number) => void} [options.onCase] - Called as each case finishes
+ * @returns {Promise<{
+ *   version: number,
+ *   libraryVersion: string,
+ *   createdAt: string,
+ *   summary: { cases: number, succeeded: number, failed: number, metrics: Object<string, { mean: number, count: number }> },
+ *   cases: Array<{ id: string, prompt: string, success: boolean, error?: string, answer: string|null, retrievedIds: string[], scores: Object<string, number>, durationMs: number }>
+ * }>} A JSON-serialisable report
+ */
+export function runEvaluation(dataset: Array<any> | {
+    cases: Array<any>;
+}, pipeline: {
+    run: Function;
+}, options?: {
+    concurrency?: number;
+    timeout?: number;
+    signal?: AbortSignal;
+    runOptions?: any;
+    onCase?: (caseResult: any, index: number) => void;
+}): Promise<{
+    version: number;
+    libraryVersion: string;
+    createdAt: string;
+    summary: {
+        cases: number;
+        succeeded: number;
+        failed: number;
+        metrics: {
+            [x: string]: {
+                mean: number;
+                count: number;
+            };
+        };
+    };
+    cases: Array<{
+        id: string;
+        prompt: string;
+        success: boolean;
+        error?: string;
+        answer: string | null;
+        retrievedIds: string[];
+        scores: {
+            [x: string]: number;
+        };
+        durationMs: number;
+    }>;
+}>;
+/**
+ * Compares two configurations on the same dataset and reports what got
+ * worse. Each side is a pipeline (run now) or a saved runEvaluation()
+ * report, so CI can compare against the base branch's saved report.
+ *
+ * A metric has regressed when its mean drops by more than `tolerance`.
+ * A case has regressed when it now fails, or a metric drops by more than
+ * `caseTolerance`; those cases are listed but don't set `regressed` on
+ * their own, except failures. Scores from LLM answers vary between runs,
+ * so keep the tolerances above the noise you see on repeated runs.
+ *
+ * @param {Array<Object>|{cases: Array<Object>}|null} dataset - Needed when either side is a pipeline
+ * @param {Object} options
+ * @param {Object} options.baseline - Pipeline or report
+ * @param {Object} options.candidate - Pipeline or report
+ * @param {number} [options.tolerance=0.02] - Allowed drop in a metric's mean
+ * @param {number} [options.caseTolerance=0.1] - Drop in one case's metric that lists the case
+ * @param {string[]} [options.metrics] - Metrics to compare (default: all both runs have)
+ * @param {Object} [options.runOptions] - Options for runEvaluation when a side is a pipeline
+ * @returns {Promise<{
+ *   regressed: boolean,
+ *   metrics: Object<string, { baseline: number, candidate: number, delta: number, regressed: boolean }>,
+ *   regressions: Array<{ id: string, prompt: string, reason: string, metric?: string, baseline?: number, candidate?: number, delta?: number }>,
+ *   improvements: Array<{ id: string, prompt: string, reason: string, metric?: string, baseline?: number, candidate?: number, delta?: number }>,
+ *   warnings: string[],
+ *   baseline: Object,
+ *   candidate: Object
+ * }>}
+ */
+export function compareConfigurations(dataset: Array<any> | {
+    cases: Array<any>;
+} | null, options?: {
+    baseline: any;
+    candidate: any;
+    tolerance?: number;
+    caseTolerance?: number;
+    metrics?: string[];
+    runOptions?: any;
+}): Promise<{
+    regressed: boolean;
+    metrics: {
+        [x: string]: {
+            baseline: number;
+            candidate: number;
+            delta: number;
+            regressed: boolean;
+        };
+    };
+    regressions: Array<{
+        id: string;
+        prompt: string;
+        reason: string;
+        metric?: string;
+        baseline?: number;
+        candidate?: number;
+        delta?: number;
+    }>;
+    improvements: Array<{
+        id: string;
+        prompt: string;
+        reason: string;
+        metric?: string;
+        baseline?: number;
+        candidate?: number;
+        delta?: number;
+    }>;
+    warnings: string[];
+    baseline: any;
+    candidate: any;
+}>;
+export class DatasetError extends Error {
+    constructor(errors: any);
+    code: string;
+    errors: any;
+}
+/** Metrics a report can hold; higher is better for all of them. */
+declare const METRICS: string[];
+export { METRICS as EVALUATION_METRICS };

package/dist/types/evaluate/dataset.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Checks an evaluation dataset before anything runs. It never throws: every
+ * problem is reported, with the case it is in.
+ *
+ * @param {Array<Object>|{cases: Array<Object>}} dataset - Cases, or an object with `cases`
+ * @param {Object} [options]
+ * @param {boolean} [options.requireExpected=false] - Every case needs `expected`
+ * @param {boolean} [options.requireSourceIds=false] - Every case needs `sourceIds`
+ * @returns {{
+ *   valid: boolean,
+ *   errors: Array<{ index: number|null, id: string|null, field: string|null, message: string }>,
+ *   warnings: Array<{ index: number|null, id: string|null, field: string|null, message: string }>,
+ *   cases: Array<{ id: string, prompt: string, expected?: string, sourceIds?: string[], tags?: string[] }>
+ * }} `cases` holds the normalised cases (aliases resolved, ids filled in)
+ *   when `valid` is true, and is empty otherwise
+ */
+export function validateDataset(dataset: Array<any> | {
+    cases: Array<any>;
+}, options?: {
+    requireExpected?: boolean;
+    requireSourceIds?: boolean;
+}): {
+    valid: boolean;
+    errors: Array<{
+        index: number | null;
+        id: string | null;
+        field: string | null;
+        message: string;
+    }>;
+    warnings: Array<{
+        index: number | null;
+        id: string | null;
+        field: string | null;
+        message: string;
+    }>;
+    cases: Array<{
+        id: string;
+        prompt: string;
+        expected?: string;
+        sourceIds?: string[];
+        tags?: string[];
+    }>;
+};

package/dist/types/evaluate/evaluator.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Evaluate a list of prompt/answer pairs using the RAG pipeline and compute scores
+ * @param {string} datasetPath - Path to JSON file with { prompt, expected }[]
+ * @param {object} _config - RAG plugin _config (loader optional)
+ * @returns {Promise<Array<{ prompt: string, expected: string, actual: string, success: boolean, scores: object }>>}
+ */
+export function evaluateRagDataset(_datasetPath: any, _config: object): Promise<Array<{
+    prompt: string;
+    expected: string;
+    actual: string;
+    success: boolean;
+    scores: object;
+}>>;
+export function normalizeText(txt: any): any;

package/dist/types/evaluate/report.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Decides whether a run passes: no case failed, every metric in `min` is at
+ * or above its minimum, and the comparison (if any) found no regression.
+ *
+ * @param {Object} input
+ * @param {Object} input.report - A runEvaluation() report
+ * @param {Object} [input.comparison] - A compareConfigurations() result
+ * @param {Object<string, number>} [input.min] - Minimum mean per metric, e.g. `{ faithfulness: 0.8 }`
+ * @returns {{
+ *   passed: boolean,
+ *   reasons: string[],
+ *   thresholds: Array<{ metric: string, min: number, actual: number|null, passed: boolean }>
+ * }}
+ */
+export function evaluationVerdict({ report, comparison, min }?: {
+    report: any;
+    comparison?: any;
+    min?: {
+        [x: string]: number;
+    };
+}): {
+    passed: boolean;
+    reasons: string[];
+    thresholds: Array<{
+        metric: string;
+        min: number;
+        actual: number | null;
+        passed: boolean;
+    }>;
+};
+/**
+ * Renders an evaluation run, and its comparison with a baseline if given,
+ * as Markdown (for a pull request comment) or a standalone HTML page.
+ *
+ * @param {Object} input
+ * @param {Object} input.report - A runEvaluation() report
+ * @param {Object} [input.comparison] - A compareConfigurations() result
+ * @param {Object<string, number>} [input.min] - Minimum mean per metric
+ * @param {Object} [options]
+ * @param {'markdown'|'html'} [options.format='markdown']
+ * @param {string} [options.title='RAG evaluation']
+ * @returns {string}
+ */
+export function renderEvaluationReport(input: {
+    report: any;
+    comparison?: any;
+    min?: {
+        [x: string]: number;
+    };
+}, options?: {
+    format?: "markdown" | "html";
+    title?: string;
+}): string;
+/** Marks the comment so the Action can find and update it. */
+declare const MARKER: "<!-- rag-eval-report -->";
+export { MARKER as REPORT_MARKER };

package/dist/types/evaluate/synthetic.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Generates question and answer pairs from documents with an LLM.
+ *
+ * Each document gets its own request, with that document as the only
+ * context, so every case knows its source. Generated answers are checked
+ * against the document with computeFaithfulness, and pairs whose answer the
+ * document doesn't support, malformed pairs and repeated questions are
+ * dropped and listed in `skipped`.
+ *
+ * @param {Array<{id?: string, content: string, metadata?: Object}|string>} documents - The corpus, e.g. the chunks you index
+ * @param {Object} options
+ * @param {{ generate: Function }} options.llm - Any llm stage: `generate(query, docs, options)` returning a string
+ * @param {number} [options.count=20] - Cases to generate at most
+ * @param {number} [options.perDocument=1] - Questions to ask per document
+ * @param {number} [options.minFaithfulness=1] - Share of an answer's sentences the document must support
+ * @param {number} [options.minLength=80] - Documents shorter than this many characters are skipped
+ * @param {string} [options.prompt] - Replaces the instruction; `{count}` is replaced by perDocument
+ * @param {number} [options.concurrency=1] - Documents processed at the same time
+ * @param {Object} [options.generateOptions] - Extra options for every `llm.generate()` call, e.g. `{ temperature: 0.7 }`
+ * @param {AbortSignal} [options.signal]
+ * @param {(testCase: Object) => void} [options.onCase] - Called for each case kept
+ * @returns {Promise<{
+ *   cases: Array<{ id: string, prompt: string, expected: string, sourceIds: string[], tags: string[], reviewed: boolean }>,
+ *   skipped: Array<{ sourceId: string, reason: string, question?: string }>
+ * }>} A dataset (`{ cases }`) plus what was dropped and why
+ */
+export function generateSyntheticDataset(documents: Array<{
+    id?: string;
+    content: string;
+    metadata?: any;
+} | string>, options?: {
+    llm: {
+        generate: Function;
+    };
+    count?: number;
+    perDocument?: number;
+    minFaithfulness?: number;
+    minLength?: number;
+    prompt?: string;
+    concurrency?: number;
+    generateOptions?: any;
+    signal?: AbortSignal;
+    onCase?: (testCase: any) => void;
+}): Promise<{
+    cases: Array<{
+        id: string;
+        prompt: string;
+        expected: string;
+        sourceIds: string[];
+        tags: string[];
+        reviewed: boolean;
+    }>;
+    skipped: Array<{
+        sourceId: string;
+        reason: string;
+        question?: string;
+    }>;
+}>;
+/**
+ * Pulls the JSON array of pairs out of a model reply. Accepts a bare array,
+ * one inside a ```json fence, or one surrounded by prose; returns null when
+ * there is none.
+ */
+export function parsePairs(reply: any): any;

package/dist/types/loader/csv-loader.d.ts

@@ -0,0 +1,10 @@
+export class CSVLoader {
+    /**
+     * Load and parse a CSV file into chunks
+     * @param {string} filePath
+     * @returns {Promise<Array<{ chunk(): string[] }>>}
+     */
+    load(filePath: string): Promise<Array<{
+        chunk(): string[];
+    }>>;
+}

package/dist/types/loader/document-reader.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Read a file or every supported file under a directory and return its
+ * chunks together with the file they came from. Unlike DirectoryLoader this
+ * keeps the source path, which ingestion needs for stable chunk ids,
+ * re-ingestion and citations.
+ *
+ * @param {string} inputPath - File or directory
+ * @param {object} [options]
+ * @param {string} [options.baseDir=process.cwd()] - Sources are reported relative to this
+ * @returns {Promise<Array<{source: string, chunks: string[]}>>}
+ */
+export function readDocuments(inputPath: string, options?: {
+    baseDir?: string;
+}): Promise<Array<{
+    source: string;
+    chunks: string[];
+}>>;
+export const SUPPORTED_EXTENSIONS: string[];