@devilsdev/rag-pipeline-utils 2.4.4 → 2.5.0

package/dist/types/enterprise/sso-integration.d.ts

@@ -0,0 +1,420 @@
+export class SSOManager extends EventEmitter<[never]> {
+    constructor(_options?: {});
+    _config: any;
+    sessions: Map<any, any>;
+    providers: Map<any, any>;
+    userCache: Map<any, any>;
+    /**
+     * Build the full SSO configuration from user-supplied options.
+     * @param {Object} _options - User-supplied configuration options
+     * @returns {Object} Complete SSO configuration
+     * @private
+     */
+    private _buildConfig;
+    /**
+     * Create and configure the rate limiter for login attempt throttling.
+     * @private
+     */
+    private _initializeRateLimiter;
+    rateLimiter: RateLimiter;
+    /**
+     * Create and configure the hardened JWT validator for token operations.
+     * @param {Object} _options - Original user-supplied options for JWT settings
+     * @private
+     */
+    private _initializeJWTValidator;
+    jwtValidator: JWTValidator;
+    /**
+     * Forward JWT validator events to the SSOManager for audit logging.
+     * @private
+     */
+    private _wireJWTEvents;
+    /**
+     * Initialize SSO providers
+     */
+    _initializeProviders(): void;
+    /**
+     * Initiate SSO login
+     */
+    initiateLogin(tenantId: any, provider: any, _redirectUrl: any): Promise<{
+        authUrl: any;
+        state: `${string}-${string}-${string}-${string}-${string}`;
+        expiresAt: number;
+    }>;
+    /**
+     * Handle SSO callback
+     */
+    handleCallback(provider: any, _callbackData: any): Promise<{
+        session: {
+            id: `${string}-${string}-${string}-${string}-${string}`;
+            tenantId: any;
+            userId: any;
+            provider: any;
+            user: {
+                id: any;
+                email: any;
+                name: any;
+                roles: any;
+                groups: any;
+                attributes: any;
+            };
+            tokens: {
+                _accessToken: string;
+                refreshToken: string;
+                externalTokens: {
+                    accessToken: any;
+                    refreshToken: any;
+                    expiresAt: any;
+                };
+            };
+            metadata: {
+                createdAt: number;
+                lastActivity: number;
+                expiresAt: any;
+                ipAddress: any;
+                userAgent: any;
+            };
+        };
+        user: any;
+        redirectUrl: any;
+    }>;
+    /**
+     * Create user session with JWT tokens
+     */
+    _createUserSession(tenantId: any, userInfo: any, provider: any, tokenData: any): Promise<{
+        id: `${string}-${string}-${string}-${string}-${string}`;
+        tenantId: any;
+        userId: any;
+        provider: any;
+        user: {
+            id: any;
+            email: any;
+            name: any;
+            roles: any;
+            groups: any;
+            attributes: any;
+        };
+        tokens: {
+            _accessToken: string;
+            refreshToken: string;
+            externalTokens: {
+                accessToken: any;
+                refreshToken: any;
+                expiresAt: any;
+            };
+        };
+        metadata: {
+            createdAt: number;
+            lastActivity: number;
+            expiresAt: any;
+            ipAddress: any;
+            userAgent: any;
+        };
+    }>;
+    /**
+     * Validate and refresh session
+     */
+    validateSession(sessionId: any, tenantId: any): Promise<any>;
+    /**
+     * Renew session tokens
+     */
+    _renewSession(session: any): Promise<void>;
+    /**
+     * Logout user and invalidate session
+     */
+    logout(sessionId: any, tenantId: any, _options?: {}): Promise<{
+        success: boolean;
+        reason: string;
+    } | {
+        success: boolean;
+        reason?: undefined;
+    }>;
+    /**
+     * Get user information from session
+     */
+    getUser(sessionId: any, tenantId: any): Promise<any>;
+    /**
+     * List active sessions for a tenant
+     */
+    getActiveSessions(tenantId: any, _options?: {}): Promise<{
+        id: any;
+        userId: any;
+        provider: any;
+        user: any;
+        metadata: any;
+    }[]>;
+    /**
+     * Revoke all sessions for a user
+     */
+    revokeUserSessions(tenantId: any, userId: any, excludeSessionId?: any): Promise<{
+        revokedSessions: number;
+    }>;
+    /**
+     * Generate JWT token using hardened JWT validator
+     * @param {object} payload - JWT payload
+     * @returns {string} Signed JWT token
+     */
+    _generateJWT(payload: object): string;
+    /**
+     * Verify and decode JWT token using hardened validator
+     * @param {string} token - JWT token to verify
+     * @returns {object} Decoded payload
+     */
+    _verifyJWT(token: string): object;
+    /**
+     * Generate refresh token
+     */
+    _generateRefreshToken(): string;
+    /**
+     * Cleanup expired sessions
+     */
+    cleanupExpiredSessions(): Promise<{
+        cleanedSessions: number;
+    }>;
+    /**
+     * Reset rate limit for a specific tenant/provider
+     * @param {string} tenantId - Tenant identifier
+     * @param {string} provider - Provider name
+     */
+    resetRateLimit(tenantId: string, provider: string): void;
+    /**
+     * Get rate limit status for a tenant/provider
+     * @param {string} tenantId - Tenant identifier
+     * @param {string} provider - Provider name
+     * @returns {Object} Rate limit status
+     */
+    getRateLimitStatus(tenantId: string, provider: string): any;
+    /**
+     * Get rate limiter statistics
+     * @returns {Object} Statistics about rate limiting
+     */
+    getRateLimiterStats(): any;
+    /**
+     * Get JWT validator statistics
+     * @returns {Object} JWT validation statistics
+     */
+    getJWTStats(): any;
+    /**
+     * Destroy SSO manager and cleanup resources
+     */
+    destroy(): void;
+}
+export class SAMLProvider {
+    /**
+     * SAML 2.0 provider.
+     *
+     * Real SAML authentication is NOT supported: verifying SAML assertions
+     * requires XML canonicalization and XML-signature verification, which
+     * must come from a vetted library rather than hand-written code. Every
+     * authentication method throws unless the explicit development bypass is
+     * active (devBypass: true and NODE_ENV !== 'production').
+     *
+     * @param {Object} _config - Provider configuration
+     */
+    constructor(_config?: any);
+    _config: any;
+    getAuthorizationUrl(state: any, _redirectUrl: any): Promise<string>;
+    exchangeCodeForTokens(_callbackData: any): Promise<{
+        _accessToken: `${string}-${string}-${string}-${string}-${string}`;
+        assertion: {
+            sessionIndex: `${string}-${string}-${string}-${string}-${string}`;
+            nameID: string;
+            attributes: {
+                email: string;
+                displayName: string;
+                roles: string[];
+            };
+            notOnOrAfter: number;
+        };
+        refreshToken: any;
+        expiresAt: number;
+    }>;
+    getUserInfo(_accessToken: any, tokenData?: {}): Promise<{
+        id: any;
+        email: any;
+        name: any;
+        roles: any;
+        groups: any;
+    }>;
+    _buildSAMLRequest(state: any): string;
+    _validateSAMLResponse(response: any): {
+        sessionIndex: `${string}-${string}-${string}-${string}-${string}`;
+        nameID: string;
+        attributes: {
+            email: string;
+            displayName: string;
+            roles: string[];
+        };
+        notOnOrAfter: number;
+    };
+}
+export class OAuth2Provider {
+    /**
+     * OAuth 2.0 authorization-code provider with PKCE (S256).
+     *
+     * @param {Object} _config - clientId, clientSecret (optional for public
+     *   clients), authorizationUrl, tokenUrl, userInfoUrl, redirectUri,
+     *   scopes, devBypass
+     */
+    constructor(_config?: any);
+    _config: any;
+    /**
+     * @param {string} state - Opaque state bound to the login request
+     * @param {string} [_redirectUrl] - Fallback redirect_uri when none is configured
+     * @param {Object} [loginContext] - { codeChallenge, redirectUri } from SSOManager
+     */
+    getAuthorizationUrl(state: string, _redirectUrl?: string, loginContext?: any): Promise<string>;
+    /**
+     * Exchange the authorization code. redirect_uri is taken from provider
+     * configuration (or the server-side login request), never from
+     * callbackData, and the PKCE code_verifier from the login request.
+     *
+     * @param {Object} callbackData - { code, state }
+     * @param {Object} [loginContext] - { codeVerifier, redirectUri }
+     */
+    exchangeCodeForTokens(callbackData: any, loginContext?: any): Promise<{
+        _accessToken: string;
+        refreshToken: string;
+        expiresAt: number;
+        tokenType?: undefined;
+    } | {
+        _accessToken: any;
+        refreshToken: any;
+        expiresAt: number;
+        tokenType: any;
+    }>;
+    getUserInfo(accessToken: any): Promise<{
+        id: string;
+        email: string;
+        name: string;
+        roles: string[];
+        attributes?: undefined;
+    } | {
+        id: any;
+        email: any;
+        name: any;
+        roles: any;
+        attributes: any;
+    }>;
+    refreshTokens(refreshToken: any): Promise<{
+        _accessToken: any;
+        refreshToken: any;
+        expiresAt: number;
+    }>;
+}
+export class ActiveDirectoryProvider {
+    /**
+     * Active Directory provider. Only the development bypass is implemented;
+     * for AD FS / Entra ID use the OIDC provider, which both support.
+     * @param {Object} _config - Provider configuration
+     */
+    constructor(_config?: any);
+    _config: any;
+    getAuthorizationUrl(state: any, _redirectUrl: any, loginContext?: {}): Promise<string>;
+    exchangeCodeForTokens(callbackData: any): Promise<{
+        _accessToken: string;
+        refreshToken: string;
+        expiresAt: number;
+    }>;
+    getUserInfo(_accessToken: any): Promise<{
+        id: string;
+        email: string;
+        name: string;
+        roles: string[];
+        groups: string[];
+    }>;
+}
+export class OIDCProvider {
+    /**
+     * OpenID Connect provider (authorization code + PKCE, validated id_token).
+     *
+     * Endpoints and keys come from discovery
+     * (`<issuer>/.well-known/openid-configuration`) unless configured:
+     * authorizationUrl, tokenUrl, userInfoUrl, jwksUri, or `jwks` (a JWK Set
+     * `{ keys: [...] }` or an array of JWKs) to avoid fetching keys.
+     *
+     * The id_token must be signed by a key from the provider JWKS (selected
+     * by `kid`) with an asymmetric algorithm (`idTokenAlgorithms`, default
+     * RS/PS/ES 256-512), and its iss, aud (clientId), exp and nonce are
+     * checked.
+     *
+     * @param {Object} _config - issuer, clientId, clientSecret, redirectUri,
+     *   scopes, jwks, jwksUri, authorizationUrl, tokenUrl, userInfoUrl,
+     *   idTokenAlgorithms, clockTolerance (seconds, default 60), devBypass
+     */
+    constructor(_config?: any);
+    _config: any;
+    _metadata: {};
+    _jwks: any;
+    /**
+     * Resolve provider metadata (configured values override discovery).
+     * @returns {Promise<Object>}
+     */
+    getMetadata(): Promise<any>;
+    getAuthorizationUrl(state: any, _redirectUrl: any, loginContext?: {}): Promise<string>;
+    /**
+     * Exchange the code and validate the returned id_token.
+     * @param {Object} callbackData - { code, state }
+     * @param {Object} [loginContext] - { codeVerifier, nonce, redirectUri }
+     */
+    exchangeCodeForTokens(callbackData: any, loginContext?: any): Promise<{
+        _accessToken: string;
+        refreshToken: string;
+        idToken: string;
+        expiresAt: number;
+        claims?: undefined;
+    } | {
+        _accessToken: any;
+        refreshToken: any;
+        idToken: any;
+        claims: any;
+        expiresAt: number;
+    }>;
+    /**
+     * Validate an id_token: signature (provider JWKS), iss, aud, exp, nonce.
+     * @param {string} idToken - Compact JWS
+     * @param {Object} options - { nonce } expected nonce (required)
+     * @returns {Promise<Object>} Verified claims
+     */
+    validateIdToken(idToken: string, { nonce }?: any): Promise<any>;
+    /**
+     * Pick the verification key for a token header from the JWKS.
+     * Refetches the JWKS once when the kid is unknown (key rotation).
+     * @private
+     */
+    private _selectJwk;
+    /** @private */
+    private _getJwks;
+    /**
+     * @param {string} accessToken - Access token for the userinfo endpoint
+     * @param {Object} [tokenData] - Result of exchangeCodeForTokens; when it
+     *   carries validated id_token claims, the userinfo `sub` must match.
+     */
+    getUserInfo(accessToken: string, tokenData?: any): Promise<{
+        id: string;
+        email: string;
+        name: string;
+        roles: string[];
+        groups?: undefined;
+        attributes?: undefined;
+    } | {
+        id: any;
+        email: any;
+        name: any;
+        roles: any;
+        groups: any;
+        attributes: any;
+    }>;
+    _generateMockIdToken(): string;
+}
+/**
+ * Create a PKCE (RFC 7636, S256) verifier/challenge pair.
+ * @returns {{codeVerifier: string, codeChallenge: string}}
+ */
+export function createPkcePair(): {
+    codeVerifier: string;
+    codeChallenge: string;
+};
+import { EventEmitter } from "events";
+import RateLimiter = require("../utils/rate-limiter");
+import { JWTValidator } from "../security/jwt-validator";

package/dist/types/evaluate/context-metrics.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Compute context precision: the fraction of retrieved documents that are relevant to the query.
+ *
+ * @param {string} query - The user query
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.2] - Minimum share of query content terms a doc must contain to be considered relevant
+ * @returns {{ score: number, relevantDocs: number, totalDocs: number, perDocScores: Array<{ index: number, score: number, isRelevant: boolean }> }}
+ */
+export function computeContextPrecision(query: string, retrievedDocs: Array<{
+    content: string;
+}>, options?: {
+    threshold?: number;
+}): {
+    score: number;
+    relevantDocs: number;
+    totalDocs: number;
+    perDocScores: Array<{
+        index: number;
+        score: number;
+        isRelevant: boolean;
+    }>;
+};
+/**
+ * Compute context recall: the fraction of the claims in a REFERENCE
+ * (ground-truth) answer that the retrieved documents support. It measures
+ * whether retrieval found everything needed to answer correctly; pass the
+ * expected answer, not the generated one (scoring the generated answer
+ * against the documents is faithfulness).
+ *
+ * A claim is one sentence of the reference answer. It is supported when some
+ * document scores at least `threshold` with computeSupportScore, including
+ * the numeric consistency check.
+ *
+ * @param {string} referenceAnswer - The reference (ground-truth) answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.5] - Minimum share of a claim's content terms a doc must contain to support it
+ * @returns {{ score: number, supportedClaims: number, totalClaims: number, details: Array<{ claim: string, score: number, isSupported: boolean, bestDocIndex: number }> }}
+ */
+export function computeContextRecall(referenceAnswer: string, retrievedDocs: Array<{
+    content: string;
+}>, options?: {
+    threshold?: number;
+}): {
+    score: number;
+    supportedClaims: number;
+    totalClaims: number;
+    details: Array<{
+        claim: string;
+        score: number;
+        isSupported: boolean;
+        bestDocIndex: number;
+    }>;
+};

package/dist/types/evaluate/faithfulness.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Compute faithfulness score for an answer against retrieved documents.
+ * Measures how well each sentence in the answer is supported by the retrieved context.
+ *
+ * @param {string} answer - The generated answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents with content
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.threshold=0.5] - Minimum similarity score to consider a sentence faithful
+ * @returns {{ score: number, faithfulSentences: number, totalSentences: number, details: Array<{ sentence: string, score: number, isFaithful: boolean, bestSource: number }> }}
+ */
+export function computeFaithfulness(answer: string, retrievedDocs: Array<{
+    content: string;
+}>, options?: {
+    threshold?: number;
+}): {
+    score: number;
+    faithfulSentences: number;
+    totalSentences: number;
+    details: Array<{
+        sentence: string;
+        score: number;
+        isFaithful: boolean;
+        bestSource: number;
+    }>;
+};
+/**
+ * Derive faithfulness score from an existing citation result.
+ * Shortcut that reuses citation analysis data rather than recomputing.
+ *
+ * @param {object} citationResult - Result from citation analysis
+ * @param {number} citationResult.groundednessScore - Pre-computed groundedness score
+ * @param {Array} [citationResult.citations] - Array of citation details
+ * @returns {{ score: number, details: Array }}
+ */
+export function computeFaithfulnessFromCitations(citationResult: {
+    groundednessScore: number;
+    citations?: any[];
+}): {
+    score: number;
+    details: any[];
+};

package/dist/types/evaluate/groundedness.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Compute a composite groundedness score from faithfulness, context precision
+ * and (when a reference answer is given) context recall.
+ *
+ * Formula: a weighted mean over the components that are available,
+ *
+ *   score = sum(w_i * m_i) / sum(w_i)
+ *
+ * With a reference answer and the default weights this is
+ *   0.4 * faithfulness + 0.3 * contextPrecision + 0.3 * contextRecall.
+ * Without one, context recall is left out and the remaining weights are
+ * renormalised:
+ *   (0.4 * faithfulness + 0.3 * contextPrecision) / 0.7.
+ * If every available weight is 0 the score is 0.
+ *
+ * @param {string} query - The user query
+ * @param {string} answer - The generated answer text
+ * @param {Array<{content: string}>} retrievedDocs - Array of retrieved documents
+ * @param {object} [options={}] - Configuration options
+ * @param {{ faithfulness: number, contextPrecision: number, contextRecall: number }} [options.weights] - Custom weights for each component
+ * @param {number} [options.threshold=0.5] - Similarity threshold passed to component metrics
+ * @param {string} [options.referenceAnswer] - Reference (ground-truth) answer used for context recall
+ * @returns {{ score: number, components: { faithfulness: object, contextPrecision: object, contextRecall: object }, weights: { faithfulness: number, contextPrecision: number, contextRecall: number } }}
+ *   `components.contextRecall` is `{ score: null, skipped: true, reason }`
+ *   when no reference answer was given.
+ */
+export function computeGroundedness(query: string, answer: string, retrievedDocs: Array<{
+    content: string;
+}>, options?: {
+    weights?: {
+        faithfulness: number;
+        contextPrecision: number;
+        contextRecall: number;
+    };
+    threshold?: number;
+    referenceAnswer?: string;
+}): {
+    score: number;
+    components: {
+        faithfulness: object;
+        contextPrecision: object;
+        contextRecall: object;
+    };
+    weights: {
+        faithfulness: number;
+        contextPrecision: number;
+        contextRecall: number;
+    };
+};
+/**
+ * Default weights for the groundedness composite score.
+ * @type {{ faithfulness: number, contextPrecision: number, contextRecall: number }}
+ */
+export const DEFAULT_WEIGHTS: {
+    faithfulness: number;
+    contextPrecision: number;
+    contextRecall: number;
+};
+/** Reason reported when context recall cannot be computed. */
+export const NO_REFERENCE_REASON: string;

package/dist/types/evaluate/pipeline-evaluator.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Evaluates RAG pipeline outputs using multiple quality metrics.
+ * Emits an 'evaluated' event after each evaluation with the full result.
+ *
+ * @extends EventEmitter
+ *
+ * @example
+ * const evaluator = new PipelineEvaluator({ metrics: ['faithfulness', 'relevance'] });
+ * evaluator.on('evaluated', (result) => console.log(result.scores));
+ * const result = evaluator.evaluate({ query: 'What is RAG?', answer: '...', results: [...] });
+ */
+export class PipelineEvaluator extends EventEmitter<[never]> {
+    /**
+     * Create a new PipelineEvaluator.
+     * @param {object} [options={}] - Configuration options
+     * @param {string[]} [options.metrics] - List of metrics to compute
+     * @param {number} [options.threshold] - Similarity threshold for metrics
+     * @param {boolean} [options.includeBLEU] - Whether to include BLEU score when reference answer is provided
+     * @param {boolean} [options.includeROUGE] - Whether to include ROUGE score when reference answer is provided
+     */
+    constructor(options?: {
+        metrics?: string[];
+        threshold?: number;
+        includeBLEU?: boolean;
+        includeROUGE?: boolean;
+    });
+    config: {
+        metrics: string[];
+        threshold: number;
+        includeBLEU: boolean;
+        includeROUGE: boolean;
+    };
+    /**
+     * Evaluate a pipeline result across all configured metrics.
+     *
+     * @param {object} pipelineResult - The pipeline output to evaluate
+     * @param {string} pipelineResult.query - The original user query
+     * @param {string} pipelineResult.answer - The generated answer
+     * @param {Array<{content: string}>} pipelineResult.results - Retrieved documents
+     * @param {object} [options={}] - Additional evaluation options
+     * @param {object} [options.citationResult] - Pre-computed citation result for faithfulness shortcut
+     * @param {string} [options.referenceAnswer] - Reference (ground-truth)
+     *   answer. Required for context recall (how much of it the retrieved
+     *   documents support) and used for BLEU/ROUGE. Without it, contextRecall is
+     *   left out of `scores`, `details.contextRecall` is
+     *   `{ score: null, skipped: true, reason }`, it is listed in
+     *   `metadata.metricsSkipped`, and groundedness renormalises over
+     *   faithfulness and context precision.
+     * @returns {{ scores: Object<string, number>, details: Object<string, object>, metadata: { metricsComputed: string[], metricsSkipped: Array<{metric: string, reason: string}>, threshold: number, timestamp: string } }}
+     */
+    evaluate(pipelineResult: {
+        query: string;
+        answer: string;
+        results: Array<{
+            content: string;
+        }>;
+    }, options?: {
+        citationResult?: object;
+        referenceAnswer?: string;
+    }): {
+        scores: {
+            [x: string]: number;
+        };
+        details: {
+            [x: string]: object;
+        };
+        metadata: {
+            metricsComputed: string[];
+            metricsSkipped: Array<{
+                metric: string;
+                reason: string;
+            }>;
+            threshold: number;
+            timestamp: string;
+        };
+    };
+}
+/**
+ * Default configuration for PipelineEvaluator.
+ * @type {{ metrics: string[], threshold: number, includeBLEU: boolean, includeROUGE: boolean }}
+ */
+export const DEFAULT_CONFIG: {
+    metrics: string[];
+    threshold: number;
+    includeBLEU: boolean;
+    includeROUGE: boolean;
+};
+import { EventEmitter } from "events";

package/dist/types/evaluate/relevance.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Compute answer relevance score measuring how well the answer addresses the query.
+ * Combines Jaccard similarity with query token coverage for a balanced metric.
+ *
+ * @param {string} query - The user query
+ * @param {string} answer - The generated answer text
+ * @param {object} [options={}] - Configuration options
+ * @param {number} [options.jaccardWeight=0.5] - Weight for Jaccard similarity component
+ * @param {number} [options.coverageWeight=0.5] - Weight for query token coverage component
+ * @returns {{ score: number, jaccardSimilarity: number, queryTokensCovered: number, totalQueryTokens: number, missingQueryTerms: string[] }}
+ */
+export function computeAnswerRelevance(query: string, answer: string, options?: {
+    jaccardWeight?: number;
+    coverageWeight?: number;
+}): {
+    score: number;
+    jaccardSimilarity: number;
+    queryTokensCovered: number;
+    totalQueryTokens: number;
+    missingQueryTerms: string[];
+};