contradiction-mcp 0.1.0

package/dist/connectors/syncService.js

@@ -0,0 +1,176 @@
+import { logger } from '../utils/logger.js';
+export class SyncService {
+    dbManager;
+    registry;
+    discoveryService;
+    constructor(dbManager, registry, discoveryService) {
+        this.dbManager = dbManager;
+        this.registry = registry;
+        this.discoveryService = discoveryService;
+    }
+    /**
+     * Synchronizes an external source using its registered connector,
+     * extracts and normalizes claims, idempotently upserts them in SQLite,
+     * and optionally runs incremental contradiction discovery.
+     */
+    async syncSource(connectorType, input, options) {
+        const startTime = performance.now();
+        logger.info('Starting external source synchronization', {
+            connector: connectorType,
+            input,
+        });
+        const connector = this.registry.getOrThrow(connectorType);
+        // 1. Fetch raw data from external source
+        const fetchResult = await connector.fetch(input);
+        // 2. Extract and normalize claims
+        const extractedClaims = await connector.extractClaims(fetchResult);
+        // 3. Upsert Source entity
+        const { source } = this.dbManager.upsertSource({
+            externalId: fetchResult.sourceIdentifier,
+            type: connector.type,
+            name: fetchResult.sourceName,
+            uri: fetchResult.sourceUri,
+            lastFetchedAt: fetchResult.fetchedAt,
+            trustScore: 1.0,
+            metadata: fetchResult.metadata,
+        });
+        // 4. Idempotently upsert claims without creating duplicates
+        let claimsCreated = 0;
+        let claimsUpdated = 0;
+        const touchedClaimIds = [];
+        for (const claim of extractedClaims) {
+            const res = this.dbManager.upsertClaim({
+                externalId: claim.externalId,
+                sourceId: source.id,
+                subject: claim.subject,
+                predicate: claim.predicate,
+                value: claim.value,
+                valueType: claim.valueType,
+                normalizedValue: claim.normalizedValue,
+                confidence: claim.confidence ?? 1.0,
+                environment: claim.environment,
+                scope: claim.scope,
+                sourceRole: claim.sourceRole,
+                isHistorical: claim.isHistorical ?? false,
+                validFrom: claim.validFrom,
+                validUntil: claim.validUntil,
+                valueConstraint: claim.valueConstraint,
+                multiValueContext: claim.multiValueContext,
+                observedAt: claim.observedAt ?? new Date(),
+                metadata: {
+                    ...claim.provenance,
+                },
+            });
+            if (res.isNew) {
+                claimsCreated++;
+                touchedClaimIds.push(res.claim.id);
+            }
+            else if (res.updated) {
+                claimsUpdated++;
+                touchedClaimIds.push(res.claim.id);
+            }
+        }
+        // 5. Run incremental contradiction discovery for touched claims
+        let contradictionsFound = 0;
+        let newContradictions = 0;
+        const runDiscovery = options?.runDiscoveryAfterSync ?? true;
+        if (runDiscovery && this.discoveryService && touchedClaimIds.length > 0) {
+            logger.info('Executing incremental contradiction discovery for synced claims', {
+                touchedCount: touchedClaimIds.length,
+            });
+            for (const claimId of touchedClaimIds) {
+                try {
+                    const scanRes = this.discoveryService.scanClaim(claimId, {
+                        minConfidence: options?.minConfidence ?? 0.5,
+                    });
+                    contradictionsFound += scanRes.contradictionsFound;
+                    newContradictions += scanRes.newContradictions;
+                }
+                catch (err) {
+                    logger.warn('Failed incremental scan for claim', { claimId, error: String(err) });
+                }
+            }
+        }
+        const durationMs = Math.round(performance.now() - startTime);
+        const summary = {
+            status: 'completed',
+            source: {
+                id: source.id,
+                type: source.type,
+                name: source.name,
+                externalId: source.externalId || fetchResult.sourceIdentifier,
+            },
+            repository: source.name,
+            filesInspected: Number(fetchResult.metadata.filesCount || 0),
+            claimsExtracted: extractedClaims.length,
+            claimsCreated,
+            claimsUpdated,
+            durationMs,
+            contradictionsFound,
+            newContradictions,
+        };
+        logger.info('External source synchronization complete', {
+            repository: summary.repository,
+            claimsCreated: summary.claimsCreated,
+            claimsUpdated: summary.claimsUpdated,
+            durationMs: summary.durationMs,
+        });
+        return summary;
+    }
+    /**
+     * Synchronizes multiple external sources with bounded concurrency and failure isolation.
+     */
+    async syncSources(requests, options) {
+        const startTime = performance.now();
+        const concurrency = Math.max(1, Math.min(10, options?.concurrency ?? 3));
+        const results = [];
+        let succeeded = 0;
+        let failed = 0;
+        let totalClaimsCreated = 0;
+        let totalClaimsUpdated = 0;
+        let totalContradictionsFound = 0;
+        // Execute with worker queue for bounded concurrency
+        let index = 0;
+        const executeWorker = async () => {
+            while (index < requests.length) {
+                const currentIdx = index++;
+                const req = requests[currentIdx];
+                try {
+                    const summary = await this.syncSource(req.connector, req.input, options);
+                    results[currentIdx] = {
+                        connector: req.connector,
+                        status: 'completed',
+                        summary,
+                    };
+                    succeeded++;
+                    totalClaimsCreated += summary.claimsCreated;
+                    totalClaimsUpdated += summary.claimsUpdated;
+                    totalContradictionsFound += summary.contradictionsFound ?? 0;
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    logger.error('Failed to sync source in batch', { connector: req.connector, error: msg });
+                    results[currentIdx] = {
+                        connector: req.connector,
+                        status: 'failed',
+                        error: msg,
+                    };
+                    failed++;
+                }
+            }
+        };
+        const workers = Array.from({ length: Math.min(concurrency, requests.length) }, () => executeWorker());
+        await Promise.all(workers);
+        return {
+            sourcesRequested: requests.length,
+            sourcesSucceeded: succeeded,
+            sourcesFailed: failed,
+            claimsCreated: totalClaimsCreated,
+            claimsUpdated: totalClaimsUpdated,
+            contradictionsFound: totalContradictionsFound,
+            durationMs: Math.round(performance.now() - startTime),
+            results,
+        };
+    }
+}
+//# sourceMappingURL=syncService.js.map

package/dist/connectors/syncService.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"syncService.js","sourceRoot":"","sources":["../../src/connectors/syncService.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AA0B5C,MAAM,OAAO,WAAW;IACL,SAAS,CAAkB;IAC3B,QAAQ,CAAoB;IAC5B,gBAAgB,CAAoB;IAErD,YACE,SAA0B,EAC1B,QAA2B,EAC3B,gBAAmC;QAEnC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAC3C,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU,CACrB,aAAqB,EACrB,KAAc,EACd,OAAqB;QAErB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE;YACtD,SAAS,EAAE,aAAa;YACxB,KAAK;SACN,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAE1D,yCAAyC;QACzC,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,kCAAkC;QAClC,MAAM,eAAe,GAAG,MAAM,SAAS,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QAEnE,0BAA0B;QAC1B,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC;YAC7C,UAAU,EAAE,WAAW,CAAC,gBAAgB;YACxC,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,IAAI,EAAE,WAAW,CAAC,UAAU;YAC5B,GAAG,EAAE,WAAW,CAAC,SAAS;YAC1B,aAAa,EAAE,WAAW,CAAC,SAAS;YACpC,UAAU,EAAE,GAAG;YACf,QAAQ,EAAE,WAAW,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,4DAA4D;QAC5D,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,eAAe,GAAa,EAAE,CAAC;QAErC,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;gBACrC,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,QAAQ,EAAE,MAAM,CAAC,EAAE;gBACnB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,GAAG;gBACnC,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAY,IAAI,KAAK;gBACzC,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;gBAC1C,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI,IAAI,EAAE;gBAC1C,QAAQ,EAAE;oBACR,GAAG,KAAK,CAAC,UAAU;iBACpB;aACF,CAAC,CAAC;YAEH,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,aAAa,EAAE,CAAC;gBAChB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrC,CAAC;iBAAM,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;gBACvB,aAAa,EAAE,CAAC;gBAChB,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACrC,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,MAAM,YAAY,GAAG,OAAO,EAAE,qBAAqB,IAAI,IAAI,CAAC;QAC5D,IAAI,YAAY,IAAI,IAAI,CAAC,gBAAgB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,iEAAiE,EAAE;gBAC7E,YAAY,EAAE,eAAe,CAAC,MAAM;aACrC,CAAC,CAAC;YAEH,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,OAAO,EAAE;wBACvD,aAAa,EAAE,OAAO,EAAE,aAAa,IAAI,GAAG;qBAC7C,CAAC,CAAC;oBACH,mBAAmB,IAAI,OAAO,CAAC,mBAAmB,CAAC;oBACnD,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;gBACjD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAgB;YAC3B,MAAM,EAAE,WAAW;YACnB,MAAM,EAAE;gBACN,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,WAAW,CAAC,gBAAgB;aAC9D;YACD,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,cAAc,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,UAAU,IAAI,CAAC,CAAC;YAC5D,eAAe,EAAE,eAAe,CAAC,MAAM;YACvC,aAAa;YACb,aAAa;YACb,UAAU;YACV,mBAAmB;YACnB,iBAAiB;SAClB,CAAC;QAEF,MAAM,CAAC,IAAI,CAAC,0CAA0C,EAAE;YACtD,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;QACH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;OAEG;IACI,KAAK,CAAC,WAAW,CACtB,QAAsD,EACtD,OAAgD;QAgBhD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,OAAO,GAKR,EAAE,CAAC;QAER,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,wBAAwB,GAAG,CAAC,CAAC;QAEjC,oDAAoD;QACpD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;YAC/B,OAAO,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAC/B,MAAM,UAAU,GAAG,KAAK,EAAE,CAAC;gBAC3B,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;oBACzE,OAAO,CAAC,UAAU,CAAC,GAAG;wBACpB,SAAS,EAAE,GAAG,CAAC,SAAS;wBACxB,MAAM,EAAE,WAAW;wBACnB,OAAO;qBACR,CAAC;oBACF,SAAS,EAAE,CAAC;oBACZ,kBAAkB,IAAI,OAAO,CAAC,aAAa,CAAC;oBAC5C,kBAAkB,IAAI,OAAO,CAAC,aAAa,CAAC;oBAC5C,wBAAwB,IAAI,OAAO,CAAC,mBAAmB,IAAI,CAAC,CAAC;gBAC/D,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;oBAC7D,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;oBACzF,OAAO,CAAC,UAAU,CAAC,GAAG;wBACpB,SAAS,EAAE,GAAG,CAAC,SAAS;wBACxB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,GAAG;qBACX,CAAC;oBACF,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAClF,aAAa,EAAE,CAChB,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,OAAO;YACL,gBAAgB,EAAE,QAAQ,CAAC,MAAM;YACjC,gBAAgB,EAAE,SAAS;YAC3B,aAAa,EAAE,MAAM;YACrB,aAAa,EAAE,kBAAkB;YACjC,aAAa,EAAE,kBAAkB;YACjC,mBAAmB,EAAE,wBAAwB;YAC7C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACrD,OAAO;SACR,CAAC;IACJ,CAAC;CACF"}

package/dist/connectors/types/connector.d.ts

@@ -0,0 +1,34 @@
+import type { SourceType } from '../../domain/entities/source.js';
+import type { FetchResult, ExtractedClaim } from './fetchResult.js';
+export interface ConnectorCapabilities {
+    supportsIncrementalSync: boolean;
+    supportsFileInspection: boolean;
+    supportedFileTypes: string[];
+}
+export interface ConnectorMetadata {
+    type: SourceType;
+    displayName: string;
+    description: string;
+    authRequirement: 'none' | 'optional' | 'required';
+    enabled: boolean;
+    capabilities: ConnectorCapabilities;
+}
+export interface ConnectionTestResult {
+    accessible: boolean;
+    authenticated: boolean;
+    targetFound: boolean;
+    rateLimit?: {
+        limit: number;
+        remaining: number;
+        resetAt: Date;
+    };
+    details?: Record<string, unknown>;
+    error?: string;
+}
+export interface Connector<TInput = unknown, TRaw = unknown> {
+    readonly type: SourceType;
+    readonly metadata: ConnectorMetadata;
+    testConnection(input?: TInput): Promise<ConnectionTestResult>;
+    fetch(input: TInput): Promise<FetchResult<TRaw>>;
+    extractClaims(fetchResult: FetchResult<TRaw>): Promise<ExtractedClaim[]>;
+}

package/dist/connectors/types/connector.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=connector.js.map

package/dist/connectors/types/connector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"connector.js","sourceRoot":"","sources":["../../../src/connectors/types/connector.ts"],"names":[],"mappings":""}

package/dist/connectors/types/fetchResult.d.ts

@@ -0,0 +1,39 @@
+import type { ClaimEnvironment, ClaimScope, ClaimSourceRole, ClaimValueType } from '../../domain/entities/claim.js';
+export interface ClaimProvenance {
+    connector: string;
+    repository?: string;
+    filePath: string;
+    url?: string;
+    extractionMethod: string;
+    lineRange?: [number, number];
+    page?: number;
+    observedAt: string;
+    evidence: string;
+}
+export interface ExtractedClaim {
+    subject: string;
+    predicate: string;
+    value: string;
+    valueType: ClaimValueType;
+    normalizedValue?: string | null;
+    confidence?: number;
+    environment?: ClaimEnvironment | null;
+    scope?: ClaimScope | null;
+    sourceRole?: ClaimSourceRole | null;
+    isHistorical?: boolean;
+    validFrom?: Date | null;
+    validUntil?: Date | null;
+    valueConstraint?: string | null;
+    multiValueContext?: string | null;
+    observedAt?: Date;
+    externalId: string;
+    provenance: ClaimProvenance;
+}
+export interface FetchResult<T = unknown> {
+    sourceIdentifier: string;
+    sourceName: string;
+    sourceUri: string;
+    fetchedAt: Date;
+    rawData: T;
+    metadata: Record<string, unknown>;
+}

package/dist/connectors/types/fetchResult.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=fetchResult.js.map

package/dist/connectors/types/fetchResult.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetchResult.js","sourceRoot":"","sources":["../../../src/connectors/types/fetchResult.ts"],"names":[],"mappings":""}

package/dist/connectors/website/ssrfGuard.d.ts

@@ -0,0 +1,11 @@
+export declare class SsrfError extends Error {
+    constructor(message: string);
+}
+/**
+ * Validates whether an IP address is considered private, loopback, link-local, or cloud metadata.
+ */
+export declare function isPrivateOrRestrictedIp(ip: string): boolean;
+/**
+ * Validates a target URL against SSRF vulnerabilities by resolving DNS and verifying the IP.
+ */
+export declare function validateSafeUrl(rawUrl: string): Promise<URL>;

package/dist/connectors/website/ssrfGuard.js

@@ -0,0 +1,122 @@
+import dns from 'node:dns/promises';
+import net from 'node:net';
+export class SsrfError extends Error {
+    constructor(message) {
+        super(`SSRF Protection: ${message}`);
+        this.name = 'SsrfError';
+    }
+}
+/**
+ * Validates whether an IP address is considered private, loopback, link-local, or cloud metadata.
+ */
+export function isPrivateOrRestrictedIp(ip) {
+    // Normalize IPv4-mapped IPv6 (e.g. ::ffff:127.0.0.1)
+    if (ip.startsWith('::ffff:')) {
+        ip = ip.substring(7);
+    }
+    // IPv4 checks
+    if (net.isIPv4(ip)) {
+        const parts = ip.split('.').map((p) => parseInt(p, 10));
+        if (parts.length !== 4 || parts.some((p) => isNaN(p) || p < 0 || p > 255)) {
+            return true; // invalid = restricted
+        }
+        const [a, b, c, _d] = parts;
+        // 0.0.0.0/8 (Current network)
+        if (a === 0)
+            return true;
+        // 127.0.0.0/8 (Loopback)
+        if (a === 127)
+            return true;
+        // 10.0.0.0/8 (Private)
+        if (a === 10)
+            return true;
+        // 172.16.0.0/12 (Private: 172.16.0.0 - 172.31.255.255)
+        if (a === 172 && b >= 16 && b <= 31)
+            return true;
+        // 192.168.0.0/16 (Private)
+        if (a === 192 && b === 168)
+            return true;
+        // 169.254.0.0/16 (Link-local & AWS/GCP metadata 169.254.169.254)
+        if (a === 169 && b === 254)
+            return true;
+        // 100.64.0.0/10 (Carrier-grade NAT)
+        if (a === 100 && b >= 64 && b <= 127)
+            return true;
+        // 192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24 (Documentation)
+        if (a === 192 && b === 0 && c === 2)
+            return true;
+        if (a === 198 && b === 51 && c === 100)
+            return true;
+        if (a === 203 && b === 0 && c === 113)
+            return true;
+        // 224.0.0.0/4 (Multicast) & 240.0.0.0/4 (Reserved / Broadcast)
+        if (a >= 224)
+            return true;
+        return false;
+    }
+    // IPv6 checks
+    if (net.isIPv6(ip)) {
+        const lower = ip.toLowerCase();
+        // ::1 (Loopback)
+        if (lower === '::1' || lower === '0000:0000:0000:0000:0000:0000:0000:0001')
+            return true;
+        // :: (Unspecified)
+        if (lower === '::' || lower === '0000:0000:0000:0000:0000:0000:0000:0000')
+            return true;
+        // fc00::/7 (Unique Local Address - ULA)
+        if (lower.startsWith('fc') || lower.startsWith('fd'))
+            return true;
+        // fe80::/10 (Link-local)
+        if (lower.startsWith('fe8') ||
+            lower.startsWith('fe9') ||
+            lower.startsWith('fea') ||
+            lower.startsWith('feb'))
+            return true;
+        return false;
+    }
+    return true;
+}
+/**
+ * Validates a target URL against SSRF vulnerabilities by resolving DNS and verifying the IP.
+ */
+export async function validateSafeUrl(rawUrl) {
+    let parsedUrl;
+    try {
+        parsedUrl = new URL(rawUrl);
+    }
+    catch {
+        throw new SsrfError(`Malformed URL: '${rawUrl}'`);
+    }
+    if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
+        throw new SsrfError(`Forbidden protocol: '${parsedUrl.protocol}'. Only http: and https: are permitted.`);
+    }
+    const hostname = parsedUrl.hostname.toLowerCase();
+    // Explicit hostnames that are strictly banned
+    const bannedHosts = ['localhost', 'metadata.google.internal', 'instance-data', '169.254.169.254'];
+    if (bannedHosts.includes(hostname) ||
+        hostname.endsWith('.localhost') ||
+        hostname.endsWith('.local')) {
+        throw new SsrfError(`Forbidden host: '${hostname}'`);
+    }
+    // Resolve DNS to verify destination IP addresses
+    try {
+        const addresses = await dns.lookup(hostname, { all: true });
+        if (!addresses || addresses.length === 0) {
+            throw new SsrfError(`Failed to resolve DNS for hostname '${hostname}'`);
+        }
+        for (const record of addresses) {
+            if (isPrivateOrRestrictedIp(record.address)) {
+                throw new SsrfError(`Resolved IP '${record.address}' for host '${hostname}' is private/loopback/restricted. Request blocked.`);
+            }
+        }
+    }
+    catch (error) {
+        if (error instanceof SsrfError) {
+            throw error;
+        }
+        const msg = error instanceof Error ? error.message : String(error);
+        throw new SsrfError(`DNS resolution failure for '${hostname}': ${msg}`);
+    }
+    return parsedUrl;
+}
+//# sourceMappingURL=ssrfGuard.js.map

package/dist/connectors/website/ssrfGuard.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ssrfGuard.js","sourceRoot":"","sources":["../../../src/connectors/website/ssrfGuard.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,mBAAmB,CAAC;AACpC,OAAO,GAAG,MAAM,UAAU,CAAC;AAE3B,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAe;QACzB,KAAK,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;IAC1B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,EAAU;IAChD,qDAAqD;IACrD,IAAI,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC7B,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,cAAc;IACd,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC;YAC1E,OAAO,IAAI,CAAC,CAAC,uBAAuB;QACtC,CAAC;QAED,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC;QAE5B,8BAA8B;QAC9B,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEzB,yBAAyB;QACzB,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAE3B,uBAAuB;QACvB,IAAI,CAAC,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;QAE1B,uDAAuD;QACvD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC;QAEjD,2BAA2B;QAC3B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAExC,iEAAiE;QACjE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAExC,oCAAoC;QACpC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;YAAE,OAAO,IAAI,CAAC;QAElD,gEAAgE;QAChE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACjD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QACpD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC;QAEnD,+DAA+D;QAC/D,IAAI,CAAC,IAAI,GAAG;YAAE,OAAO,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,cAAc;IACd,IAAI,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;QAE/B,iBAAiB;QACjB,IAAI,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,yCAAyC;YAAE,OAAO,IAAI,CAAC;QAExF,mBAAmB;QACnB,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,yCAAyC;YAAE,OAAO,IAAI,CAAC;QAEvF,wCAAwC;QACxC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QAElE,yBAAyB;QACzB,IACE,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YACvB,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;YAEvB,OAAO,IAAI,CAAC;QAEd,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAc;IAClD,IAAI,SAAc,CAAC;IACnB,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,SAAS,CAAC,mBAAmB,MAAM,GAAG,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACtE,MAAM,IAAI,SAAS,CACjB,wBAAwB,SAAS,CAAC,QAAQ,yCAAyC,CACpF,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElD,8CAA8C;IAC9C,MAAM,WAAW,GAAG,CAAC,WAAW,EAAE,0BAA0B,EAAE,eAAe,EAAE,iBAAiB,CAAC,CAAC;IAClG,IACE,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC9B,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC;QAC/B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAC3B,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,oBAAoB,QAAQ,GAAG,CAAC,CAAC;IACvD,CAAC;IAED,iDAAiD;IACjD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,SAAS,CAAC,uCAAuC,QAAQ,GAAG,CAAC,CAAC;QAC1E,CAAC;QAED,KAAK,MAAM,MAAM,IAAI,SAAS,EAAE,CAAC;YAC/B,IAAI,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5C,MAAM,IAAI,SAAS,CACjB,gBAAgB,MAAM,CAAC,OAAO,eAAe,QAAQ,oDAAoD,CAC1G,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,SAAS,EAAE,CAAC;YAC/B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,SAAS,CAAC,+BAA+B,QAAQ,MAAM,GAAG,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/connectors/website/websiteConnector.d.ts

@@ -0,0 +1,30 @@
+import { Connector, ConnectorMetadata, ConnectionTestResult } from '../types/connector.js';
+import { FetchResult, ExtractedClaim } from '../types/fetchResult.js';
+import { ConnectorSecurityDescriptor } from '../base/security.js';
+export interface WebsiteInput {
+    url: string;
+    sourceName?: string;
+    scope?: string;
+    environment?: string;
+    timeoutMs?: number;
+}
+export interface WebsiteRawData {
+    url: string;
+    canonicalUrl?: string;
+    title: string;
+    statusCode: number;
+    contentType: string;
+    headers: Record<string, string>;
+    textSnippet: string;
+    extractedLines: string[];
+}
+export declare class WebsiteConnector implements Connector<WebsiteInput, WebsiteRawData> {
+    readonly type: "website";
+    readonly metadata: ConnectorMetadata;
+    readonly security: ConnectorSecurityDescriptor;
+    testConnection(input?: WebsiteInput): Promise<ConnectionTestResult>;
+    fetch(input: WebsiteInput): Promise<FetchResult<WebsiteRawData>>;
+    extractClaims(fetchResult: FetchResult<WebsiteRawData>): Promise<ExtractedClaim[]>;
+    private extractTitle;
+    private extractCleanLines;
+}