@xapp/arachne-utils 1.10.0 → 1.10.7

package/lib/detectDocumentLinks.d.ts

@@ -0,0 +1,53 @@
+/*! Copyright (c) 2025, XAPP AI */
+import type { Page } from "puppeteer-core";
+/**
+ * Document type enumeration
+ */
+export type DocumentType = "PDF" | "MS_WORD" | "MS_EXCEL" | "PPT";
+/**
+ * Interface representing a document link detected on a page
+ */
+export interface DocumentLink {
+    /**
+     * The absolute URL to the document
+     */
+    url: string;
+    /**
+     * The type of document
+     */
+    type: DocumentType;
+    /**
+     * Optional title/text of the link
+     */
+    title?: string;
+}
+/**
+ * All supported document extensions
+ */
+export declare const DOCUMENT_EXTENSIONS: string[];
+/**
+ * Detects document links on a page
+ *
+ * This function identifies document links by checking file extensions in both the pathname
+ * and query parameters of URLs. It detects common document formats including PDF, Word,
+ * Excel, and PowerPoint files.
+ *
+ * **Limitations**:
+ * - Only detects documents with recognizable file extensions
+ * - Does not detect documents served via content-type headers without extensions
+ * - Does not make HEAD requests to verify content types
+ *
+ * @param page - The Puppeteer page to scan for document links
+ * @param allowedTypes - Optional array of document types to filter by (e.g., ['PDF', 'MS_WORD'])
+ * @returns Promise resolving to an array of detected document links
+ *
+ * @example
+ * ```typescript
+ * const documentLinks = await detectDocumentLinks(page);
+ * // Returns all document links
+ *
+ * const pdfLinks = await detectDocumentLinks(page, ['PDF']);
+ * // Returns only PDF links
+ * ```
+ */
+export declare function detectDocumentLinks(page: Page, allowedTypes?: DocumentType[]): Promise<DocumentLink[]>;

package/lib/detectDocumentLinks.js

@@ -0,0 +1,131 @@
+"use strict";
+/*! Copyright (c) 2025, XAPP AI */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DOCUMENT_EXTENSIONS = void 0;
+exports.detectDocumentLinks = detectDocumentLinks;
+const hasExtension_1 = require("./hasExtension");
+/**
+ * Extension to document type mapping
+ */
+const EXTENSION_TO_TYPE = {
+    '.pdf': 'PDF',
+    '.doc': 'MS_WORD',
+    '.docx': 'MS_WORD',
+    '.xls': 'MS_EXCEL',
+    '.xlsx': 'MS_EXCEL',
+    '.ppt': 'PPT',
+    '.pptx': 'PPT'
+};
+/**
+ * All supported document extensions
+ */
+exports.DOCUMENT_EXTENSIONS = Object.keys(EXTENSION_TO_TYPE);
+/**
+ * Extract document type from URL by checking both pathname and query parameters
+ * @param url - The URL to check
+ * @returns The document type if found, undefined otherwise
+ */
+function getDocumentTypeFromUrl(url) {
+    // First check pathname for extensions
+    for (const [extension, type] of Object.entries(EXTENSION_TO_TYPE)) {
+        if ((0, hasExtension_1.hasExtension)(url, extension)) {
+            return type;
+        }
+    }
+    // Also check query parameters for extensions
+    // e.g., https://example.com/download?file=report.pdf&token=xyz
+    try {
+        const urlObj = new URL(url);
+        const queryString = urlObj.search.toLowerCase();
+        for (const [extension, type] of Object.entries(EXTENSION_TO_TYPE)) {
+            if (queryString.includes(extension)) {
+                return type;
+            }
+        }
+    }
+    catch (e) {
+        // Invalid URL, skip query parameter check
+    }
+    return undefined;
+}
+/**
+ * Detects document links on a page
+ *
+ * This function identifies document links by checking file extensions in both the pathname
+ * and query parameters of URLs. It detects common document formats including PDF, Word,
+ * Excel, and PowerPoint files.
+ *
+ * **Limitations**:
+ * - Only detects documents with recognizable file extensions
+ * - Does not detect documents served via content-type headers without extensions
+ * - Does not make HEAD requests to verify content types
+ *
+ * @param page - The Puppeteer page to scan for document links
+ * @param allowedTypes - Optional array of document types to filter by (e.g., ['PDF', 'MS_WORD'])
+ * @returns Promise resolving to an array of detected document links
+ *
+ * @example
+ * ```typescript
+ * const documentLinks = await detectDocumentLinks(page);
+ * // Returns all document links
+ *
+ * const pdfLinks = await detectDocumentLinks(page, ['PDF']);
+ * // Returns only PDF links
+ * ```
+ */
+function detectDocumentLinks(page, allowedTypes) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const baseUrl = page.url();
+        // Extract all links from the page
+        const links = yield page.$$eval('a[href]', (anchors, baseUrl) => {
+            return anchors.map(anchor => {
+                var _a;
+                const href = anchor.getAttribute('href');
+                if (!href)
+                    return null;
+                try {
+                    // Resolve relative URLs
+                    const absoluteUrl = new URL(href, baseUrl).toString();
+                    const text = ((_a = anchor.textContent) === null || _a === void 0 ? void 0 : _a.trim()) || anchor.getAttribute('title') || '';
+                    return {
+                        url: absoluteUrl,
+                        title: text
+                    };
+                }
+                catch (e) {
+                    // Invalid URL, skip it
+                    return null;
+                }
+            }).filter(link => link !== null);
+        }, baseUrl);
+        // Filter and classify document links
+        const documentLinks = [];
+        for (const link of links) {
+            if (!link)
+                continue;
+            // Get document type from URL (checks both pathname and query parameters)
+            const type = getDocumentTypeFromUrl(link.url);
+            if (type) {
+                // If allowedTypes is specified, filter by it
+                if (!allowedTypes || allowedTypes.includes(type)) {
+                    documentLinks.push({
+                        url: link.url,
+                        type,
+                        title: link.title
+                    });
+                }
+            }
+        }
+        return documentLinks;
+    });
+}
+//# sourceMappingURL=detectDocumentLinks.js.map

package/lib/detectDocumentLinks.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"detectDocumentLinks.js","sourceRoot":"","sources":["../src/detectDocumentLinks.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;AAsGlC,kDAkDC;AArJD,iDAA8C;AAyB9C;;GAEG;AACH,MAAM,iBAAiB,GAAiC;IACpD,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,SAAS;IACjB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,UAAU;IAClB,OAAO,EAAE,UAAU;IACnB,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,KAAK;CACjB,CAAC;AAEF;;GAEG;AACU,QAAA,mBAAmB,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAElE;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,GAAW;IACvC,sCAAsC;IACtC,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAChE,IAAI,IAAA,2BAAY,EAAC,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/B,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,6CAA6C;IAC7C,+DAA+D;IAC/D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAEhD,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAChE,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBAClC,OAAO,IAAI,CAAC;YAChB,CAAC;QACL,CAAC;IACL,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,0CAA0C;IAC9C,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,SAAsB,mBAAmB,CACrC,IAAU,EACV,YAA6B;;QAE7B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,kCAAkC;QAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;YAC5D,OAAO,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;;gBACxB,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;gBACzC,IAAI,CAAC,IAAI;oBAAE,OAAO,IAAI,CAAC;gBAEvB,IAAI,CAAC;oBACD,wBAAwB;oBACxB,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,QAAQ,EAAE,CAAC;oBACtD,MAAM,IAAI,GAAG,CAAA,MAAA,MAAM,CAAC,WAAW,0CAAE,IAAI,EAAE,KAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;oBAE9E,OAAO;wBACH,GAAG,EAAE,WAAW;wBAChB,KAAK,EAAE,IAAI;qBACd,CAAC;gBACN,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,uBAAuB;oBACvB,OAAO,IAAI,CAAC;gBAChB,CAAC;YACL,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACrC,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,qCAAqC;QACrC,MAAM,aAAa,GAAmB,EAAE,CAAC;QAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,yEAAyE;YACzE,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE9C,IAAI,IAAI,EAAE,CAAC;gBACP,6CAA6C;gBAC7C,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC/C,aAAa,CAAC,IAAI,CAAC;wBACf,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,IAAI;wBACJ,KAAK,EAAE,IAAI,CAAC,KAAK;qBACpB,CAAC,CAAC;gBACP,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,aAAa,CAAC;IACzB,CAAC;CAAA"}

package/lib/downloadDocument.d.ts

@@ -0,0 +1,94 @@
+/*! Copyright (c) 2025, XAPP AI */
+/**
+ * Default maximum file size for downloads (50MB)
+ */
+export declare const DEFAULT_MAX_FILE_SIZE: number;
+/**
+ * Default download timeout (30 seconds)
+ */
+export declare const DEFAULT_DOWNLOAD_TIMEOUT = 30000;
+/**
+ * Options for downloading documents
+ */
+export interface DownloadDocumentOptions {
+    /**
+     * Maximum file size in bytes. Downloads larger than this will be rejected.
+     * Default: 50MB (52,428,800 bytes)
+     */
+    maxFileSize?: number;
+    /**
+     * Download timeout in milliseconds
+     * Default: 30000 (30 seconds)
+     */
+    timeout?: number;
+    /**
+     * Whether to perform SSRF protection checks
+     * Default: true
+     */
+    ssrfProtection?: boolean;
+    /**
+     * Optional custom headers to include in the request
+     */
+    headers?: Record<string, string>;
+    /**
+     * Whether to validate Content-Type header matches expected document types
+     * Default: true
+     */
+    validateContentType?: boolean;
+}
+/**
+ * Result of a document download operation
+ */
+export interface DownloadDocumentResult {
+    /**
+     * The downloaded document as a Buffer
+     */
+    data: Buffer;
+    /**
+     * Content-Type header from the response
+     */
+    contentType?: string;
+    /**
+     * Content-Length header from the response (file size in bytes)
+     */
+    contentLength?: number;
+    /**
+     * The final URL after any redirects
+     */
+    finalUrl: string;
+}
+/**
+ * Download a document from a URL with safety checks and size limits
+ *
+ * This function provides:
+ * - SSRF protection (blocks private IPs, localhost, metadata endpoints)
+ * - File size limits to prevent memory exhaustion
+ * - Streaming download with incremental size checking
+ * - Timeout handling
+ * - Content-Type validation
+ *
+ * The download uses streaming to protect against memory exhaustion attacks. If a server
+ * sends more data than the `maxFileSize` limit, the download is aborted immediately
+ * rather than loading the entire file into memory.
+ *
+ * @param url - The URL of the document to download
+ * @param options - Download options
+ * @returns Promise resolving to the downloaded document data
+ * @throws Error if download fails, exceeds size limit, or violates SSRF protection
+ *
+ * @example
+ * ```typescript
+ * try {
+ *   const result = await downloadDocument('https://example.com/doc.pdf', {
+ *     maxFileSize: 10 * 1024 * 1024, // 10MB
+ *     timeout: 15000 // 15 seconds
+ *   });
+ *
+ *   // Save to file
+ *   fs.writeFileSync('downloaded.pdf', result.data);
+ * } catch (error) {
+ *   console.error('Download failed:', error.message);
+ * }
+ * ```
+ */
+export declare function downloadDocument(url: string, options?: DownloadDocumentOptions): Promise<DownloadDocumentResult>;

package/lib/downloadDocument.js

@@ -0,0 +1,158 @@
+"use strict";
+/*! Copyright (c) 2025, XAPP AI */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_DOWNLOAD_TIMEOUT = exports.DEFAULT_MAX_FILE_SIZE = void 0;
+exports.downloadDocument = downloadDocument;
+const ssrfProtection_1 = require("./ssrfProtection");
+/**
+ * Default maximum file size for downloads (50MB)
+ */
+exports.DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB in bytes
+/**
+ * Default download timeout (30 seconds)
+ */
+exports.DEFAULT_DOWNLOAD_TIMEOUT = 30000; // 30 seconds
+/**
+ * Expected Content-Type values for document types
+ */
+const DOCUMENT_CONTENT_TYPES = [
+    'application/pdf',
+    'application/msword',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'application/vnd.ms-excel',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'application/vnd.ms-powerpoint',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    'application/octet-stream', // Sometimes servers use generic binary type
+];
+/**
+ * Download a document from a URL with safety checks and size limits
+ *
+ * This function provides:
+ * - SSRF protection (blocks private IPs, localhost, metadata endpoints)
+ * - File size limits to prevent memory exhaustion
+ * - Streaming download with incremental size checking
+ * - Timeout handling
+ * - Content-Type validation
+ *
+ * The download uses streaming to protect against memory exhaustion attacks. If a server
+ * sends more data than the `maxFileSize` limit, the download is aborted immediately
+ * rather than loading the entire file into memory.
+ *
+ * @param url - The URL of the document to download
+ * @param options - Download options
+ * @returns Promise resolving to the downloaded document data
+ * @throws Error if download fails, exceeds size limit, or violates SSRF protection
+ *
+ * @example
+ * ```typescript
+ * try {
+ *   const result = await downloadDocument('https://example.com/doc.pdf', {
+ *     maxFileSize: 10 * 1024 * 1024, // 10MB
+ *     timeout: 15000 // 15 seconds
+ *   });
+ *
+ *   // Save to file
+ *   fs.writeFileSync('downloaded.pdf', result.data);
+ * } catch (error) {
+ *   console.error('Download failed:', error.message);
+ * }
+ * ```
+ */
+function downloadDocument(url_1) {
+    return __awaiter(this, arguments, void 0, function* (url, options = {}) {
+        var _a;
+        const { maxFileSize = exports.DEFAULT_MAX_FILE_SIZE, timeout = exports.DEFAULT_DOWNLOAD_TIMEOUT, ssrfProtection = true, headers = {}, validateContentType = true } = options;
+        // SSRF protection check
+        if (ssrfProtection && !(0, ssrfProtection_1.isSafeURL)(url)) {
+            throw new Error(`SSRF protection: URL is not safe to fetch: ${url}`);
+        }
+        // Create AbortController for timeout
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), timeout);
+        try {
+            const response = yield fetch(url, {
+                headers: Object.assign({ 'User-Agent': 'Mozilla/5.0 (compatible; Arachne/1.0; +https://github.com/xapp-ai/arachne)' }, headers),
+                signal: controller.signal,
+                redirect: 'follow'
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            // Validate Content-Type if requested
+            const contentType = response.headers.get('content-type');
+            if (validateContentType && contentType) {
+                const isValidType = DOCUMENT_CONTENT_TYPES.some(type => contentType.toLowerCase().includes(type.toLowerCase()));
+                if (!isValidType) {
+                    throw new Error(`Invalid Content-Type: ${contentType}. Expected a document type.`);
+                }
+            }
+            // Check Content-Length before downloading
+            const contentLengthHeader = response.headers.get('content-length');
+            if (contentLengthHeader) {
+                const contentLength = parseInt(contentLengthHeader, 10);
+                if (contentLength > maxFileSize) {
+                    throw new Error(`File size (${contentLength} bytes) exceeds maximum allowed size (${maxFileSize} bytes)`);
+                }
+            }
+            // Stream the download and check size incrementally to prevent memory exhaustion
+            const reader = (_a = response.body) === null || _a === void 0 ? void 0 : _a.getReader();
+            if (!reader) {
+                throw new Error('Response body is not readable');
+            }
+            const chunks = [];
+            let receivedLength = 0;
+            try {
+                while (true) {
+                    const { done, value } = yield reader.read();
+                    if (done)
+                        break;
+                    receivedLength += value.length;
+                    // Check size during download to prevent memory exhaustion
+                    if (receivedLength > maxFileSize) {
+                        reader.cancel();
+                        throw new Error(`File size exceeds maximum allowed size (${maxFileSize} bytes) during download`);
+                    }
+                    chunks.push(value);
+                }
+            }
+            catch (error) {
+                // Ensure reader is cancelled on any error
+                try {
+                    yield reader.cancel();
+                }
+                catch (_b) {
+                    // Ignore cancellation errors
+                }
+                throw error;
+            }
+            // Combine chunks into a single buffer
+            const buffer = Buffer.concat(chunks.map(chunk => Buffer.from(chunk)));
+            return {
+                data: buffer,
+                contentType: contentType || undefined,
+                contentLength: buffer.length,
+                finalUrl: response.url
+            };
+        }
+        catch (error) {
+            if (error.name === 'AbortError') {
+                throw new Error(`Download timeout after ${timeout}ms`);
+            }
+            throw error;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    });
+}
+//# sourceMappingURL=downloadDocument.js.map

package/lib/downloadDocument.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloadDocument.js","sourceRoot":"","sources":["../src/downloadDocument.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;AAyHlC,4CA+GC;AAtOD,qDAA6C;AAE7C;;GAEG;AACU,QAAA,qBAAqB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,gBAAgB;AAEvE;;GAEG;AACU,QAAA,wBAAwB,GAAG,KAAK,CAAC,CAAC,aAAa;AA6D5D;;GAEG;AACH,MAAM,sBAAsB,GAAG;IAC3B,iBAAiB;IACjB,oBAAoB;IACpB,yEAAyE;IACzE,0BAA0B;IAC1B,mEAAmE;IACnE,+BAA+B;IAC/B,2EAA2E;IAC3E,0BAA0B,EAAE,4CAA4C;CAC3E,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,SAAsB,gBAAgB;yDAClC,GAAW,EACX,UAAmC,EAAE;;QAErC,MAAM,EACF,WAAW,GAAG,6BAAqB,EACnC,OAAO,GAAG,gCAAwB,EAClC,cAAc,GAAG,IAAI,EACrB,OAAO,GAAG,EAAE,EACZ,mBAAmB,GAAG,IAAI,EAC7B,GAAG,OAAO,CAAC;QAEZ,wBAAwB;QACxB,IAAI,cAAc,IAAI,CAAC,IAAA,0BAAS,EAAC,GAAG,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,8CAA8C,GAAG,EAAE,CAAC,CAAC;QACzE,CAAC;QAED,qCAAqC;QACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAEhE,IAAI,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAC9B,OAAO,kBACH,YAAY,EAAE,4EAA4E,IACvF,OAAO,CACb;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACvE,CAAC;YAED,qCAAqC;YACrC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YACzD,IAAI,mBAAmB,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,WAAW,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnD,WAAW,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CACzD,CAAC;gBAEF,IAAI,CAAC,WAAW,EAAE,CAAC;oBACf,MAAM,IAAI,KAAK,CAAC,yBAAyB,WAAW,6BAA6B,CAAC,CAAC;gBACvF,CAAC;YACL,CAAC;YAED,0CAA0C;YAC1C,MAAM,mBAAmB,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YACnE,IAAI,mBAAmB,EAAE,CAAC;gBACtB,MAAM,aAAa,GAAG,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;gBACxD,IAAI,aAAa,GAAG,WAAW,EAAE,CAAC;oBAC9B,MAAM,IAAI,KAAK,CACX,cAAc,aAAa,yCAAyC,WAAW,SAAS,CAC3F,CAAC;gBACN,CAAC;YACL,CAAC;YAED,gFAAgF;YAChF,MAAM,MAAM,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;YACrD,CAAC;YAED,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,cAAc,GAAG,CAAC,CAAC;YAEvB,IAAI,CAAC;gBACD,OAAO,IAAI,EAAE,CAAC;oBACV,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,cAAc,IAAI,KAAK,CAAC,MAAM,CAAC;oBAE/B,0DAA0D;oBAC1D,IAAI,cAAc,GAAG,WAAW,EAAE,CAAC;wBAC/B,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,KAAK,CACX,2CAA2C,WAAW,yBAAyB,CAClF,CAAC;oBACN,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACvB,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,0CAA0C;gBAC1C,IAAI,CAAC;oBACD,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC1B,CAAC;gBAAC,WAAM,CAAC;oBACL,6BAA6B;gBACjC,CAAC;gBACD,MAAM,KAAK,CAAC;YAChB,CAAC;YAED,sCAAsC;YACtC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtE,OAAO;gBACH,IAAI,EAAE,MAAM;gBACZ,WAAW,EAAE,WAAW,IAAI,SAAS;gBACrC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,QAAQ,EAAE,QAAQ,CAAC,GAAG;aACzB,CAAC;QACN,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YAClB,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CAAC,0BAA0B,OAAO,IAAI,CAAC,CAAC;YAC3D,CAAC;YACD,MAAM,KAAK,CAAC;QAChB,CAAC;gBAAS,CAAC;YACP,YAAY,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;CAAA"}

package/lib/ssrfProtection.d.ts

@@ -0,0 +1,41 @@
+/*! Copyright (c) 2025, XAPP AI */
+/**
+ * Check if an IP address (v4 or v6) is private or reserved
+ *
+ * This includes:
+ * - Loopback addresses (127.0.0.1, ::1)
+ * - Private network ranges (10.x.x.x, 192.168.x.x, 172.16-31.x.x)
+ * - Link-local addresses (169.254.x.x, fe80::)
+ * - Other reserved ranges
+ *
+ * @param ip - The IP address to check
+ * @returns true if the IP is private/reserved, false otherwise
+ */
+export declare function isPrivateIP(ip: string): boolean;
+/**
+ * Check if a URL is safe to fetch (not vulnerable to SSRF attacks)
+ *
+ * This function blocks:
+ * - localhost and 127.0.0.1
+ * - Private IP ranges (10.x.x.x, 192.168.x.x, 172.16-31.x.x)
+ * - AWS metadata endpoint (169.254.169.254)
+ * - IPv6 local addresses (::1, fe80::, etc.)
+ * - Cloud provider metadata endpoints
+ *
+ * **Security Limitation**: This function only checks the URL at the time of the call.
+ * It does not protect against DNS rebinding attacks, where a malicious domain could
+ * resolve to a public IP during this check but then resolve to a private IP (e.g., 127.0.0.1)
+ * during the actual fetch. For high-security environments, consider implementing additional
+ * DNS resolution validation or using a network-level SSRF protection solution.
+ *
+ * @param url - The URL to validate
+ * @returns true if the URL is safe to fetch, false if it should be blocked
+ *
+ * @example
+ * ```typescript
+ * if (!isSafeURL('http://localhost/file.pdf')) {
+ *   throw new Error('SSRF protection: Cannot fetch from localhost');
+ * }
+ * ```
+ */
+export declare function isSafeURL(url: string | URL): boolean;

package/lib/ssrfProtection.js

@@ -0,0 +1,157 @@
+"use strict";
+/*! Copyright (c) 2025, XAPP AI */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPrivateIP = isPrivateIP;
+exports.isSafeURL = isSafeURL;
+/**
+ * Private IPv4 ranges as defined by RFC 1918 and other reserved ranges
+ */
+const PRIVATE_IPV4_RANGES = [
+    { min: '10.0.0.0', max: '10.255.255.255' }, // Class A private
+    { min: '172.16.0.0', max: '172.31.255.255' }, // Class B private
+    { min: '192.168.0.0', max: '192.168.255.255' }, // Class C private
+    { min: '127.0.0.0', max: '127.255.255.255' }, // Loopback
+    { min: '169.254.0.0', max: '169.254.255.255' }, // Link-local (AWS metadata)
+    { min: '0.0.0.0', max: '0.255.255.255' }, // Current network
+];
+/**
+ * Private/reserved IPv6 ranges
+ */
+const PRIVATE_IPV6_PATTERNS = [
+    /^::1$/, // Loopback
+    /^::$/, // Unspecified
+    /^fe80:/, // Link-local
+    /^fc00:/, // Unique local
+    /^fd00:/, // Unique local
+];
+/**
+ * Blocked hostnames that should never be accessible
+ */
+const BLOCKED_HOSTNAMES = [
+    'localhost',
+    'metadata.google.internal', // GCP metadata
+];
+/**
+ * Convert IPv4 address string to a number for comparison
+ * Returns -1 if the IP is invalid
+ */
+function ipv4ToNumber(ip) {
+    const parts = ip.split('.').map(Number);
+    // Validate that we have exactly 4 parts and each is in valid range (0-255)
+    if (parts.length !== 4 || parts.some(p => p < 0 || p > 255 || isNaN(p))) {
+        return -1;
+    }
+    return (parts[0] << 24) + (parts[1] << 16) + (parts[2] << 8) + parts[3];
+}
+/**
+ * Check if an IPv4 address is in a private range
+ */
+function isPrivateIPv4(ip) {
+    const ipNum = ipv4ToNumber(ip);
+    // Invalid IP address - treat as unsafe
+    if (ipNum === -1) {
+        return true;
+    }
+    for (const range of PRIVATE_IPV4_RANGES) {
+        const minNum = ipv4ToNumber(range.min);
+        const maxNum = ipv4ToNumber(range.max);
+        if (ipNum >= minNum && ipNum <= maxNum) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Check if an IPv6 address is private/reserved
+ */
+function isPrivateIPv6(ip) {
+    const normalized = ip.toLowerCase();
+    for (const pattern of PRIVATE_IPV6_PATTERNS) {
+        if (pattern.test(normalized)) {
+            return true;
+        }
+    }
+    return false;
+}
+/**
+ * Check if an IP address (v4 or v6) is private or reserved
+ *
+ * This includes:
+ * - Loopback addresses (127.0.0.1, ::1)
+ * - Private network ranges (10.x.x.x, 192.168.x.x, 172.16-31.x.x)
+ * - Link-local addresses (169.254.x.x, fe80::)
+ * - Other reserved ranges
+ *
+ * @param ip - The IP address to check
+ * @returns true if the IP is private/reserved, false otherwise
+ */
+function isPrivateIP(ip) {
+    // Check for IPv4
+    if (/^(\d{1,3}\.){3}\d{1,3}$/.test(ip)) {
+        return isPrivateIPv4(ip);
+    }
+    // Check for IPv6
+    if (ip.includes(':')) {
+        return isPrivateIPv6(ip);
+    }
+    return false;
+}
+/**
+ * Check if a URL is safe to fetch (not vulnerable to SSRF attacks)
+ *
+ * This function blocks:
+ * - localhost and 127.0.0.1
+ * - Private IP ranges (10.x.x.x, 192.168.x.x, 172.16-31.x.x)
+ * - AWS metadata endpoint (169.254.169.254)
+ * - IPv6 local addresses (::1, fe80::, etc.)
+ * - Cloud provider metadata endpoints
+ *
+ * **Security Limitation**: This function only checks the URL at the time of the call.
+ * It does not protect against DNS rebinding attacks, where a malicious domain could
+ * resolve to a public IP during this check but then resolve to a private IP (e.g., 127.0.0.1)
+ * during the actual fetch. For high-security environments, consider implementing additional
+ * DNS resolution validation or using a network-level SSRF protection solution.
+ *
+ * @param url - The URL to validate
+ * @returns true if the URL is safe to fetch, false if it should be blocked
+ *
+ * @example
+ * ```typescript
+ * if (!isSafeURL('http://localhost/file.pdf')) {
+ *   throw new Error('SSRF protection: Cannot fetch from localhost');
+ * }
+ * ```
+ */
+function isSafeURL(url) {
+    let parsedUrl;
+    try {
+        parsedUrl = typeof url === 'string' ? new URL(url) : url;
+    }
+    catch (e) {
+        // Invalid URL
+        return false;
+    }
+    // Only allow HTTP and HTTPS protocols
+    if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
+        return false;
+    }
+    const hostname = parsedUrl.hostname.toLowerCase();
+    // Check blocked hostnames
+    if (BLOCKED_HOSTNAMES.includes(hostname)) {
+        return false;
+    }
+    // Check if hostname is an IP address (v4 or v6)
+    // For IPv4
+    if (/^(\d{1,3}\.){3}\d{1,3}$/.test(hostname)) {
+        return !isPrivateIPv4(hostname);
+    }
+    // For IPv6 (can be wrapped in brackets in hostname)
+    const ipv6Match = hostname.match(/^\[?([a-f0-9:]+)\]?$/i);
+    if (ipv6Match) {
+        return !isPrivateIPv6(ipv6Match[1]);
+    }
+    // Hostname is a domain name, which is generally safe
+    // (DNS rebinding attacks are a separate concern)
+    return true;
+}
+//# sourceMappingURL=ssrfProtection.js.map

package/lib/ssrfProtection.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ssrfProtection.js","sourceRoot":"","sources":["../src/ssrfProtection.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAkGlC,kCAYC;AA4BD,8BAqCC;AA7KD;;GAEG;AACH,MAAM,mBAAmB,GAAG;IACxB,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAY,kBAAkB;IACxE,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAU,kBAAkB;IACxE,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,iBAAiB,EAAE,EAAQ,kBAAkB;IACxE,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,iBAAiB,EAAE,EAAU,WAAW;IACjE,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,iBAAiB,EAAE,EAAQ,4BAA4B;IAClF,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,eAAe,EAAE,EAAc,kBAAkB;CAC3E,CAAC;AAEF;;GAEG;AACH,MAAM,qBAAqB,GAAG;IAC1B,OAAO,EAAuB,WAAW;IACzC,MAAM,EAAwB,cAAc;IAC5C,QAAQ,EAAsB,aAAa;IAC3C,QAAQ,EAAsB,eAAe;IAC7C,QAAQ,EAAsB,eAAe;CAChD,CAAC;AAEF;;GAEG;AACH,MAAM,iBAAiB,GAAG;IACtB,WAAW;IACX,0BAA0B,EAAG,eAAe;CAC/C,CAAC;AAEF;;;GAGG;AACH,SAAS,YAAY,CAAC,EAAU;IAC5B,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAExC,2EAA2E;IAC3E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,CAAC;IACd,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,EAAU;IAC7B,MAAM,KAAK,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;IAE/B,uCAAuC;IACvC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,mBAAmB,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvC,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,EAAU;IAC7B,MAAM,UAAU,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IAEpC,KAAK,MAAM,OAAO,IAAI,qBAAqB,EAAE,CAAC;QAC1C,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,WAAW,CAAC,EAAU;IAClC,iBAAiB;IACjB,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,iBAAiB;IACjB,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,aAAa,CAAC,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,SAAgB,SAAS,CAAC,GAAiB;IACvC,IAAI,SAAc,CAAC;IAEnB,IAAI,CAAC;QACD,SAAS,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACT,cAAc;QACd,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,sCAAsC;IACtC,IAAI,SAAS,CAAC,QAAQ,KAAK,OAAO,IAAI,SAAS,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACpE,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAElD,0BAA0B;IAC1B,IAAI,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,gDAAgD;IAChD,WAAW;IACX,IAAI,yBAAyB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED,oDAAoD;IACpD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IAC1D,IAAI,SAAS,EAAE,CAAC;QACZ,OAAO,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IAED,qDAAqD;IACrD,iDAAiD;IACjD,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@xapp/arachne-utils",
-    "version": "1.10.0",
+    "version": "1.10.7",
     "types": "lib/index",
     "main": "lib/index",
     "files": [
@@ -14,10 +14,10 @@
         "node": "^14 || ^16 || ^18 || ^20 || ^22"
     },
     "devDependencies": {
-        "@microsoft/api-extractor": "7.55.2",
+        "@microsoft/api-extractor": "7.57.2",
         "@types/chai": "4.3.20",
         "@types/mocha": "10.0.10",
-        "@types/node": "22.18.1",
+        "@types/node": "22.19.7",
         "@types/sinon": "17.0.4",
         "@types/sinon-chai": "3.2.12",
         "@xapp/config": "0.2.3",
@@ -38,5 +38,5 @@
         "clean": "rm -rf ./lib/*",
         "test": "mocha --recursive -r ts-node/register \"./src/**/*.test.ts\""
     },
-    "gitHead": "155ba31908c244d545a00ccd40064d798ebf8e3e"
+    "gitHead": "eafca4e497b35839d1c92dd25ccbd38c0d57e3b6"
 }