@juspay/neurolink 9.1.1 → 9.2.0

package/dist/processors/markup/MarkdownProcessor.js

@@ -0,0 +1,244 @@
+/**
+ * Markdown File Processor
+ *
+ * Processes Markdown files with structure extraction and analysis.
+ * Markdown files are analyzed to extract metadata about their structure
+ * including headings, code blocks, and tables.
+ *
+ * Features:
+ * - Original content preservation
+ * - Line count calculation
+ * - Code block detection
+ * - Table detection
+ * - Heading extraction (all levels)
+ *
+ * @module processors/markup/MarkdownProcessor
+ *
+ * @example
+ * ```typescript
+ * import { markdownProcessor, processMarkdown, isMarkdownFile } from "./markup/MarkdownProcessor.js";
+ *
+ * // Check if file is Markdown
+ * if (isMarkdownFile(mimetype, filename)) {
+ *   const result = await processMarkdown(fileInfo);
+ *   if (result.success) {
+ *     console.log('Line count:', result.data.lineCount);
+ *     console.log('Headings:', result.data.headings);
+ *     console.log('Has code blocks:', result.data.hasCodeBlocks);
+ *   }
+ * }
+ * ```
+ */
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import { MARKDOWN_EXTENSIONS, SIZE_LIMITS, TEXT_MIME_TYPES, } from "../config/index.js";
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+/**
+ * Supported Markdown MIME types.
+ * Derived from the centralized TEXT_MIME_TYPES config.
+ */
+const SUPPORTED_MARKDOWN_TYPES = [
+    TEXT_MIME_TYPES.MARKDOWN,
+    TEXT_MIME_TYPES.MARKDOWN_ALT,
+];
+/**
+ * Supported Markdown file extensions.
+ * Derived from the centralized MARKDOWN_EXTENSIONS config.
+ */
+const SUPPORTED_MARKDOWN_EXTENSIONS = MARKDOWN_EXTENSIONS;
+/** Default timeout for Markdown processing (30 seconds) */
+const MARKDOWN_TIMEOUT_MS = 30000;
+// =============================================================================
+// MARKDOWN PROCESSOR
+// =============================================================================
+/**
+ * Markdown Processor - processes Markdown files with structure analysis.
+ *
+ * This processor analyzes Markdown documents to extract structural metadata
+ * including headings, code blocks, and tables. The original content is
+ * preserved for AI processing.
+ *
+ * Priority: 40 (before JSON at 50, before generic text at 110)
+ *
+ * @example
+ * ```typescript
+ * const processor = new MarkdownProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'md-123',
+ *   name: 'README.md',
+ *   mimetype: 'text/markdown',
+ *   size: 4096,
+ *   url: 'https://example.com/README.md',
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Headings:', result.data.headings);
+ *   console.log('Has code blocks:', result.data.hasCodeBlocks);
+ * }
+ * ```
+ */
+export class MarkdownProcessor extends BaseFileProcessor {
+    constructor() {
+        super({
+            maxSizeMB: SIZE_LIMITS.TEXT_MAX_MB,
+            timeoutMs: MARKDOWN_TIMEOUT_MS,
+            supportedMimeTypes: [...SUPPORTED_MARKDOWN_TYPES],
+            supportedExtensions: [...SUPPORTED_MARKDOWN_EXTENSIONS],
+            fileTypeName: "Markdown",
+            defaultFilename: "document.md",
+        });
+    }
+    /**
+     * Validate downloaded Markdown file.
+     * Markdown is very permissive - almost any text is valid.
+     *
+     * @param buffer - Downloaded file content
+     * @param _fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    async validateDownloadedFile(buffer, _fileInfo) {
+        // Markdown is very permissive - any text is valid markdown
+        // We only check for completely empty files
+        if (buffer.length === 0) {
+            return "Invalid Markdown - file is empty";
+        }
+        // Check if the content appears to be binary
+        const content = buffer.toString("utf-8");
+        const nullByteIndex = content.indexOf("\0");
+        if (nullByteIndex !== -1 && nullByteIndex < 1000) {
+            return "Invalid Markdown - appears to be binary file";
+        }
+        return null;
+    }
+    /**
+     * Build processed Markdown result with structure analysis.
+     *
+     * Processing steps:
+     * 1. Preserve original content
+     * 2. Count lines
+     * 3. Detect fenced code blocks
+     * 4. Detect tables
+     * 5. Extract headings
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed Markdown result
+     */
+    buildProcessedResult(buffer, fileInfo) {
+        const content = buffer.toString("utf-8");
+        const filename = this.getFilename(fileInfo);
+        // Split into lines for analysis
+        const lines = content.split("\n");
+        // Extract headings (# lines at any level 1-6)
+        const headings = lines
+            .filter((line) => /^#{1,6}\s+/.test(line))
+            .map((line) => line.replace(/^#+\s+/, "").trim());
+        // Detect fenced code blocks (```)
+        const hasCodeBlocks = /```/.test(content);
+        // Detect Markdown tables (pipe-delimited rows with at least 2 pipes)
+        // Looking for patterns like: | col1 | col2 |
+        const hasTables = /\|.*\|.*\|/.test(content);
+        return {
+            content,
+            lineCount: lines.length,
+            hasCodeBlocks,
+            hasTables,
+            headings,
+            buffer,
+            mimetype: fileInfo.mimetype || "text/markdown",
+            size: fileInfo.size,
+            filename,
+        };
+    }
+}
+// =============================================================================
+// SINGLETON INSTANCE
+// =============================================================================
+/**
+ * Singleton Markdown processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { markdownProcessor } from "./markup/MarkdownProcessor.js";
+ *
+ * const result = await markdownProcessor.processFile(fileInfo);
+ * ```
+ */
+export const markdownProcessor = new MarkdownProcessor();
+// =============================================================================
+// HELPER FUNCTIONS
+// =============================================================================
+/**
+ * Check if a file is a Markdown file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is a Markdown file
+ *
+ * @example
+ * ```typescript
+ * if (isMarkdownFile('text/markdown', 'README.md')) {
+ *   // Handle as Markdown
+ * }
+ *
+ * // Also works with just filename
+ * if (isMarkdownFile('', 'CHANGELOG.markdown')) {
+ *   // Handle as Markdown based on extension
+ * }
+ * ```
+ */
+export function isMarkdownFile(mimetype, filename) {
+    return markdownProcessor.isFileSupported(mimetype, filename);
+}
+/**
+ * Validate Markdown file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit
+ *
+ * @example
+ * ```typescript
+ * if (!validateMarkdownSize(fileInfo.size)) {
+ *   console.error('Markdown file is too large');
+ * }
+ * ```
+ */
+export function validateMarkdownSize(sizeBytes) {
+    const maxBytes = SIZE_LIMITS.TEXT_MAX_MB * 1024 * 1024;
+    return sizeBytes <= maxBytes;
+}
+/**
+ * Process a single Markdown file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with Markdown content and structure analysis
+ *
+ * @example
+ * ```typescript
+ * const result = await processMarkdown({
+ *   id: 'md-123',
+ *   name: 'README.md',
+ *   mimetype: 'text/markdown',
+ *   size: 4096,
+ *   buffer: markdownBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Line count:', result.data.lineCount);
+ *   console.log('Headings:', result.data.headings);
+ *   if (result.data.hasCodeBlocks) {
+ *     console.log('Document contains code examples');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export async function processMarkdown(fileInfo, options) {
+    return markdownProcessor.processFile(fileInfo, options);
+}

package/dist/processors/markup/SvgProcessor.d.ts

@@ -0,0 +1,156 @@
+/**
+ * SVG File Processor
+ *
+ * Processes SVG files as TEXT content, not as images, because:
+ * 1. Most AI vision models don't support SVG format directly
+ * 2. SVG is XML-based and can be analyzed as code/markup
+ * 3. SVG can contain security risks (scripts, XSS vectors)
+ *
+ * Security: Uses OWASP-compliant allowlist-based SVG sanitization
+ *
+ * @module processors/markup/SvgProcessor
+ *
+ * @example
+ * ```typescript
+ * import { svgProcessor, processSvg, isSvgFile } from "./markup/SvgProcessor.js";
+ *
+ * // Check if file is SVG
+ * if (isSvgFile(mimetype, filename)) {
+ *   const result = await processSvg(fileInfo);
+ *   if (result.success) {
+ *     console.log('Sanitized SVG:', result.data.textContent);
+ *     if (result.data.securityWarnings.length > 0) {
+ *       console.warn('Security warnings:', result.data.securityWarnings);
+ *     }
+ *   }
+ * }
+ * ```
+ */
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import type { FileInfo, FileProcessingResult, ProcessOptions } from "../base/types.js";
+export type { ProcessedSvg } from "../base/types.js";
+import type { ProcessedSvg } from "../base/types.js";
+/**
+ * SVG Processor - processes SVG as TEXT, not as image.
+ *
+ * Why text instead of image:
+ * 1. Most AI vision models don't support SVG format
+ * 2. SVG is XML-based and can be analyzed as markup
+ * 3. Security: SVG can contain scripts and XSS vectors
+ *
+ * Priority: 5 (before IMAGE at priority 10)
+ *
+ * @example
+ * ```typescript
+ * const processor = new SvgProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'svg-123',
+ *   name: 'diagram.svg',
+ *   mimetype: 'image/svg+xml',
+ *   size: 2048,
+ *   url: 'https://example.com/diagram.svg',
+ * });
+ *
+ * if (result.success) {
+ *   // Use sanitized SVG text content
+ *   console.log(result.data.textContent);
+ * }
+ * ```
+ */
+export declare class SvgProcessor extends BaseFileProcessor<ProcessedSvg> {
+    constructor();
+    /**
+     * Validate downloaded SVG file.
+     * Checks for valid XML structure (SVG must contain <svg> element).
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    protected validateDownloadedFile(buffer: Buffer, _fileInfo: FileInfo): Promise<string | null>;
+    /**
+     * Build processed SVG result with sanitized content.
+     * Applies security sanitization to remove potentially malicious content.
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed SVG result with sanitized text content
+     */
+    protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedSvg;
+}
+/**
+ * Singleton SVG processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { svgProcessor } from "./markup/SvgProcessor.js";
+ *
+ * const result = await svgProcessor.processFile(fileInfo);
+ * ```
+ */
+export declare const svgProcessor: SvgProcessor;
+/**
+ * Check if a file is an SVG file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is an SVG
+ *
+ * @example
+ * ```typescript
+ * if (isSvgFile('image/svg+xml', 'diagram.svg')) {
+ *   // Handle as SVG
+ * }
+ *
+ * // Also works with just filename
+ * if (isSvgFile('', 'icon.svg')) {
+ *   // Handle as SVG based on extension
+ * }
+ * ```
+ */
+export declare function isSvgFile(mimetype: string, filename: string): boolean;
+/**
+ * Validate SVG file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit (5 MB)
+ *
+ * @example
+ * ```typescript
+ * if (!validateSvgSize(fileInfo.size)) {
+ *   console.error('SVG file is too large');
+ * }
+ * ```
+ */
+export declare function validateSvgSize(sizeBytes: number): boolean;
+/**
+ * Process a single SVG file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with sanitized SVG text content
+ *
+ * @example
+ * ```typescript
+ * const result = await processSvg({
+ *   id: 'svg-123',
+ *   name: 'diagram.svg',
+ *   mimetype: 'image/svg+xml',
+ *   size: 2048,
+ *   buffer: svgBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Processed SVG:', result.data.textContent);
+ *   if (result.data.sanitized) {
+ *     console.log('Content was sanitized for security');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export declare function processSvg(fileInfo: FileInfo, options?: ProcessOptions): Promise<FileProcessingResult<ProcessedSvg>>;

package/dist/processors/markup/SvgProcessor.js

@@ -0,0 +1,240 @@
+/**
+ * SVG File Processor
+ *
+ * Processes SVG files as TEXT content, not as images, because:
+ * 1. Most AI vision models don't support SVG format directly
+ * 2. SVG is XML-based and can be analyzed as code/markup
+ * 3. SVG can contain security risks (scripts, XSS vectors)
+ *
+ * Security: Uses OWASP-compliant allowlist-based SVG sanitization
+ *
+ * @module processors/markup/SvgProcessor
+ *
+ * @example
+ * ```typescript
+ * import { svgProcessor, processSvg, isSvgFile } from "./markup/SvgProcessor.js";
+ *
+ * // Check if file is SVG
+ * if (isSvgFile(mimetype, filename)) {
+ *   const result = await processSvg(fileInfo);
+ *   if (result.success) {
+ *     console.log('Sanitized SVG:', result.data.textContent);
+ *     if (result.data.securityWarnings.length > 0) {
+ *       console.warn('Security warnings:', result.data.securityWarnings);
+ *     }
+ *   }
+ * }
+ * ```
+ */
+import { isSvgContentSafe, sanitizeSvgContent, } from "../../utils/sanitizers/svg.js";
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+/** SVG MIME type */
+const SUPPORTED_SVG_TYPES = ["image/svg+xml"];
+/** SVG file extension */
+const SUPPORTED_SVG_EXTENSIONS = [".svg"];
+// =============================================================================
+// SVG PROCESSOR
+// =============================================================================
+/**
+ * SVG Processor - processes SVG as TEXT, not as image.
+ *
+ * Why text instead of image:
+ * 1. Most AI vision models don't support SVG format
+ * 2. SVG is XML-based and can be analyzed as markup
+ * 3. Security: SVG can contain scripts and XSS vectors
+ *
+ * Priority: 5 (before IMAGE at priority 10)
+ *
+ * @example
+ * ```typescript
+ * const processor = new SvgProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'svg-123',
+ *   name: 'diagram.svg',
+ *   mimetype: 'image/svg+xml',
+ *   size: 2048,
+ *   url: 'https://example.com/diagram.svg',
+ * });
+ *
+ * if (result.success) {
+ *   // Use sanitized SVG text content
+ *   console.log(result.data.textContent);
+ * }
+ * ```
+ */
+export class SvgProcessor extends BaseFileProcessor {
+    constructor() {
+        super({
+            maxSizeMB: 5,
+            timeoutMs: 30000,
+            supportedMimeTypes: [...SUPPORTED_SVG_TYPES],
+            supportedExtensions: [...SUPPORTED_SVG_EXTENSIONS],
+            fileTypeName: "SVG",
+            defaultFilename: "image.svg",
+        });
+    }
+    /**
+     * Validate downloaded SVG file.
+     * Checks for valid XML structure (SVG must contain <svg> element).
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    async validateDownloadedFile(buffer, _fileInfo) {
+        const content = buffer.toString("utf-8").trim();
+        // Check for valid SVG content
+        // Valid SVG can start with: <?xml, <!DOCTYPE, <svg, or just contain <svg
+        const startsWithXml = content.startsWith("<?xml");
+        const startsWithDoctype = content.toLowerCase().startsWith("<!doctype");
+        const startsWithSvg = content.toLowerCase().startsWith("<svg");
+        const containsSvgTag = content.toLowerCase().includes("<svg");
+        if (!startsWithXml &&
+            !startsWithDoctype &&
+            !startsWithSvg &&
+            !containsSvgTag) {
+            // Check if it might be HTML (download error page)
+            if (content.toLowerCase().includes("<html")) {
+                return "Download failed - received HTML instead of SVG content";
+            }
+            return "Invalid SVG - missing <svg> element";
+        }
+        return null;
+    }
+    /**
+     * Build processed SVG result with sanitized content.
+     * Applies security sanitization to remove potentially malicious content.
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed SVG result with sanitized text content
+     */
+    buildProcessedResult(buffer, fileInfo) {
+        const rawContent = buffer.toString("utf-8");
+        const filename = this.getFilename(fileInfo);
+        // Check if content is safe before sanitization
+        const wasSafe = isSvgContentSafe(rawContent);
+        // Build security warnings (initialized before try/catch so catch block can append)
+        const securityWarnings = [];
+        // Apply security sanitization using allowlist-based approach
+        let textContent;
+        try {
+            textContent = sanitizeSvgContent(rawContent);
+        }
+        catch {
+            // Fail closed: if sanitization fails (e.g., malformed XML with XXE),
+            // return a safe empty SVG instead of attempting regex-based cleanup.
+            // Regex cannot safely sanitize XML/HTML (context-free grammar).
+            textContent = '<svg xmlns="http://www.w3.org/2000/svg"></svg>';
+            securityWarnings.push("SVG sanitization failed - malformed content replaced with empty SVG for security");
+        }
+        if (!wasSafe) {
+            securityWarnings.push("SVG contained potentially unsafe content that was sanitized");
+        }
+        // Check if content was actually modified
+        const contentWasModified = textContent !== rawContent;
+        return {
+            textContent,
+            // Only include raw content if sanitization actually changed it (for debugging)
+            rawContent: contentWasModified ? rawContent : undefined,
+            sanitized: !wasSafe || contentWasModified,
+            securityWarnings,
+            buffer,
+            mimetype: fileInfo.mimetype || "image/svg+xml",
+            size: fileInfo.size,
+            filename,
+        };
+    }
+}
+// =============================================================================
+// SINGLETON INSTANCE
+// =============================================================================
+/**
+ * Singleton SVG processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { svgProcessor } from "./markup/SvgProcessor.js";
+ *
+ * const result = await svgProcessor.processFile(fileInfo);
+ * ```
+ */
+export const svgProcessor = new SvgProcessor();
+// =============================================================================
+// HELPER FUNCTIONS
+// =============================================================================
+/**
+ * Check if a file is an SVG file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is an SVG
+ *
+ * @example
+ * ```typescript
+ * if (isSvgFile('image/svg+xml', 'diagram.svg')) {
+ *   // Handle as SVG
+ * }
+ *
+ * // Also works with just filename
+ * if (isSvgFile('', 'icon.svg')) {
+ *   // Handle as SVG based on extension
+ * }
+ * ```
+ */
+export function isSvgFile(mimetype, filename) {
+    return svgProcessor.isFileSupported(mimetype, filename);
+}
+/**
+ * Validate SVG file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit (5 MB)
+ *
+ * @example
+ * ```typescript
+ * if (!validateSvgSize(fileInfo.size)) {
+ *   console.error('SVG file is too large');
+ * }
+ * ```
+ */
+export function validateSvgSize(sizeBytes) {
+    const maxBytes = 5 * 1024 * 1024; // 5 MB
+    return sizeBytes <= maxBytes;
+}
+/**
+ * Process a single SVG file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with sanitized SVG text content
+ *
+ * @example
+ * ```typescript
+ * const result = await processSvg({
+ *   id: 'svg-123',
+ *   name: 'diagram.svg',
+ *   mimetype: 'image/svg+xml',
+ *   size: 2048,
+ *   buffer: svgBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Processed SVG:', result.data.textContent);
+ *   if (result.data.sanitized) {
+ *     console.log('Content was sanitized for security');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export async function processSvg(fileInfo, options) {
+    return svgProcessor.processFile(fileInfo, options);
+}