@juspay/neurolink 9.1.1 → 9.2.0

package/dist/lib/processors/integration/index.js

@@ -0,0 +1,45 @@
+/**
+ * File Processor Integration Module
+ *
+ * Provides integration between the ProcessorRegistry and message building.
+ * Exports utilities for processing files through registered processors
+ * with automatic type detection and batch processing support.
+ *
+ * @module processors/integration
+ *
+ * @example
+ * ```typescript
+ * import {
+ *   // Single file processing
+ *   processFileWithRegistry,
+ *
+ *   // Batch processing
+ *   processBatchWithRegistry,
+ *
+ *   // Discovery utilities
+ *   getSupportedFileTypes,
+ *   isFileTypeSupported,
+ *   getProcessorForFile,
+ *
+ *   // Types
+ *   type FileProcessingOptions,
+ *   type BatchFileProcessingResult,
+ * } from "./integration/index.js";
+ *
+ * // Process a single file with auto-detection
+ * const { processorName, result } = await processFileWithRegistry(fileInfo);
+ *
+ * // Process multiple files
+ * const batchResult = await processBatchWithRegistry(files, { maxFiles: 50 });
+ *
+ * // Check supported types
+ * const supported = getSupportedFileTypes();
+ * const isSupported = isFileTypeSupported("application/pdf", "doc.pdf");
+ * const match = getProcessorForFile("image/jpeg", "photo.jpg");
+ * ```
+ */
+// =============================================================================
+// FUNCTION EXPORTS
+// =============================================================================
+export { getProcessorForFile, getSupportedFileTypes, isFileTypeSupported, processBatchWithRegistry, processFileWithRegistry, } from "./FileProcessorIntegration.js";
+//# sourceMappingURL=index.js.map

package/dist/lib/processors/markup/HtmlProcessor.d.ts

@@ -0,0 +1,169 @@
+/**
+ * HTML File Processor
+ *
+ * Processes HTML files with text extraction and security analysis.
+ * HTML files are processed as text content for AI analysis, with
+ * extraction of plain text content (tags stripped) for easier processing.
+ *
+ * Features:
+ * - Original HTML content preservation
+ * - Text extraction (all tags stripped)
+ * - Script and style tag detection
+ * - Title extraction
+ * - Security warnings for dangerous content
+ *
+ * Security: Uses OWASP-compliant HTML sanitization utilities
+ *
+ * @module processors/markup/HtmlProcessor
+ *
+ * @example
+ * ```typescript
+ * import { htmlProcessor, processHtml, isHtmlFile } from "./markup/HtmlProcessor.js";
+ *
+ * // Check if file is HTML
+ * if (isHtmlFile(mimetype, filename)) {
+ *   const result = await processHtml(fileInfo);
+ *   if (result.success) {
+ *     console.log('Text content:', result.data.textContent);
+ *     console.log('Has scripts:', result.data.hasScripts);
+ *     if (result.data.title) {
+ *       console.log('Page title:', result.data.title);
+ *     }
+ *   }
+ * }
+ * ```
+ */
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import type { FileInfo, FileProcessingResult, ProcessOptions } from "../base/types.js";
+export type { ProcessedHtml } from "../base/types.js";
+import type { ProcessedHtml } from "../base/types.js";
+/**
+ * HTML Processor - processes HTML files with text extraction.
+ *
+ * This processor extracts both the original HTML content and a plain text
+ * version with all tags stripped. It also performs security analysis to
+ * detect potentially dangerous content.
+ *
+ * Priority: 20 (after SVG at priority 5, before generic text)
+ *
+ * @example
+ * ```typescript
+ * const processor = new HtmlProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'html-123',
+ *   name: 'page.html',
+ *   mimetype: 'text/html',
+ *   size: 8192,
+ *   url: 'https://example.com/page.html',
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Title:', result.data.title);
+ *   console.log('Text content:', result.data.textContent);
+ * }
+ * ```
+ */
+export declare class HtmlProcessor extends BaseFileProcessor<ProcessedHtml> {
+    constructor();
+    /**
+     * Validate downloaded HTML file.
+     * Performs basic validation to ensure content appears to be HTML.
+     *
+     * @param buffer - Downloaded file content
+     * @param _fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    protected validateDownloadedFile(buffer: Buffer, _fileInfo: FileInfo): Promise<string | null>;
+    /**
+     * Build processed HTML result with text extraction.
+     *
+     * Processing steps:
+     * 1. Preserve original HTML content
+     * 2. Extract plain text (strip all tags)
+     * 3. Detect script and style tags
+     * 4. Extract page title if present
+     * 5. Check for dangerous content
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed HTML result
+     */
+    protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedHtml;
+}
+/**
+ * Singleton HTML processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { htmlProcessor } from "./markup/HtmlProcessor.js";
+ *
+ * const result = await htmlProcessor.processFile(fileInfo);
+ * ```
+ */
+export declare const htmlProcessor: HtmlProcessor;
+/**
+ * Check if a file is an HTML file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is an HTML file
+ *
+ * @example
+ * ```typescript
+ * if (isHtmlFile('text/html', 'page.html')) {
+ *   // Handle as HTML
+ * }
+ *
+ * // Also works with just filename
+ * if (isHtmlFile('', 'index.htm')) {
+ *   // Handle as HTML based on extension
+ * }
+ * ```
+ */
+export declare function isHtmlFile(mimetype: string, filename: string): boolean;
+/**
+ * Validate HTML file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit
+ *
+ * @example
+ * ```typescript
+ * if (!validateHtmlSize(fileInfo.size)) {
+ *   console.error('HTML file is too large');
+ * }
+ * ```
+ */
+export declare function validateHtmlSize(sizeBytes: number): boolean;
+/**
+ * Process a single HTML file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with HTML content and extracted text
+ *
+ * @example
+ * ```typescript
+ * const result = await processHtml({
+ *   id: 'html-123',
+ *   name: 'page.html',
+ *   mimetype: 'text/html',
+ *   size: 8192,
+ *   buffer: htmlBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Page title:', result.data.title);
+ *   console.log('Text content:', result.data.textContent);
+ *   if (result.data.hasDangerousContent) {
+ *     console.warn('HTML contains potentially dangerous content');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export declare function processHtml(fileInfo: FileInfo, options?: ProcessOptions): Promise<FileProcessingResult<ProcessedHtml>>;

package/dist/lib/processors/markup/HtmlProcessor.js

@@ -0,0 +1,250 @@
+/**
+ * HTML File Processor
+ *
+ * Processes HTML files with text extraction and security analysis.
+ * HTML files are processed as text content for AI analysis, with
+ * extraction of plain text content (tags stripped) for easier processing.
+ *
+ * Features:
+ * - Original HTML content preservation
+ * - Text extraction (all tags stripped)
+ * - Script and style tag detection
+ * - Title extraction
+ * - Security warnings for dangerous content
+ *
+ * Security: Uses OWASP-compliant HTML sanitization utilities
+ *
+ * @module processors/markup/HtmlProcessor
+ *
+ * @example
+ * ```typescript
+ * import { htmlProcessor, processHtml, isHtmlFile } from "./markup/HtmlProcessor.js";
+ *
+ * // Check if file is HTML
+ * if (isHtmlFile(mimetype, filename)) {
+ *   const result = await processHtml(fileInfo);
+ *   if (result.success) {
+ *     console.log('Text content:', result.data.textContent);
+ *     console.log('Has scripts:', result.data.hasScripts);
+ *     if (result.data.title) {
+ *       console.log('Page title:', result.data.title);
+ *     }
+ *   }
+ * }
+ * ```
+ */
+import { containsDangerousHtml, stripHtmlTags, } from "../../utils/sanitizers/html.js";
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import { SIZE_LIMITS } from "../config/index.js";
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+/** Supported HTML MIME types */
+const SUPPORTED_HTML_TYPES = ["text/html", "application/xhtml+xml"];
+/** Supported HTML file extensions */
+const SUPPORTED_HTML_EXTENSIONS = [".html", ".htm", ".xhtml"];
+/** Default timeout for HTML processing (30 seconds) */
+const HTML_TIMEOUT_MS = 30000;
+// =============================================================================
+// HTML PROCESSOR
+// =============================================================================
+/**
+ * HTML Processor - processes HTML files with text extraction.
+ *
+ * This processor extracts both the original HTML content and a plain text
+ * version with all tags stripped. It also performs security analysis to
+ * detect potentially dangerous content.
+ *
+ * Priority: 20 (after SVG at priority 5, before generic text)
+ *
+ * @example
+ * ```typescript
+ * const processor = new HtmlProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'html-123',
+ *   name: 'page.html',
+ *   mimetype: 'text/html',
+ *   size: 8192,
+ *   url: 'https://example.com/page.html',
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Title:', result.data.title);
+ *   console.log('Text content:', result.data.textContent);
+ * }
+ * ```
+ */
+export class HtmlProcessor extends BaseFileProcessor {
+    constructor() {
+        super({
+            maxSizeMB: SIZE_LIMITS.TEXT_MAX_MB,
+            timeoutMs: HTML_TIMEOUT_MS,
+            supportedMimeTypes: [...SUPPORTED_HTML_TYPES],
+            supportedExtensions: [...SUPPORTED_HTML_EXTENSIONS],
+            fileTypeName: "HTML",
+            defaultFilename: "page.html",
+        });
+    }
+    /**
+     * Validate downloaded HTML file.
+     * Performs basic validation to ensure content appears to be HTML.
+     *
+     * @param buffer - Downloaded file content
+     * @param _fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    async validateDownloadedFile(buffer, _fileInfo) {
+        const content = buffer.toString("utf-8").trim();
+        // Check minimum size
+        if (content.length === 0) {
+            return "Invalid HTML - file is empty";
+        }
+        // Very basic HTML detection - must contain at least one tag
+        // We're lenient here because HTML can be quite varied
+        const hasHtmlContent = content.includes("<") ||
+            content.toLowerCase().includes("<!doctype") ||
+            content.toLowerCase().includes("<html") ||
+            content.toLowerCase().includes("<body") ||
+            content.toLowerCase().includes("<head");
+        if (!hasHtmlContent) {
+            return "Invalid HTML - no HTML content detected";
+        }
+        return null;
+    }
+    /**
+     * Build processed HTML result with text extraction.
+     *
+     * Processing steps:
+     * 1. Preserve original HTML content
+     * 2. Extract plain text (strip all tags)
+     * 3. Detect script and style tags
+     * 4. Extract page title if present
+     * 5. Check for dangerous content
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed HTML result
+     */
+    buildProcessedResult(buffer, fileInfo) {
+        const content = buffer.toString("utf-8");
+        const filename = this.getFilename(fileInfo);
+        // Extract text content (strip all tags)
+        const textContent = stripHtmlTags(content);
+        // Check for script and style tags
+        const hasScripts = /<script[\s>]/i.test(content);
+        const hasStyles = /<style[\s>]/i.test(content);
+        // Extract title if present
+        const titleMatch = content.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+        const title = titleMatch ? titleMatch[1].trim() : undefined;
+        // Check for dangerous content (XSS vectors)
+        const hasDangerousContent = containsDangerousHtml(content);
+        // Build base result
+        const result = {
+            content,
+            textContent,
+            hasScripts,
+            hasStyles,
+            hasDangerousContent,
+            buffer,
+            mimetype: fileInfo.mimetype || "text/html",
+            size: fileInfo.size,
+            filename,
+        };
+        // Only include title if it was found (avoid undefined property with exactOptionalPropertyTypes)
+        if (title) {
+            result.title = title;
+        }
+        return result;
+    }
+}
+// =============================================================================
+// SINGLETON INSTANCE
+// =============================================================================
+/**
+ * Singleton HTML processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { htmlProcessor } from "./markup/HtmlProcessor.js";
+ *
+ * const result = await htmlProcessor.processFile(fileInfo);
+ * ```
+ */
+export const htmlProcessor = new HtmlProcessor();
+// =============================================================================
+// HELPER FUNCTIONS
+// =============================================================================
+/**
+ * Check if a file is an HTML file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is an HTML file
+ *
+ * @example
+ * ```typescript
+ * if (isHtmlFile('text/html', 'page.html')) {
+ *   // Handle as HTML
+ * }
+ *
+ * // Also works with just filename
+ * if (isHtmlFile('', 'index.htm')) {
+ *   // Handle as HTML based on extension
+ * }
+ * ```
+ */
+export function isHtmlFile(mimetype, filename) {
+    return htmlProcessor.isFileSupported(mimetype, filename);
+}
+/**
+ * Validate HTML file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit
+ *
+ * @example
+ * ```typescript
+ * if (!validateHtmlSize(fileInfo.size)) {
+ *   console.error('HTML file is too large');
+ * }
+ * ```
+ */
+export function validateHtmlSize(sizeBytes) {
+    const maxBytes = SIZE_LIMITS.TEXT_MAX_MB * 1024 * 1024;
+    return sizeBytes <= maxBytes;
+}
+/**
+ * Process a single HTML file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with HTML content and extracted text
+ *
+ * @example
+ * ```typescript
+ * const result = await processHtml({
+ *   id: 'html-123',
+ *   name: 'page.html',
+ *   mimetype: 'text/html',
+ *   size: 8192,
+ *   buffer: htmlBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Page title:', result.data.title);
+ *   console.log('Text content:', result.data.textContent);
+ *   if (result.data.hasDangerousContent) {
+ *     console.warn('HTML contains potentially dangerous content');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export async function processHtml(fileInfo, options) {
+    return htmlProcessor.processFile(fileInfo, options);
+}
+//# sourceMappingURL=HtmlProcessor.js.map

package/dist/lib/processors/markup/MarkdownProcessor.d.ts

@@ -0,0 +1,165 @@
+/**
+ * Markdown File Processor
+ *
+ * Processes Markdown files with structure extraction and analysis.
+ * Markdown files are analyzed to extract metadata about their structure
+ * including headings, code blocks, and tables.
+ *
+ * Features:
+ * - Original content preservation
+ * - Line count calculation
+ * - Code block detection
+ * - Table detection
+ * - Heading extraction (all levels)
+ *
+ * @module processors/markup/MarkdownProcessor
+ *
+ * @example
+ * ```typescript
+ * import { markdownProcessor, processMarkdown, isMarkdownFile } from "./markup/MarkdownProcessor.js";
+ *
+ * // Check if file is Markdown
+ * if (isMarkdownFile(mimetype, filename)) {
+ *   const result = await processMarkdown(fileInfo);
+ *   if (result.success) {
+ *     console.log('Line count:', result.data.lineCount);
+ *     console.log('Headings:', result.data.headings);
+ *     console.log('Has code blocks:', result.data.hasCodeBlocks);
+ *   }
+ * }
+ * ```
+ */
+import { BaseFileProcessor } from "../base/BaseFileProcessor.js";
+import type { FileInfo, FileProcessingResult, ProcessOptions } from "../base/types.js";
+export type { ProcessedMarkdown } from "../base/types.js";
+import type { ProcessedMarkdown } from "../base/types.js";
+/**
+ * Markdown Processor - processes Markdown files with structure analysis.
+ *
+ * This processor analyzes Markdown documents to extract structural metadata
+ * including headings, code blocks, and tables. The original content is
+ * preserved for AI processing.
+ *
+ * Priority: 40 (before JSON at 50, before generic text at 110)
+ *
+ * @example
+ * ```typescript
+ * const processor = new MarkdownProcessor();
+ *
+ * const result = await processor.processFile({
+ *   id: 'md-123',
+ *   name: 'README.md',
+ *   mimetype: 'text/markdown',
+ *   size: 4096,
+ *   url: 'https://example.com/README.md',
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Headings:', result.data.headings);
+ *   console.log('Has code blocks:', result.data.hasCodeBlocks);
+ * }
+ * ```
+ */
+export declare class MarkdownProcessor extends BaseFileProcessor<ProcessedMarkdown> {
+    constructor();
+    /**
+     * Validate downloaded Markdown file.
+     * Markdown is very permissive - almost any text is valid.
+     *
+     * @param buffer - Downloaded file content
+     * @param _fileInfo - Original file information
+     * @returns null if valid, error message if invalid
+     */
+    protected validateDownloadedFile(buffer: Buffer, _fileInfo: FileInfo): Promise<string | null>;
+    /**
+     * Build processed Markdown result with structure analysis.
+     *
+     * Processing steps:
+     * 1. Preserve original content
+     * 2. Count lines
+     * 3. Detect fenced code blocks
+     * 4. Detect tables
+     * 5. Extract headings
+     *
+     * @param buffer - Downloaded file content
+     * @param fileInfo - Original file information
+     * @returns Processed Markdown result
+     */
+    protected buildProcessedResult(buffer: Buffer, fileInfo: FileInfo): ProcessedMarkdown;
+}
+/**
+ * Singleton Markdown processor instance.
+ * Use this for most processing needs.
+ *
+ * @example
+ * ```typescript
+ * import { markdownProcessor } from "./markup/MarkdownProcessor.js";
+ *
+ * const result = await markdownProcessor.processFile(fileInfo);
+ * ```
+ */
+export declare const markdownProcessor: MarkdownProcessor;
+/**
+ * Check if a file is a Markdown file.
+ *
+ * @param mimetype - MIME type of the file
+ * @param filename - Filename (for extension-based detection)
+ * @returns true if the file is a Markdown file
+ *
+ * @example
+ * ```typescript
+ * if (isMarkdownFile('text/markdown', 'README.md')) {
+ *   // Handle as Markdown
+ * }
+ *
+ * // Also works with just filename
+ * if (isMarkdownFile('', 'CHANGELOG.markdown')) {
+ *   // Handle as Markdown based on extension
+ * }
+ * ```
+ */
+export declare function isMarkdownFile(mimetype: string, filename: string): boolean;
+/**
+ * Validate Markdown file size against configured limit.
+ *
+ * @param sizeBytes - File size in bytes
+ * @returns true if size is within the allowed limit
+ *
+ * @example
+ * ```typescript
+ * if (!validateMarkdownSize(fileInfo.size)) {
+ *   console.error('Markdown file is too large');
+ * }
+ * ```
+ */
+export declare function validateMarkdownSize(sizeBytes: number): boolean;
+/**
+ * Process a single Markdown file.
+ * Convenience function that uses the singleton processor.
+ *
+ * @param fileInfo - File information (can include URL or buffer)
+ * @param options - Optional processing options (auth headers, timeout, retry config)
+ * @returns Processing result with Markdown content and structure analysis
+ *
+ * @example
+ * ```typescript
+ * const result = await processMarkdown({
+ *   id: 'md-123',
+ *   name: 'README.md',
+ *   mimetype: 'text/markdown',
+ *   size: 4096,
+ *   buffer: markdownBuffer,
+ * });
+ *
+ * if (result.success) {
+ *   console.log('Line count:', result.data.lineCount);
+ *   console.log('Headings:', result.data.headings);
+ *   if (result.data.hasCodeBlocks) {
+ *     console.log('Document contains code examples');
+ *   }
+ * } else {
+ *   console.error('Processing failed:', result.error.userMessage);
+ * }
+ * ```
+ */
+export declare function processMarkdown(fileInfo: FileInfo, options?: ProcessOptions): Promise<FileProcessingResult<ProcessedMarkdown>>;