@fin.cx/einvoice 5.0.2

package/ts/formats/pdf/extractors/base.extractor.ts

@@ -0,0 +1,355 @@
+import { PDFDocument, PDFDict, PDFName, PDFRawStream, PDFArray, PDFString, pako } from '../../../plugins.js';
+
+/**
+ * Base class for PDF XML extractors with common functionality
+ */
+export abstract class BaseXMLExtractor {
+  /**
+   * Known XML file names for different invoice formats
+   */
+  protected readonly knownFileNames = [
+    'factur-x.xml',
+    'zugferd-invoice.xml',
+    'ZUGFeRD-invoice.xml',
+    'xrechnung.xml',
+    'ubl-invoice.xml',
+    'invoice.xml',
+    'metadata.xml'
+  ];
+
+  /**
+   * Known XML formats to validate extracted content
+   */
+  protected readonly knownFormats = [
+    'CrossIndustryInvoice',
+    'CrossIndustryDocument',
+    'Invoice',
+    'CreditNote',
+    'ubl:Invoice',
+    'ubl:CreditNote',
+    'rsm:CrossIndustryInvoice',
+    'rsm:CrossIndustryDocument',
+    'ram:CrossIndustryDocument',
+    'urn:un:unece:uncefact',
+    'urn:ferd:CrossIndustryDocument',
+    'urn:zugferd',
+    'urn:factur-x',
+    'factur-x.eu',
+    'ZUGFeRD',
+    'FatturaElettronica'
+  ];
+
+  /**
+   * Known XML end tags for extracting content from strings
+   */
+  protected readonly knownEndTags = [
+    '</CrossIndustryInvoice>',
+    '</CrossIndustryDocument>',
+    '</Invoice>',
+    '</CreditNote>',
+    '</rsm:CrossIndustryInvoice>',
+    '</rsm:CrossIndustryDocument>',
+    '</ram:CrossIndustryDocument>',
+    '</ubl:Invoice>',
+    '</ubl:CreditNote>',
+    '</FatturaElettronica>'
+  ];
+
+  /**
+   * Extract XML from a PDF buffer
+   * @param pdfBuffer PDF buffer
+   * @returns XML content or null if not found
+   */
+  public abstract extractXml(pdfBuffer: Uint8Array | Buffer): Promise<string | null>;
+
+  /**
+   * Check if an XML string is valid
+   * @param xmlString XML string to check
+   * @returns True if the XML is valid
+   */
+  protected isValidXml(xmlString: string): boolean {
+    try {
+      // Basic checks for XML validity
+      if (!xmlString || typeof xmlString !== 'string') {
+        return false;
+      }
+
+      // Check if it starts with XML declaration or a valid element
+      if (!xmlString.includes('<?xml') && !this.hasKnownXmlElement(xmlString)) {
+        return false;
+      }
+
+      // Check if the XML string contains known invoice formats
+      const hasKnownFormat = this.hasKnownFormat(xmlString);
+      if (!hasKnownFormat) {
+        return false;
+      }
+
+      // Check if the XML string contains binary data or invalid characters
+      if (this.hasBinaryData(xmlString)) {
+        return false;
+      }
+
+      // Check if the XML string is too short
+      if (xmlString.length < 100) {
+        return false;
+      }
+
+      // Check if XML has a proper structure (contains both opening and closing tags)
+      if (!this.hasProperXmlStructure(xmlString)) {
+        return false;
+      }
+
+      return true;
+    } catch (error) {
+      console.error('Error validating XML:', error);
+      return false;
+    }
+  }
+
+  /**
+   * Check if the XML string contains a known element
+   * @param xmlString XML string to check
+   * @returns True if the XML contains a known element
+   */
+  protected hasKnownXmlElement(xmlString: string): boolean {
+    for (const format of this.knownFormats) {
+      // Check for opening tag of format
+      if (xmlString.includes(`<${format}`)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if the XML string contains a known format
+   * @param xmlString XML string to check
+   * @returns True if the XML contains a known format
+   */
+  protected hasKnownFormat(xmlString: string): boolean {
+    for (const format of this.knownFormats) {
+      if (xmlString.includes(format)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if the XML string has a proper structure
+   * @param xmlString XML string to check
+   * @returns True if the XML has a proper structure
+   */
+  protected hasProperXmlStructure(xmlString: string): boolean {
+    // Check for at least one matching opening and closing tag
+    for (const endTag of this.knownEndTags) {
+      const startTag = endTag.replace('/', '');
+      if (xmlString.includes(startTag) && xmlString.includes(endTag)) {
+        return true;
+      }
+    }
+    
+    // If no specific tag is found but it has a basic XML structure
+    return (
+      (xmlString.includes('<?xml') && xmlString.includes('?>')) ||
+      (xmlString.match(/<[^>]+>/) !== null && xmlString.match(/<\/[^>]+>/) !== null)
+    );
+  }
+
+  /**
+   * Check if the XML string contains binary data
+   * @param xmlString XML string to check
+   * @returns True if the XML contains binary data
+   */
+  protected hasBinaryData(xmlString: string): boolean {
+    // Check for common binary data indicators
+    const binaryChars = ['\u0000', '\u0001', '\u0002', '\u0003', '\u0004', '\u0005'];
+    const consecutiveNulls = '\u0000\u0000\u0000';
+    
+    // Check for control characters that shouldn't be in XML
+    if (binaryChars.some(char => xmlString.includes(char))) {
+      return true;
+    }
+    
+    // Check for consecutive null bytes which indicate binary data
+    if (xmlString.includes(consecutiveNulls)) {
+      return true;
+    }
+    
+    // Check for high concentration of non-printable characters
+    const nonPrintableCount = (xmlString.match(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g) || []).length;
+    if (nonPrintableCount > xmlString.length * 0.05) { // More than 5% non-printable
+      return true;
+    }
+    
+    return false;
+  }
+
+  /**
+   * Extract XML from a string
+   * @param text Text to extract XML from
+   * @param startIndex Index to start extraction from
+   * @returns XML content or null if not found
+   */
+  protected extractXmlFromString(text: string, startIndex: number = 0): string | null {
+    try {
+      // Find the start of the XML document
+      let xmlStartIndex = text.indexOf('<?xml', startIndex);
+      
+      // If no XML declaration, try to find known elements
+      if (xmlStartIndex === -1) {
+        for (const format of this.knownFormats) {
+          const formatStartIndex = text.indexOf(`<${format.split(':').pop()}`, startIndex);
+          if (formatStartIndex !== -1) {
+            xmlStartIndex = formatStartIndex;
+            break;
+          }
+        }
+        
+        // Still didn't find any start marker
+        if (xmlStartIndex === -1) {
+          return null;
+        }
+      }
+
+      // Try to find the end of the XML document
+      let xmlEndIndex = -1;
+      for (const endTag of this.knownEndTags) {
+        const endIndex = text.indexOf(endTag, xmlStartIndex);
+        if (endIndex !== -1) {
+          xmlEndIndex = endIndex + endTag.length;
+          break;
+        }
+      }
+
+      // If no known end tag found, try to use a heuristic approach
+      if (xmlEndIndex === -1) {
+        // Try to find the last closing tag
+        const lastClosingTagMatch = text.slice(xmlStartIndex).match(/<\/[^>]+>(?!.*<\/[^>]+>)/);
+        if (lastClosingTagMatch && lastClosingTagMatch.index !== undefined) {
+          xmlEndIndex = xmlStartIndex + lastClosingTagMatch.index + lastClosingTagMatch[0].length;
+        } else {
+          return null;
+        }
+      }
+
+      // Extract the XML content
+      const xmlContent = text.substring(xmlStartIndex, xmlEndIndex);
+      
+      // Validate the extracted content
+      if (this.isValidXml(xmlContent)) {
+        return xmlContent;
+      }
+      
+      return null;
+    } catch (error) {
+      console.error('Error extracting XML from string:', error);
+      return null;
+    }
+  }
+
+  /**
+   * Decompress and decode XML content from a PDF stream
+   * @param stream PDF stream containing XML data
+   * @param fileName Name of the file (for logging)
+   * @returns XML content or null if not valid
+   */
+  protected async extractXmlFromStream(stream: PDFRawStream, fileName: string): Promise<string | null> {
+    try {
+      // Get the raw bytes from the stream
+      const rawBytes = stream.getContents();
+      
+      // First try without decompression (in case the content is not compressed)
+      let xmlContent = this.tryDecodeBuffer(rawBytes);
+      if (xmlContent && this.isValidXml(xmlContent)) {
+        console.log(`Successfully extracted uncompressed XML from PDF file. File name: ${fileName}`);
+        return xmlContent;
+      }
+      
+      // Try with decompression
+      try {
+        const decompressedBytes = this.tryDecompress(rawBytes);
+        if (decompressedBytes) {
+          xmlContent = this.tryDecodeBuffer(decompressedBytes);
+          if (xmlContent && this.isValidXml(xmlContent)) {
+            console.log(`Successfully extracted decompressed XML from PDF file. File name: ${fileName}`);
+            return xmlContent;
+          }
+        }
+      } catch (decompressError) {
+        console.log(`Decompression failed for ${fileName}: ${decompressError}`);
+      }
+      
+      return null;
+    } catch (error) {
+      console.error('Error extracting XML from stream:', error);
+      return null;
+    }
+  }
+  
+  /**
+   * Try to decompress a buffer using different methods
+   * @param buffer Buffer to decompress
+   * @returns Decompressed buffer or null if decompression failed
+   */
+  protected tryDecompress(buffer: Uint8Array): Uint8Array | null {
+    try {
+      // Try pako inflate (for deflate/zlib compression)
+      return pako.inflate(buffer);
+    } catch (error) {
+      // If pako fails, try other methods if needed
+      console.warn('Pako decompression failed, might be uncompressed or using a different algorithm');
+      return null;
+    }
+  }
+  
+  /**
+   * Try to decode a buffer to a string using different encodings
+   * @param buffer Buffer to decode
+   * @returns Decoded string or null if decoding failed
+   */
+  protected tryDecodeBuffer(buffer: Uint8Array): string | null {
+    try {
+      // Try UTF-8 first
+      let content = new TextDecoder('utf-8').decode(buffer);
+      if (this.isPlausibleXml(content)) {
+        return content;
+      }
+      
+      // Try ISO-8859-1 (Latin1)
+      content = this.decodeLatin1(buffer);
+      if (this.isPlausibleXml(content)) {
+        return content;
+      }
+      
+      return null;
+    } catch (error) {
+      console.warn('Error decoding buffer:', error);
+      return null;
+    }
+  }
+  
+  /**
+   * Decode a buffer using ISO-8859-1 (Latin1) encoding
+   * @param buffer Buffer to decode
+   * @returns Decoded string
+   */
+  protected decodeLatin1(buffer: Uint8Array): string {
+    return Array.from(buffer)
+      .map(byte => String.fromCharCode(byte))
+      .join('');
+  }
+  
+  /**
+   * Check if a string is plausibly XML (quick check before validation)
+   * @param content String to check
+   * @returns True if the string is plausibly XML
+   */
+  protected isPlausibleXml(content: string): boolean {
+    return content.includes('<') && 
+           content.includes('>') && 
+           (content.includes('<?xml') || 
+            this.knownFormats.some(format => content.includes(format)));
+  }
+}

package/ts/formats/pdf/extractors/index.ts

@@ -0,0 +1,4 @@
+export * from './base.extractor.js';
+export * from './standard.extractor.js';
+export * from './associated.extractor.js';
+export * from './text.extractor.js';

package/ts/formats/pdf/extractors/standard.extractor.ts

@@ -0,0 +1,86 @@
+import { PDFDocument, PDFDict, PDFName, PDFRawStream, PDFArray, PDFString } from '../../../plugins.js';
+import { BaseXMLExtractor } from './base.extractor.js';
+
+/**
+ * Standard PDF XML extractor that extracts XML from embedded files
+ * Works with PDF/A-3 documents that follow the standard for embedding files
+ */
+export class StandardXMLExtractor extends BaseXMLExtractor {
+  /**
+   * Extract XML from a PDF buffer using standard PDF/A-3 embedded files
+   * @param pdfBuffer PDF buffer
+   * @returns XML content or null if not found
+   */
+  public async extractXml(pdfBuffer: Uint8Array | Buffer): Promise<string | null> {
+    try {
+      const pdfDoc = await PDFDocument.load(pdfBuffer);
+
+      // Get the document's metadata dictionary
+      const namesDictObj = pdfDoc.catalog.lookup(PDFName.of('Names'));
+      if (!(namesDictObj instanceof PDFDict)) {
+        console.warn('No Names dictionary found in PDF! This PDF does not contain embedded files.');
+        return null;
+      }
+
+      // Get the embedded files dictionary
+      const embeddedFilesDictObj = namesDictObj.lookup(PDFName.of('EmbeddedFiles'));
+      if (!(embeddedFilesDictObj instanceof PDFDict)) {
+        console.warn('No EmbeddedFiles dictionary found! This PDF does not contain embedded files.');
+        return null;
+      }
+
+      // Get the names array
+      const filesSpecObj = embeddedFilesDictObj.lookup(PDFName.of('Names'));
+      if (!(filesSpecObj instanceof PDFArray)) {
+        console.warn('No files specified in EmbeddedFiles dictionary!');
+        return null;
+      }
+
+      // Try to find an XML file in the embedded files
+      for (let i = 0; i < filesSpecObj.size(); i += 2) {
+        const fileNameObj = filesSpecObj.lookup(i);
+        const fileSpecObj = filesSpecObj.lookup(i + 1);
+
+        if (!(fileNameObj instanceof PDFString) || !(fileSpecObj instanceof PDFDict)) {
+          continue;
+        }
+
+        // Get the filename as string
+        const fileName = fileNameObj.decodeText();
+
+        // Check if it's a known invoice XML file name
+        const isKnownFileName = this.knownFileNames.some(
+          knownName => fileName.toLowerCase() === knownName.toLowerCase()
+        );
+
+        // Check if it's any XML file or has invoice-related keywords
+        const isXmlFile = fileName.toLowerCase().endsWith('.xml') ||
+                          fileName.toLowerCase().includes('zugferd') ||
+                          fileName.toLowerCase().includes('factur-x') ||
+                          fileName.toLowerCase().includes('xrechnung') ||
+                          fileName.toLowerCase().includes('invoice');
+
+        if (isKnownFileName || isXmlFile) {
+          const efDictObj = fileSpecObj.lookup(PDFName.of('EF'));
+          if (!(efDictObj instanceof PDFDict)) {
+            continue;
+          }
+
+          const fileStream = efDictObj.lookup(PDFName.of('F'));
+          if (fileStream instanceof PDFRawStream) {
+            const xmlContent = await this.extractXmlFromStream(fileStream, fileName);
+            if (xmlContent) {
+              return xmlContent;
+            }
+          }
+        }
+      }
+
+      console.warn('No valid XML found in embedded files');
+      return null;
+    } catch (error) {
+      console.error('Error in standard extraction:', error);
+      return null;
+    }
+  }
+}

package/ts/formats/pdf/extractors/text.extractor.ts

@@ -0,0 +1,162 @@
+import { BaseXMLExtractor } from './base.extractor.js';
+
+/**
+ * Text-based XML extractor for PDF documents
+ * Extracts XML by searching for XML patterns in the PDF text
+ * Used as a fallback when other extraction methods fail
+ */
+export class TextXMLExtractor extends BaseXMLExtractor {
+  // Maximum chunk size to process at once (4MB)
+  private readonly CHUNK_SIZE = 4 * 1024 * 1024;
+  
+  // Maximum number of chunks to check (effective 20MB search limit)
+  private readonly MAX_CHUNKS = 5;
+  
+  // Common XML patterns to look for
+  private readonly XML_PATTERNS = [
+    '<?xml', 
+    '<CrossIndustryInvoice',
+    '<CrossIndustryDocument',
+    '<Invoice',
+    '<CreditNote',
+    '<rsm:CrossIndustryInvoice',
+    '<rsm:CrossIndustryDocument',
+    '<ram:CrossIndustryDocument',
+    '<ubl:Invoice',
+    '<ubl:CreditNote',
+    '<FatturaElettronica'
+  ];
+
+  /**
+   * Extract XML from a PDF buffer by searching for XML patterns in the text
+   * Uses a chunked approach to handle large files efficiently
+   * @param pdfBuffer PDF buffer
+   * @returns XML content or null if not found
+   */
+  public async extractXml(pdfBuffer: Uint8Array | Buffer): Promise<string | null> {
+    try {
+      console.log('Attempting text-based XML extraction from PDF...');
+      
+      // Convert Buffer to Uint8Array if needed
+      const buffer = Buffer.isBuffer(pdfBuffer) ? new Uint8Array(pdfBuffer) : pdfBuffer;
+      
+      // Try extracting XML using the chunked approach
+      return this.extractXmlFromBufferChunked(buffer);
+    } catch (error) {
+      console.error('Error in text-based extraction:', error);
+      return null;
+    }
+  }
+
+  /**
+   * Extract XML from buffer using a chunked approach
+   * This helps avoid memory issues with large PDFs
+   * @param buffer Buffer to search in
+   * @returns XML content or null if not found
+   */
+  private extractXmlFromBufferChunked(buffer: Uint8Array): string | null {
+    // Process the PDF in chunks
+    for (let chunkIndex = 0; chunkIndex < this.MAX_CHUNKS; chunkIndex++) {
+      const startPos = chunkIndex * this.CHUNK_SIZE;
+      if (startPos >= buffer.length) break;
+      
+      const endPos = Math.min(startPos + this.CHUNK_SIZE, buffer.length);
+      const chunk = buffer.slice(startPos, endPos);
+      
+      // Try to extract XML from this chunk
+      const chunkResult = this.processChunk(chunk, startPos);
+      if (chunkResult) {
+        return chunkResult;
+      }
+    }
+    
+    console.warn('No valid XML found in any chunk of the PDF');
+    return null;
+  }
+
+  /**
+   * Process a single chunk of the PDF buffer
+   * @param chunk Chunk buffer to process
+   * @param chunkOffset Offset position of the chunk in the original buffer
+   * @returns XML content or null if not found
+   */
+  private processChunk(chunk: Uint8Array, chunkOffset: number): string | null {
+    try {
+      // First try UTF-8 encoding for this chunk
+      const utf8String = this.decodeBufferToString(chunk, 'utf-8');
+      let xmlContent = this.searchForXmlInString(utf8String);
+      
+      if (xmlContent) {
+        console.log(`Found XML content in chunk at offset ${chunkOffset} using UTF-8 encoding`);
+        return xmlContent;
+      }
+      
+      // If UTF-8 fails, try Latin-1 (ISO-8859-1) which can handle binary better
+      const latin1String = this.decodeBufferToString(chunk, 'latin1');
+      xmlContent = this.searchForXmlInString(latin1String);
+      
+      if (xmlContent) {
+        console.log(`Found XML content in chunk at offset ${chunkOffset} using Latin-1 encoding`);
+        return xmlContent;
+      }
+      
+      // No XML found in this chunk
+      return null;
+    } catch (error) {
+      console.warn(`Error processing chunk at offset ${chunkOffset}:`, error);
+      return null;
+    }
+  }
+
+  /**
+   * Safely decode a buffer to string using the specified encoding
+   * @param buffer Buffer to decode
+   * @param encoding Encoding to use ('utf-8' or 'latin1')
+   * @returns Decoded string
+   */
+  private decodeBufferToString(buffer: Uint8Array, encoding: 'utf-8' | 'latin1'): string {
+    try {
+      if (encoding === 'utf-8') {
+        return new TextDecoder('utf-8', { fatal: false }).decode(buffer);
+      } else {
+        // For Latin-1 we can use a direct mapping (bytes 0-255 map directly to code points 0-255)
+        // This is more reliable for binary data than TextDecoder for legacy encodings
+        return Array.from(buffer)
+          .map(byte => String.fromCharCode(byte))
+          .join('');
+      }
+    } catch (error) {
+      console.warn(`Error decoding buffer using ${encoding}:`, error);
+      // Return empty string on error to allow processing to continue
+      return '';
+    }
+  }
+
+  /**
+   * Search for XML patterns in a string
+   * @param content String to search in
+   * @returns XML content or null if not found
+   */
+  private searchForXmlInString(content: string): string | null {
+    if (!content) return null;
+    
+    // Search for each XML pattern
+    for (const pattern of this.XML_PATTERNS) {
+      const patternIndex = content.indexOf(pattern);
+      if (patternIndex !== -1) {
+        console.log(`Found XML pattern "${pattern}" at position ${patternIndex}`);
+        
+        // Try to extract the XML content starting from the pattern position
+        const xmlContent = this.extractXmlFromString(content, patternIndex);
+        
+        // Validate the extracted content
+        if (xmlContent && this.isValidXml(xmlContent)) {
+          console.log('Successfully extracted and validated XML from text');
+          return xmlContent;
+        }
+      }
+    }
+    
+    return null;
+  }
+}