mindee 3.13.0 → 3.15.0

data/lib/mindee/pdf/pdf_compressor.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'pdf-reader'
+PDFReader = PDF
+
+module Mindee
+  module PDF
+    # Image compressor module to handle PDF compression.
+    module PDFCompressor
+      # Compresses each page of a provided PDF stream. Skips if force_source_text isn't set and source text is detected.
+      # @param quality [Integer] Compression quality (70-100 for most JPG images in the test dataset).
+      # @param force_source_text_compression [Boolean] If true, attempts to re-write detected text.
+      # @param disable_source_text [Boolean] If true, doesn't re-apply source text to the original PDF.
+      def self.compress_pdf(pdf_data, quality: 85, force_source_text_compression: false, disable_source_text: true)
+        if PDFTools.source_text?(pdf_data)
+          if force_source_text_compression
+            if disable_source_text
+              puts "\e[33m[WARNING] Re-writing PDF source-text is an EXPERIMENTAL feature.\e[0m"
+            else
+              puts "\e[33m[WARNING] Source-file contains text, but disable_source_text flag is ignored. " \
+                   "Resulting file will not contain any embedded text.\e[0m"
+            end
+          else
+            puts "\e[33m[WARNING] Source-text detected in input PDF. Aborting operation.\e[0m"
+            return pdf_data
+          end
+        end
+
+        pdf_data.rewind
+        pdf = Origami::PDF.read(pdf_data)
+        pages = process_pdf_pages(pdf, quality)
+
+        output_pdf = create_output_pdf(pages, disable_source_text, pdf_data)
+
+        output_stream = StringIO.new
+        output_pdf.save(output_stream)
+        output_stream
+      end
+
+      # Processes all pages in the PDF.
+      # @param pdf [Origami::PDF] The Origami PDF object to process.
+      # @param quality [Integer] Compression quality.
+      # @return [Array<Origami::Page>] Processed pages.
+      def self.process_pdf_pages(pdf, quality)
+        pdf.pages.map.with_index do |page, index|
+          process_pdf_page(Mindee::PDF::PdfProcessor.get_page(pdf, index), index, quality, page[:MediaBox])
+        end
+      end
+
+      # Creates the output PDF with processed pages.
+      # @param pages [Array] Processed pages.
+      # @param disable_source_text [Boolean] Whether to disable source text.
+      # @param pdf_data [StringIO] Original PDF data.
+      # @return [Origami::PDF] Output PDF object.
+      def self.create_output_pdf(pages, disable_source_text, pdf_data)
+        output_pdf = Origami::PDF.new
+        # NOTE: Page order and XObject handling require adjustment due to origami adding the last page first.
+        pages.rotate!(1) if pages.count >= 2
+
+        inject_text(pdf_data, pages) unless disable_source_text
+
+        pages.each { |page| output_pdf.append_page(page) }
+
+        output_pdf
+      end
+
+      # Extracts text from a source text PDF, and injects it into a newly-created one.
+      # @param pdf_data [StringIO] Stream representation of the PDF.
+      # @param pages [Array<Origami::Page>] Array of pages containing the rasterized version of the initial pages.
+      def self.inject_text(pdf_data, pages)
+        reader = PDFReader::Reader.new(pdf_data)
+
+        reader.pages.each_with_index do |original_page, index|
+          break if index >= pages.length
+
+          receiver = PDFReader::Reader::PageTextReceiver.new
+          original_page.walk(receiver)
+
+          receiver.runs.each do |text_run|
+            x = text_run.origin.x
+            y = text_run.origin.y
+            text = text_run.text
+            font_size = text_run.font_size
+
+            content_stream = Origami::Stream.new
+            content_stream.dictionary[:Filter] = :FlateDecode
+            content_stream.data = "BT\n/F1 #{font_size} Tf\n#{x} #{y} Td\n(#{text}) Tj\nET\n"
+
+            pages[index].Contents.data += content_stream.data
+          end
+        end
+      end
+
+      # Takes in a page stream, rasterizes it into a JPEG image, and applies the result onto a new Origami PDF page.
+      # @param page_stream [StringIO] Stream representation of a single page from the initial PDF.
+      # @param page_index [Integer] Index of the current page. Technically not needed, but left for debugging purposes.
+      # @param image_quality [Integer] Quality to apply to the rasterized page.
+      # @param media_box [Array<Integer>, nil] Extracted media box from the page. Can be nil.
+      # @return [Origami::Page]
+      def self.process_pdf_page(page_stream, page_index, image_quality, media_box)
+        new_page = Origami::Page.new
+        compressed_image = Mindee::Image::ImageUtils.pdf_to_magick_image(page_stream, image_quality)
+        width, height = Mindee::Image::ImageUtils.calculate_dimensions_from_media_box(compressed_image, media_box)
+
+        compressed_xobject = PDF::PDFTools.create_xobject(compressed_image)
+        PDF::PDFTools.set_xobject_properties(compressed_xobject, compressed_image)
+
+        xobject_name = "X#{page_index + 1}"
+        PDF::PDFTools.add_content_to_page(new_page, xobject_name, width, height)
+        new_page.add_xobject(compressed_xobject, xobject_name)
+
+        PDF::PDFTools.set_page_dimensions(new_page, width, height)
+        new_page
+      end
+    end
+  end
+end

data/lib/mindee/pdf/{pdf_processing.rb → pdf_processor.rb}

@@ -18,6 +18,7 @@ module Mindee
 
       # @param io_stream [StreamIO]
       # @param options [Hash]
+      # @return [StringIO]
       def self.parse(io_stream, options)
         options = DEFAULT_OPTIONS.merge(options)
 
@@ -74,6 +75,22 @@ module Mindee
         io_stream.seek(0)
         pdf_parser.parse(io_stream)
       end
+
+      # Retrieves a PDF document's page.
+      #
+      # @param [Origami::PDF] pdf_doc Origami PDF handle.
+      # @param [Integer] page_id Page ID.
+      # @return [StringIO]
+      def self.get_page(pdf_doc, page_id)
+        stream = StringIO.new
+        pdf_doc.save(stream)
+
+        options = {
+          page_indexes: [page_id - 1],
+        }
+
+        parse(stream, options)
+      end
     end
   end
 end

data/lib/mindee/pdf/pdf_tools.rb

@@ -29,6 +29,106 @@ module Mindee
         io_stream.set_encoding Encoding::BINARY
         io_stream
       end
+
+      # Checks a PDFs stream content for text operators
+      # See https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf page 243-251.
+      # @param [StringIO] stream Stream object from a PDFs page.
+      # @return [Boolean] True if a text operator is found in the stream.
+      def self.stream_has_text?(stream)
+        data = stream.data
+        return false if data.nil? || data.empty?
+
+        text_operators = ['Tc', 'Tw', 'Th', 'TL', 'Tf', 'Tk', 'Tr', 'Tm', 'T*', 'Tj', 'TJ', "'", '"']
+        text_operators.any? { |op| data.include?(op) }
+      end
+
+      # Checks whether the file has source_text. Sends false if the file isn't a PDF.
+      # @param [StringIO] pdf_data
+      # @return [Boolean] True if the pdf has source text, false otherwise.
+      def self.source_text?(pdf_data)
+        begin
+          pdf_data.rewind
+          pdf = Origami::PDF.read(pdf_data)
+
+          pdf.each_page do |page|
+            next unless page[:Contents]
+
+            contents = page[:Contents].solve
+            contents = [contents] unless contents.is_a?(Origami::Array)
+
+            contents.each do |stream_ref|
+              stream = stream_ref.solve
+              return true if stream_has_text?(stream)
+            end
+          end
+
+          false
+        end
+
+        false
+      rescue Origami::InvalidPDFError
+        false
+      end
+
+      def self.create_xobject(image)
+        image_io = Mindee::Image::ImageUtils.image_to_stringio(image)
+        Origami::Graphics::ImageXObject.from_image_file(image_io, 'jpg')
+      end
+
+      def self.set_xobject_properties(xobject, image)
+        xobject.dictionary[:BitsPerComponent] = 8
+        xobject.dictionary[:Filter] = determine_filter(image)
+        xobject.dictionary[:Width] = image[:width]
+        xobject.dictionary[:Height] = image[:height]
+        xobject.dictionary[:ColorSpace] = determine_colorspace(image)
+      end
+
+      def self.determine_filter(image)
+        filter = image.data['properties']['filter']
+        case filter
+        when %r{Zip}i then :FlateDecode
+        when %r{LZW}i then :LZWDecode
+        else :DCTDecode
+        end
+      end
+
+      def self.determine_colorspace(image)
+        colorspace = image.data['colorspace']
+        case colorspace
+        when 'CMYK' then :DeviceCMYK
+        when 'Gray', 'PseudoClass Gray' then :DeviceGray
+        else :DeviceRGB
+        end
+      end
+
+      def self.add_content_to_page(page, xobject_name, width, height)
+        content = "q\n#{width} 0 0 #{height} 0 0 cm\n/#{xobject_name} Do\nQ\n"
+        content_stream = Origami::Stream.new(content)
+        page.Contents = content_stream
+      end
+
+      def self.set_page_dimensions(page, width, height)
+        page[:MediaBox] = [0, 0, width, height]
+        page[:CropBox] = [0, 0, width, height]
+      end
+
+      def self.process_image_xobject(image_data, image_quality, width, height)
+        compressed_data = Image::ImageCompressor.compress_image(
+          image_data,
+          quality: image_quality,
+          max_width: width,
+          max_height: height
+        )
+
+        new_image = Origami::Graphics::ImageXObject.new
+        new_image.data = compressed_data
+        new_image.Width = width
+        new_image.Height = height
+        new_image.ColorSpace = :DeviceRGB
+        new_image.BitsPerComponent = 8
+
+        new_image
+      end
     end
   end
 end

data/lib/mindee/pdf.rb

@@ -1,3 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'pdf/pdf_processing'
+require_relative 'pdf/pdf_compressor'
+require_relative 'pdf/pdf_processor'
+require_relative 'pdf/pdf_tools'

data/lib/mindee/product/financial_document/financial_document_v1_document.rb

@@ -6,7 +6,7 @@ require_relative 'financial_document_v1_line_item'
 module Mindee
   module Product
     module FinancialDocument
-      # Financial Document API version 1.9 document data.
+      # Financial Document API version 1.10 document data.
       class FinancialDocumentV1Document < Mindee::Parsing::Common::Prediction
         include Mindee::Parsing::Standard
         # The customer's address used for billing.
@@ -48,6 +48,12 @@ module Mindee
         # The locale detected on the document.
         # @return [Mindee::Parsing::Standard::LocaleField]
         attr_reader :locale
+        # The date on which the payment is due / fullfilled.
+        # @return [Mindee::Parsing::Standard::DateField]
+        attr_reader :payment_date
+        # The purchase order number.
+        # @return [Mindee::Parsing::Standard::StringField]
+        attr_reader :po_number
         # The receipt number or identifier only if document is a receipt.
         # @return [Mindee::Parsing::Standard::StringField]
         attr_reader :receipt_number
@@ -123,6 +129,8 @@ module Mindee
             @line_items.push(FinancialDocumentV1LineItem.new(item, page_id))
           end
           @locale = LocaleField.new(prediction['locale'], page_id)
+          @payment_date = DateField.new(prediction['payment_date'], page_id)
+          @po_number = StringField.new(prediction['po_number'], page_id)
           @receipt_number = StringField.new(prediction['receipt_number'], page_id)
           @reference_numbers = []
           prediction['reference_numbers'].each do |item|
@@ -161,11 +169,13 @@ module Mindee
           out_str = String.new
           out_str << "\n:Locale: #{@locale}".rstrip
           out_str << "\n:Invoice Number: #{@invoice_number}".rstrip
+          out_str << "\n:Purchase Order Number: #{@po_number}".rstrip
           out_str << "\n:Receipt Number: #{@receipt_number}".rstrip
           out_str << "\n:Document Number: #{@document_number}".rstrip
           out_str << "\n:Reference Numbers: #{reference_numbers}".rstrip
           out_str << "\n:Purchase Date: #{@date}".rstrip
           out_str << "\n:Due Date: #{@due_date}".rstrip
+          out_str << "\n:Payment Date: #{@payment_date}".rstrip
           out_str << "\n:Total Net: #{@total_net}".rstrip
           out_str << "\n:Total Amount: #{@total_amount}".rstrip
           out_str << "\n:Taxes:#{@taxes}".rstrip

data/lib/mindee/product/financial_document/financial_document_v1_page.rb

@@ -6,7 +6,7 @@ require_relative 'financial_document_v1_document'
 module Mindee
   module Product
     module FinancialDocument
-      # Financial Document API version 1.9 page data.
+      # Financial Document API version 1.10 page data.
       class FinancialDocumentV1Page < Mindee::Parsing::Common::Page
         # @param prediction [Hash]
         def initialize(prediction)

data/lib/mindee/product/invoice/invoice_v4_document.rb

@@ -6,7 +6,7 @@ require_relative 'invoice_v4_line_item'
 module Mindee
   module Product
     module Invoice
-      # Invoice API version 4.7 document data.
+      # Invoice API version 4.8 document data.
       class InvoiceV4Document < Mindee::Parsing::Common::Prediction
         include Mindee::Parsing::Standard
         # The customer's address used for billing.
@@ -42,6 +42,12 @@ module Mindee
         # The locale detected on the document.
         # @return [Mindee::Parsing::Standard::LocaleField]
         attr_reader :locale
+        # The date on which the payment is due/ was full-filled.
+        # @return [Mindee::Parsing::Standard::DateField]
+        attr_reader :payment_date
+        # The purchase order number.
+        # @return [Mindee::Parsing::Standard::StringField]
+        attr_reader :po_number
         # List of Reference numbers, including PO number.
         # @return [Array<Mindee::Parsing::Standard::StringField>]
         attr_reader :reference_numbers
@@ -103,6 +109,8 @@ module Mindee
             @line_items.push(InvoiceV4LineItem.new(item, page_id))
           end
           @locale = LocaleField.new(prediction['locale'], page_id)
+          @payment_date = DateField.new(prediction['payment_date'], page_id)
+          @po_number = StringField.new(prediction['po_number'], page_id)
           @reference_numbers = []
           prediction['reference_numbers'].each do |item|
             @reference_numbers.push(StringField.new(item, page_id))
@@ -137,9 +145,11 @@ module Mindee
           out_str = String.new
           out_str << "\n:Locale: #{@locale}".rstrip
           out_str << "\n:Invoice Number: #{@invoice_number}".rstrip
+          out_str << "\n:Purchase Order Number: #{@po_number}".rstrip
           out_str << "\n:Reference Numbers: #{reference_numbers}".rstrip
           out_str << "\n:Purchase Date: #{@date}".rstrip
           out_str << "\n:Due Date: #{@due_date}".rstrip
+          out_str << "\n:Payment Date: #{@payment_date}".rstrip
           out_str << "\n:Total Net: #{@total_net}".rstrip
           out_str << "\n:Total Amount: #{@total_amount}".rstrip
           out_str << "\n:Total Tax: #{@total_tax}".rstrip

data/lib/mindee/product/invoice/invoice_v4_page.rb

@@ -6,7 +6,7 @@ require_relative 'invoice_v4_document'
 module Mindee
   module Product
     module Invoice
-      # Invoice API version 4.7 page data.
+      # Invoice API version 4.8 page data.
       class InvoiceV4Page < Mindee::Parsing::Common::Page
         # @param prediction [Hash]
         def initialize(prediction)

data/lib/mindee/version.rb

@@ -3,7 +3,7 @@
 # Mindee
 module Mindee
   # Current version.
-  VERSION = '3.13.0'
+  VERSION = '3.15.0'
 
   # Finds and return the current platform.
   # @return [String]

data/lib/mindee.rb

@@ -19,6 +19,16 @@ module Mindee
     end
   end
 
+  module Image
+    # Miscellaneous image operations.
+    module ImageUtils
+    end
+
+    # Image compressor module to handle image compression.
+    module ImageCompressor
+    end
+  end
+
   # Custom extraction module
   module Extraction
   end

data/mindee.gemspec

@@ -30,8 +30,9 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new('>= 2.6')
 
   spec.add_runtime_dependency 'marcel', '~> 1.0.2'
-  spec.add_runtime_dependency 'mini_magick', '~> 4.13.0'
+  spec.add_runtime_dependency 'mini_magick', '>=4', '< 6'
   spec.add_runtime_dependency 'origamindee', '~> 3.1.0'
+  spec.add_runtime_dependency 'pdf-reader', '~> 2.12.0'
 
   spec.add_development_dependency 'rake', '~> 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.12.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mindee
 version: !ruby/object:Gem::Version
-  version: 3.13.0
+  version: 3.15.0
 platform: ruby
 authors:
 - Mindee, SA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-09-18 00:00:00.000000000 Z
+date: 2024-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: marcel
@@ -28,16 +28,22 @@ dependencies:
   name: mini_magick
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 4.13.0
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 4.13.0
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: origamindee
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.12.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -198,6 +218,7 @@ files:
 - docs/us_mail_v2.md
 - docs/us_w9_v1.md
 - examples/auto_invoice_splitter_extraction.rb
+- examples/auto_multi_receipts_detector_extraction.rb
 - lib/mindee.rb
 - lib/mindee/client.rb
 - lib/mindee/extraction.rb
@@ -223,6 +244,9 @@ files:
 - lib/mindee/http/endpoint.rb
 - lib/mindee/http/error.rb
 - lib/mindee/http/response_validation.rb
+- lib/mindee/image.rb
+- lib/mindee/image/image_compressor.rb
+- lib/mindee/image/image_utils.rb
 - lib/mindee/input.rb
 - lib/mindee/input/local_response.rb
 - lib/mindee/input/sources.rb
@@ -261,7 +285,8 @@ files:
 - lib/mindee/parsing/standard/string_field.rb
 - lib/mindee/parsing/standard/tax_field.rb
 - lib/mindee/pdf.rb
-- lib/mindee/pdf/pdf_processing.rb
+- lib/mindee/pdf/pdf_compressor.rb
+- lib/mindee/pdf/pdf_processor.rb
 - lib/mindee/pdf/pdf_tools.rb
 - lib/mindee/product.rb
 - lib/mindee/product/.rubocop.yml