mindee-lite 5.0.0.beta1

data/lib/mindee/pdf/pdf_extractor.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Mindee
+  # Pdf Extraction Module.
+  module PDF
+    # Pdf extraction class.
+    class PDFExtractor
+      # @param local_input [Mindee::Input::Source::LocalInputSource]
+      def initialize(local_input)
+        unless Mindee::Dependency.all_deps_available?
+          raise NotImplementedError, Mindee::Dependency::MINDEE_DEPENDENCIES_LOAD_ERROR
+        end
+
+        @filename = local_input.filename
+        if local_input.pdf?
+          @source_pdf = local_input.io_stream
+        else
+          pdf_image = Image::ImageExtractor.attach_image_as_new_file(local_input.io_stream)
+          io_buffer = StringIO.new
+          pdf_image.save(io_buffer)
+
+          @source_pdf = io_buffer
+        end
+      end
+
+      # Retrieves the page count for the Pdf object.
+      # @return [Integer]
+      def page_count
+        Mindee::PDF::PDFProcessor.open_pdf(@source_pdf).pages.size
+      end
+
+      # Creates a new Pdf from pages and save it into a buffer.
+      # @param page_indexes [Array<Integer>] List of page number to use for merging in the original Pdf.
+      # @return [StringIO] The buffer containing the new Pdf.
+      def cut_pages(page_indexes)
+        options = PageOptions.new(params: {
+                                    page_indexes: page_indexes,
+                                  })
+
+        Mindee::PDF::PDFProcessor.parse(@source_pdf, options)
+      end
+
+      # Extract the sub-documents from the main pdf, based on the given list of page indexes.
+      # @param page_indexes [Array<Array<Integer>>] List of page number to use for merging in the original Pdf.
+      # @return [Array<Mindee::PDF::ExtractedPDF>] The buffer containing the new Pdf.
+      def extract_sub_documents(page_indexes)
+        extracted_pdfs = [] # @type var extracted_pdfs: Array[Mindee::PDF::ExtractedPDF]
+        extension = File.extname(@filename)
+        basename = File.basename(@filename, extension)
+        page_indexes.each do |page_index_list|
+          if page_index_list.nil? || page_index_list.empty?
+            raise Error::MindeePDFError, "Empty indexes aren't allowed for extraction #{page_index_list}"
+          end
+
+          page_index_list.each do |page_index|
+            if (page_index > page_count) || page_index.negative?
+              raise Error::MindeePDFError,
+                    "Index #{page_index} is out of range."
+            end
+          end
+          formatted_max_index = format('%03d', page_index_list[-1] + 1).to_s
+          field_filename = "#{basename}_#{format('%03d',
+                                                 page_index_list[0] + 1)}-#{formatted_max_index}#{extension}"
+          extracted_pdf = Mindee::PDF::ExtractedPDF.new(cut_pages(page_index_list),
+                                                        field_filename)
+          extracted_pdfs << extracted_pdf
+        end
+        extracted_pdfs
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+
+      # Extracts invoices as complete PDFs from the document.
+      # @param page_indexes [Array<Array<Integer>, InvoiceSplitterV1InvoicePageGroup>]
+      # @param strict [bool]
+      # @return [Array<Mindee::PDF::ExtractedPDF>]
+      def extract_invoices(page_indexes, strict: false)
+        raise Error::MindeePDFError, 'No indexes provided.' if page_indexes.empty?
+
+        if page_indexes[0].is_a?(Array) && page_indexes[0].all?(Integer)
+          page_indexes_as_array = page_indexes # @type var page_indexes : Array[Array[Integer]]
+          return extract_sub_documents(page_indexes_as_array)
+        end
+        p_ids = page_indexes # @type var page_indexes: Product::InvoiceSplitter::InvoiceSplitterV1InvoicePageGroups
+        return extract_sub_documents(p_ids.map(&:page_indexes)) unless strict
+
+        correct_page_indexes = [] # @type var correct_page_indexes: Array[Array[Integer]]
+        current_list = [] # @type var current_list: Array[Integer]
+        previous_confidence = nil
+        p_ids.each_with_index do |p_i, i|
+          page_index = p_i # @type var page_index: Product::InvoiceSplitter::InvoiceSplitterV1InvoicePageGroup
+          confidence = page_index.confidence.to_f
+          page_list = page_index.page_indexes
+
+          if confidence >= 0.5 && previous_confidence.nil?
+            current_list = page_list
+          elsif confidence >= 0.5 && i < p_ids.length - 1
+            correct_page_indexes << current_list
+            current_list = page_list
+          elsif confidence < 0.5 && i == p_ids.length - 1
+            current_list.concat page_list
+            correct_page_indexes << current_list
+          else
+            correct_page_indexes << current_list
+            correct_page_indexes << page_list
+          end
+          previous_confidence = confidence
+        end
+        extract_sub_documents(correct_page_indexes)
+      end
+
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      private
+
+      attr_reader :source_pdf, :filename
+    end
+  end
+end

data/lib/mindee/pdf/pdf_processor.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+Mindee::Dependency.require_all_deps!
+require 'origami'
+require_relative 'pdf_tools'
+
+module Mindee
+  module PDF
+    # PDF document processing
+    module PDFProcessor
+      Origami::PDF.class_eval { include PDFTools }
+      # @param io_stream [StreamIO]
+      # @param options [PageOptions, Hash]
+      # @return [StringIO]
+      def self.parse(io_stream, options)
+        current_pdf = open_pdf(io_stream)
+        pages_count = current_pdf.pages.size
+        return current_pdf.to_io_stream if options.on_min_pages.to_i > pages_count
+
+        all_pages = (0..(pages_count - 1)).to_a
+
+        if options.operation == :KEEP_ONLY
+          pages_to_remove = indexes_from_keep(options.page_indexes, all_pages)
+        elsif options.operation == :REMOVE
+          pages_to_remove = indexes_from_remove(options.page_indexes, all_pages)
+        else
+          raise ArgumentError, "operation must be one of :KEEP_ONLY or :REMOVE, sent '#{options.operation}'"
+        end
+
+        current_pdf.delete_pages_at(pages_to_remove) if pages_to_remove.to_a != all_pages.to_a
+        current_pdf.to_io_stream
+      end
+
+      # @param page_indexes [Array<Integer>]
+      # @param all_pages [Array<Integer>]
+      def self.indexes_from_keep(page_indexes, all_pages)
+        pages_to_keep = Set.new
+        page_indexes.each do |idx|
+          idx = (all_pages.length - (idx + 2)) if idx.negative?
+          page = all_pages[idx]
+          next if page.nil?
+
+          pages_to_keep << page
+        end
+        (all_pages.to_set - pages_to_keep).to_a
+      end
+
+      # @param page_indexes [Array<Integer>]
+      # @param all_pages [Array<Integer>]
+      def self.indexes_from_remove(page_indexes, all_pages)
+        pages_to_remove = Set.new
+        page_indexes.each do |idx|
+          idx = (all_pages.length - (idx + 2)) if idx.negative?
+          page = all_pages[idx]
+          next if page.nil?
+
+          pages_to_remove << page
+        end
+      end
+
+      # @param io_stream [StringIO]
+      # @return [Origami::PDF]
+      def self.open_pdf(io_stream)
+        unless PDFTools.pdf_header?(io_stream)
+          raise Origami::InvalidPDFError,
+                'Input stream does not contain a PDF header.'
+        end
+
+        pdf_parser = Origami::PDF::LinearParser.new({ verbosity: Origami::Parser::VERBOSE_QUIET })
+        io_stream.seek(0)
+        pdf_parser.parse(io_stream)
+      end
+
+      # Retrieves a PDF document's page.
+      #
+      # @param [Origami::PDF] pdf_doc Origami PDF handle.
+      # @param [Integer] page_id Page ID.
+      # @return [StringIO]
+      def self.get_page(pdf_doc, page_id)
+        stream = StringIO.new
+        pdf_doc.save(stream)
+
+        options = PageOptions.new(params: {
+                                    page_indexes: [page_id - 1],
+                                  })
+
+        parse(stream, options)
+      end
+    end
+  end
+end

data/lib/mindee/pdf/pdf_tools.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+Mindee::Dependency.require_all_deps!
+require 'origami'
+
+module Mindee
+  module PDF
+    # Collection of miscellaneous PDF operations,as well as some monkey-patching for Origami.
+    module PDFTools
+      # Converts the current PDF document into a binary-encoded StringIO stream.
+      #
+      # @param [Hash] params Optional settings to override default processing flags.
+      #   - :delinearize [bool] (default: true) Whether to convert a linearized PDF to its full form.
+      #   - :recompile [bool] (default: true) Whether to recompile the PDF after processing.
+      #   - :decrypt [bool] (default: false) Whether to attempt to decrypt the PDF.
+      #   - Other keys such as :intent, :rebuild_xrefs, :noindent, and :obfuscate may be modified automatically.
+      #
+      # @return [StringIO] A binary-encoded stream representing the processed PDF.
+      def to_io_stream(params = {})
+        options = {
+          delinearize: true,
+          recompile: true,
+          decrypt: false,
+          noindent: nil,
+        }
+        options.update(params)
+
+        if frozen? # incompatible flags with frozen doc (signed)
+          options[:recompile] = nil
+          options[:rebuild_xrefs] = nil
+          options[:noindent] = nil
+          options[:obfuscate] = false
+        end
+        load_all_objects unless @loaded
+
+        intents_as_pdfa1 if options[:intent].to_s =~ %r{pdf[/-]?A1?/i}
+        delinearize! if options[:delinearize] && linearized?
+        compile(options) if options[:recompile]
+
+        io_stream = StringIO.new(output(options))
+        io_stream.set_encoding Encoding::BINARY
+        io_stream
+      end
+
+      # Checks a PDFs stream content for text operators
+      # See https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf page 243-251.
+      # @param [StringIO] stream Stream object from a PDFs page.
+      # @return [bool] `true` if a text operator is found in the stream.
+      def self.stream_has_text?(stream)
+        data = stream.data
+        return false if data.nil? || data.empty?
+
+        text_operators = ['Tc', 'Tw', 'Th', 'TL', 'Tf', 'Tk', 'Tr', 'Tm', 'T*', 'Tj', 'TJ', "'", '"']
+        text_operators.any? { |op| data.include?(op) }
+      end
+
+      # Checks whether a stream contains a PDF header near the beginning.
+      # @param [StringIO] io_stream Binary-encoded stream.
+      # @param [Integer] maximum_offset Maximum allowed offset to find '%PDF-'.
+      # @return [bool] `true` when the stream appears to be a PDF.
+      def self.pdf_header?(io_stream, maximum_offset: 500)
+        initial_pos = nil
+        initial_pos = io_stream.pos if io_stream.respond_to?(:pos)
+        io_stream.seek(0)
+        io_stream.gets('%PDF-')
+        !(io_stream.eof? || io_stream.pos > maximum_offset)
+      rescue TypeError, IOError, SystemCallError
+        false
+      ensure
+        io_stream.seek(initial_pos) if !initial_pos.nil? && io_stream.respond_to?(:seek)
+      end
+
+      # Checks whether the file has source_text. Sends false if the file isn't a PDF.
+      # @param [StringIO] pdf_data Abinary-encoded stream representing the PDF file.
+      # @return [bool] `true` if the pdf has source text, false otherwise.
+      def self.source_text?(pdf_data)
+        return false unless pdf_header?(pdf_data)
+
+        begin
+          pdf_data.rewind
+          pdf = Origami::PDF.read(pdf_data)
+
+          pdf.each_page do |page|
+            next unless page[:Contents]
+
+            contents = page[:Contents].solve
+            contents = [contents] unless contents.is_a?(Origami::Array)
+
+            contents.each do |stream_ref|
+              stream = stream_ref.solve
+              return true if stream_has_text?(stream)
+            end
+          end
+
+          false
+        end
+
+        false
+      rescue Origami::InvalidPDFError
+        false
+      end
+
+      # Creates an image XObject from the provided image.
+      #
+      # Converts the given image to a binary stream using Mindee's image utilities, then creates
+      # an Origami::Graphics::ImageXObject with a JPEG filter.
+      #
+      # @param [MiniMagick::Image] image An image object with the necessary data and structure.
+      # @return [Origami::Graphics::ImageXObject] The created image XObject.
+      def self.create_xobject(image)
+        image_io = Mindee::Image::ImageUtils.image_to_stringio(image)
+        Origami::Graphics::ImageXObject.from_image_file(image_io, 'jpg')
+      end
+
+      # Sets properties on the provided image XObject based on image metadata.
+      #
+      # @param [Origami::Graphics::ImageXObject] xobject The image XObject to update.
+      # @param [Hash] image A hash containing image metadata (such as width, height, properties, etc.).
+      def self.set_xobject_properties(xobject, image)
+        xobject.dictionary[:BitsPerComponent] = 8
+        xobject.dictionary[:Filter] = determine_filter(image)
+        xobject.dictionary[:Width] = image[:width]
+        xobject.dictionary[:Height] = image[:height]
+        xobject.dictionary[:ColorSpace] = determine_colorspace(image)
+      end
+
+      # Determines the appropriate filter for an image based on its properties.
+      #
+      # @param [Hash] image The image data hash containing properties.
+      # @return [Symbol] One of :FlateDecode, :LZWDecode or :DCTDecode.
+      def self.determine_filter(image)
+        filter = image.data['properties']['filter']
+        case filter
+        when %r{Zip}i then :FlateDecode
+        when %r{LZW}i then :LZWDecode
+        else :DCTDecode
+        end
+      end
+
+      # Determines the colorspace for an image based on its metadata.
+      #
+      # @param [Hash] image The image data hash.
+      # @return [Symbol] One of :DeviceCMYK, :DeviceGray or :DeviceRGB.
+      def self.determine_colorspace(image)
+        colorspace = image.data['colorspace']
+        case colorspace
+        when 'CMYK' then :DeviceCMYK
+        when 'Gray', 'PseudoClass Gray' then :DeviceGray
+        else :DeviceRGB
+        end
+      end
+
+      # Adds a content stream to the specified PDF page to display an image XObject.
+      #
+      # @param [Origami::Page] page The PDF page to which content will be added.
+      # @param [String] xobject_name The name identifying the XObject.
+      # @param [Integer] width The width for the transformation matrix.
+      # @param [Integer] height The height for the transformation matrix.
+      def self.add_content_to_page(page, xobject_name, width, height)
+        content = "q\n#{width} 0 0 #{height} 0 0 cm\n/#{xobject_name} Do\nQ\n"
+        content_stream = Origami::Stream.new(content)
+        page.Contents = content_stream
+      end
+
+      # Sets the dimensions for the specified PDF page.
+      #
+      # @param [Origami::Page] page The PDF page whose dimensions are being set.
+      # @param [Numeric] width The target width of the page.
+      # @param [Numeric] height The target height of the page.
+      def self.set_page_dimensions(page, width, height)
+        page[:MediaBox] = [0, 0, width, height]
+        page[:CropBox] = [0, 0, width, height]
+      end
+
+      # Processes an image into an image XObject for PDF embedding.
+      #
+      # @param [MiniMagick::Image, StringIO] image_data The raw image data.
+      # @param [Integer] image_quality The quality setting for image compression.
+      # @param [Numeric] width The desired width of the output image.
+      # @param [Numeric] height The desired height of the output image.
+      # @return [Origami::Graphics::ImageXObject] The resulting image XObject.
+      def self.process_image_xobject(image_data, image_quality, width, height)
+        compressed_data = Image::ImageCompressor.compress_image(
+          image_data,
+          quality: image_quality,
+          max_width: width,
+          max_height: height
+        )
+
+        new_image = Origami::Graphics::ImageXObject.new
+        new_image.data = compressed_data
+        new_image.Width = width
+        new_image.Height = height
+        new_image.ColorSpace = :DeviceRGB
+        new_image.BitsPerComponent = 8
+
+        new_image
+      end
+    end
+  end
+end

data/lib/mindee/pdf.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative 'pdf/extracted_pdf'
+require_relative 'pdf/pdf_compressor'
+require_relative 'pdf/pdf_extractor'
+require_relative 'pdf/pdf_processor'
+require_relative 'pdf/pdf_tools'