mindee-lite 5.0.0.beta1

data/lib/mindee/input/sources/local_input_source.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'marcel'
+require 'fileutils'
+
+require_relative '../../dependency'
+require_relative '../../pdf' if Mindee::Dependency.all_deps_available?
+require_relative '../../image' if Mindee::Dependency.all_deps_available?
+
+module Mindee
+  module Input
+    # Document source handling.
+    module Source
+      # Mime types accepted by the server.
+      ALLOWED_MIME_TYPES = [
+        'application/pdf',
+        'image/heic',
+        'image/png',
+        'image/jpeg',
+        'image/tiff',
+        'image/webp',
+      ].freeze
+
+      # Base class for loading documents.
+      class LocalInputSource
+        # @return [String]
+        attr_reader :filename
+        # @return [String]
+        attr_reader :file_mimetype
+        # @return [StringIO | File]
+        attr_reader :io_stream
+
+        # @param io_stream [StringIO, File]
+        # @param filename [String]
+        # @param repair_pdf [bool]
+        def initialize(io_stream, filename, repair_pdf: false)
+          @io_stream = io_stream
+          @filename = filename
+          @file_mimetype = if repair_pdf
+                             Marcel::MimeType.for @io_stream
+                           else
+                             Marcel::MimeType.for @io_stream, name: @filename
+                           end
+          if ALLOWED_MIME_TYPES.include? @file_mimetype
+            logger.debug("Loaded new input #{@filename} from #{self.class}")
+            return
+          end
+
+          if filename.end_with?('.pdf') && repair_pdf
+            fix_pdf!
+
+            logger.debug("Loaded new input #{@filename} from #{self.class}")
+            return if ALLOWED_MIME_TYPES.include? @file_mimetype
+          end
+
+          raise Error::MindeeMimeTypeError, @file_mimetype.to_s
+        end
+
+        # @deprecated See {#fix_pdf!} or {#self.fix_pdf} instead.
+        def rescue_broken_pdf(_)
+          fix_pdf!
+        end
+
+        # Shorthand for PDF mimetype validation.
+        def pdf?
+          @file_mimetype.to_s == 'application/pdf'
+        end
+
+        # Attempts to fix the PDF data in the file.
+        # @param maximum_offset [Integer] Maximum offset to look for the PDF header.
+        # @return [void]
+        # @raise [Mindee::Error::MindeePDFError]
+        def fix_pdf!(maximum_offset: 500)
+          @io_stream = LocalInputSource.fix_pdf(@io_stream, maximum_offset: maximum_offset)
+          @io_stream.rewind
+          @file_mimetype = Marcel::MimeType.for @io_stream
+        end
+
+        # Attempt to fix the PDF data in the given stream.
+        # @param stream [StringIO] The stream to fix.
+        # @param maximum_offset [Integer] Maximum offset to look for the PDF header.
+        # @return [StringIO] The fixed stream.
+        # @raise [Mindee::Error::MindeePDFError]
+        def self.fix_pdf(stream, maximum_offset: 500)
+          out_stream = StringIO.new
+          stream.gets('%PDF-')
+          raise Error::MindeePDFError if stream.eof? || stream.pos > maximum_offset
+
+          stream.pos = stream.pos - 5
+          out_stream << stream.read
+        end
+
+        # Cuts a PDF file according to provided options.
+        # @param options [PageOptions, nil] Page cutting/merge options:
+        #
+        #  * `:page_indexes` Zero-based list of page indexes.
+        #  * `:operation` Operation to apply on the document, given the `page_indexes specified:
+        #      * `:KEEP_ONLY` - keep only the specified pages, and remove all others.
+        #      * `:REMOVE` - remove the specified pages, and keep all others.
+        #  * `:on_min_pages` Apply the operation only if document has at least this many pages.
+        def apply_page_options(options)
+          @io_stream.seek(0)
+          @io_stream = PDF::PDFProcessor.parse(@io_stream, options)
+        end
+
+        # @deprecated Use {#apply_page_options} instead.
+        # @see #apply_page_options
+        def process_pdf(options)
+          apply_page_options(options)
+        end
+
+        # Reads a document.
+        # @param close [bool]
+        # @return [Array<>]
+        def read_contents(close: true)
+          logger.debug("Reading data from: #{@filename}")
+          @io_stream.seek(0)
+          # Avoids needlessly re-packing some files
+          data = @io_stream.read
+          @io_stream.rewind
+          @io_stream.close if close
+          [data, { filename: Mindee::Input::Source.convert_to_unicode_escape(@filename) }]
+        end
+
+        # Write the file to a given path. Uses the initial file name by default.
+        # @param path [String] Path to write the file to.
+        def write_to_file(path)
+          t_path = if File.directory?(path || '') || path.to_s.end_with?('/')
+                     File.join(path || '', @filename)
+                   else
+                     path
+                   end
+          full_path = File.expand_path(t_path || '')
+          FileUtils.mkdir_p(File.dirname(full_path))
+          @io_stream.rewind
+          File.binwrite(full_path, @io_stream.read || '')
+          logger.debug("Wrote file successfully to #{full_path}")
+          @io_stream.rewind
+        end
+
+        # Returns the page count for a document.
+        # Defaults to one for images.
+        # @return [Integer]
+        def page_count
+          unless Mindee::Dependency.all_deps_available?
+            raise NotImplementedError, Mindee::Dependency::MINDEE_DEPENDENCIES_LOAD_ERROR
+          end
+          return 1 unless pdf?
+
+          @io_stream.seek(0)
+          pdf_processor = Mindee::PDF::PDFProcessor.open_pdf(@io_stream)
+          pdf_processor.pages.size
+        end
+
+        # Compresses the file, according to the provided info.
+        # @param [Integer] quality Quality of the output file.
+        # @param [Integer, nil] max_width Maximum width (Ignored for PDFs).
+        # @param [Integer, nil] max_height Maximum height (Ignored for PDFs).
+        # @param [bool] force_source_text Whether to force the operation on PDFs with source text.
+        #   This will attempt to re-render PDF text over the rasterized original. If disabled, ignored the operation.
+        #   WARNING: this operation is strongly discouraged.
+        # @param [bool] disable_source_text If the PDF has source text, whether to re-apply it to the original or
+        #   not. Needs force_source_text to work.
+        def compress!(quality: 85, max_width: nil, max_height: nil, force_source_text: false, disable_source_text: true)
+          unless Mindee::Dependency.all_deps_available?
+            raise NotImplementedError, Mindee::Dependency::MINDEE_DEPENDENCIES_LOAD_ERROR
+          end
+
+          buffer = if pdf?
+                     Mindee::PDF::PDFCompressor.compress_pdf(
+                       @io_stream,
+                       quality: quality,
+                       force_source_text_compression: force_source_text,
+                       disable_source_text: disable_source_text
+                     )
+                   else
+                     Mindee::Image::ImageCompressor.compress_image(
+                       @io_stream,
+                       quality: quality,
+                       max_width: max_width,
+                       max_height: max_height
+                     )
+                   end
+          @io_stream = buffer
+          @io_stream.rewind
+        end
+
+        # Checks whether the file has source text if it is a pdf. `false` otherwise
+        # @return [bool] `true` if the file is a PDF and has source text.
+        def source_text?
+          unless Mindee::Dependency.all_deps_available?
+            raise NotImplementedError, Mindee::Dependency::MINDEE_DEPENDENCIES_LOAD_ERROR
+          end
+
+          Mindee::PDF::PDFTools.source_text?(@io_stream)
+        end
+      end
+
+      # Replaces non-ASCII characters by their UNICODE escape sequence.
+      # Keeps other characters as is.
+      # @return A clean String.
+      def self.convert_to_unicode_escape(string)
+        unicode_escape_string = ''.dup
+        string.each_char do |char|
+          unicode_escape_string << if char.bytesize > 1
+                                     "\\u#{format('%04x', char.unpack1('U'))}"
+                                   else
+                                     char
+                                   end
+        end
+        unicode_escape_string
+      end
+    end
+  end
+end

data/lib/mindee/input/sources/path_input_source.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module Mindee
+  module Input
+    # Document source handling.
+    module Source
+      # Load a document from a path.
+      class PathInputSource < LocalInputSource
+        # @param filepath [String]
+        # @param repair_pdf [bool]
+        def initialize(filepath, repair_pdf: false)
+          io_stream = File.new(filepath, 'rb')
+          super(io_stream, File.basename(filepath), repair_pdf: repair_pdf)
+        end
+      end
+    end
+  end
+end

data/lib/mindee/input/sources/url_input_source.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'fileutils'
+require_relative '../../logging'
+
+module Mindee
+  module Input
+    module Source
+      # Load a remote document from a file url.
+      class URLInputSource
+        # @return [String]
+        attr_reader :url
+
+        def initialize(url)
+          raise Error::MindeeInputError, 'URL must be HTTPS' unless url.start_with? 'https://'
+
+          logger.debug("URL input: #{url}")
+
+          @url = url
+        end
+
+        # Downloads the file from the URL and saves it to the specified path.
+        #
+        # @param path [String] Path to save the file to.
+        # @param filename [String, nil] Optional name to give to the file.
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [String] The full path of the saved file.
+        def write_to_file(path, filename: nil, username: nil, password: nil, token: nil, max_redirects: 3)
+          response_body = fetch_file_content(username: username, password: password, token: token,
+                                             max_redirects: max_redirects)
+
+          filename = fill_filename(filename)
+
+          full_path = File.join(path.chomp('/'), filename)
+          File.write(full_path, response_body)
+
+          full_path
+        end
+
+        # Downloads the file from the url, and returns a BytesInputSource wrapper object for it.
+        #
+        # @param filename [String, nil] Optional name to give to the file.
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [BytesInputSource] The full path of the saved file.
+        def as_local_input_source(filename: nil, username: nil, password: nil, token: nil, max_redirects: 3)
+          filename = fill_filename(filename)
+          response_body = fetch_file_content(username: username, password: password, token: token,
+                                             max_redirects: max_redirects)
+          bytes = StringIO.new(response_body)
+
+          BytesInputSource.new(bytes.read || '', filename || '')
+        end
+
+        # Fetches the file content from the URL.
+        #
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [String] The downloaded file content.
+        def fetch_file_content(username: nil, password: nil, token: nil, max_redirects: 3)
+          uri = URI.parse(@url)
+          request = Net::HTTP::Get.new(uri)
+
+          request['Authorization'] = "Bearer #{token}" if token
+          request.basic_auth(username, password) if username && password
+
+          response = make_request(uri, request, max_redirects)
+          if response.code.to_i > 299
+            raise Error::MindeeAPIError, "Failed to download file: HTTP status code #{response.code}"
+          elsif response.code.to_i < 200
+            raise Error::MindeeAPIError, "Failed to download file: Invalid response code #{response.code}."
+          end
+
+          response.body
+        end
+
+        private
+
+        def extract_filename_from_url(uri)
+          filename = File.basename(uri.path.to_s)
+          filename.empty? ? '' : filename
+        end
+
+        def fill_filename(filename)
+          filename ||= extract_filename_from_url(URI.parse(@url))
+          if filename.empty? || File.extname(filename).empty?
+            filename = generate_file_name(extension: get_file_extension(filename))
+          end
+          filename
+        end
+
+        def make_request(uri, request, max_redirects)
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+            response = http.request(request)
+            if response.is_a?(Net::HTTPRedirection) && max_redirects.positive?
+              location = response['location']
+              raise Error::MindeeInputError, 'No location in redirection header.' if location.nil?
+
+              new_uri = URI.parse(location)
+              request = Net::HTTP::Get.new(new_uri)
+              make_request(new_uri, request, max_redirects - 1)
+            else
+              response
+            end
+          end
+        end
+
+        def get_file_extension(filename)
+          ext = File.extname(filename)
+          ext.empty? ? nil : ext.downcase
+        end
+
+        def generate_file_name(extension: nil)
+          extension ||= '.tmp'
+          random_string = Array.new(8) { rand(36).to_s(36) }.join
+          "mindee_temp_#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}_#{random_string}#{extension}"
+        end
+      end
+    end
+  end
+end

data/lib/mindee/input/sources.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative 'sources/local_input_source'
+require_relative 'sources/bytes_input_source'
+require_relative 'sources/base64_input_source'
+require_relative 'sources/file_input_source'
+require_relative 'sources/path_input_source'
+require_relative 'sources/url_input_source'

data/lib/mindee/input.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'input/polling_options'
+require_relative 'input/sources'

data/lib/mindee/logging/logger.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Mindee
+  # Mindee logging module.
+  module Logging
+    log_level = ENV.fetch('MINDEE_LOG_LEVEL', 'WARN')
+    log_output = ENV.fetch('MINDEE_LOG_OUTPUT', 'stderr')
+    @logger = if log_output == 'stderr'
+                Logger.new($stderr)
+              elsif log_output == 'stdout'
+                Logger.new($stdout)
+              else
+                warn "Invalid MINDEE_LOG_OUTPUT='#{log_output}', defaulting to 'stderr'"
+                Logger.new($stderr)
+              end
+    @logger.level = Logger.const_get(log_level)
+
+    class << self
+      attr_accessor :logger
+    end
+  end
+end

data/lib/mindee/logging.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'logging/logger'

data/lib/mindee/page_options.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Mindee
+  # Class for page options in parse calls.
+  # @!attribute page_indexes [Array[Integer]] Zero-based list of page indexes.
+  # @!attribute operation [:KEEP_ONLY, :REMOVE] Operation to apply on the document, given the specified page indexes:
+  #   * `:KEEP_ONLY` - keep only the specified pages, and remove all others.
+  #   * `:REMOVE` - remove the specified pages, and keep all others.
+  # @!attribute on_min_pages [Integer, nil] Apply the operation only if the document has at least this many pages.
+  class PageOptions
+    attr_accessor :page_indexes, :operation, :on_min_pages
+
+    def initialize(params: {})
+      params ||= {} # : Hash[Symbol, untyped]
+      params = params.transform_keys(&:to_sym)
+      @page_indexes = params.fetch(
+        :page_indexes,
+        [] # : Array[Integer]
+      )
+      @operation = params.fetch(:operation, :KEEP_ONLY)
+      @on_min_pages = params.fetch(:on_min_pages, nil)
+    end
+  end
+end

data/lib/mindee/pdf/extracted_pdf.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Mindee
+  # PDF Extraction Module.
+  module PDF
+    # An extracted sub-Pdf.
+    class ExtractedPDF
+      # Byte contents of the pdf
+      # @return [StringIO]
+      attr_reader :pdf_bytes
+
+      # Name of the file.
+      # @return [String]
+      attr_reader :filename
+
+      # @param pdf_stream [StringIO, File]
+      # @param filename [String]
+      def initialize(pdf_stream, filename)
+        @filename = filename
+
+        if pdf_stream.is_a?(File)
+          pdf_stream.rewind
+          @pdf_bytes = StringIO.new(pdf_stream.read)
+        else
+          @pdf_bytes = pdf_stream
+        end
+      end
+
+      # Retrieves the page count for a given pdf.
+      # @return [Integer]
+      def page_count
+        current_pdf = Mindee::PDF::PDFProcessor.open_pdf(pdf_bytes)
+        current_pdf.pages.size
+      rescue TypeError, Origami::InvalidPDFError
+        raise Error::MindeePDFError, 'Could not retrieve page count from Extracted PDF object.'
+      end
+
+      # Writes the contents of the current PDF object to a file.
+      # @param output_path [String] Path to write to.
+      # @param override [bool] Whether to override the destination file.
+      def write_to_file(output_path, override: false)
+        raise Error::MindeePDFError, 'Provided path is not a file' if File.directory?(output_path)
+        raise Error::MindeePDFError, 'Invalid save path provided' unless File.exist?(
+          File.expand_path('..', output_path)
+        ) && !override
+
+        if File.extname(output_path).downcase == 'pdf'
+          base_path = File.expand_path('..', output_path)
+          output_path = File.expand_path("#{File.basename(output_path)}.pdf", base_path)
+        end
+
+        @pdf_bytes.rewind if @pdf_bytes.respond_to?(:rewind)
+        File.binwrite(output_path, @pdf_bytes.read.to_s)
+        @pdf_bytes.rewind if @pdf_bytes.respond_to?(:rewind)
+      end
+
+      # Returns the current PDF object as a usable BytesInputSource.
+      # @return [Mindee::Input::Source::BytesInputSource]
+      def as_input_source
+        raise Error::MindeePDFError, 'Bytes object is nil.' if @pdf_bytes.nil?
+
+        @pdf_bytes.rewind if @pdf_bytes.respond_to?(:rewind)
+        data = @pdf_bytes.read || ''
+        @pdf_bytes.rewind if @pdf_bytes.respond_to?(:rewind)
+
+        Mindee::Input::Source::BytesInputSource.new(data, @filename)
+      end
+    end
+  end
+end

data/lib/mindee/pdf/pdf_compressor.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+Mindee::Dependency.require_all_deps!
+require 'pdf-reader'
+
+# Shorthand for pdf-reader's PDF namespace, to avoid mixups with the local Origami fork.
+PDFReader = PDF
+
+module Mindee
+  module PDF
+    # Image compressor module to handle PDF compression.
+    module PDFCompressor
+      # Compresses each page of a provided PDF stream. Skips if force_source_text isn't set and source text is detected.
+      # @param pdf_data [StringIO] StringIO handle of the file.
+      # @param quality [Integer] Compression quality (70-100 for most JPG images in the test dataset).
+      # @param force_source_text_compression [bool] If true, attempts to re-write detected text.
+      # @param disable_source_text [bool] If true, doesn't re-apply source text to the original PDF.
+      def self.compress_pdf(pdf_data, quality: 85, force_source_text_compression: false, disable_source_text: true)
+        if PDFTools.source_text?(pdf_data)
+          if force_source_text_compression
+            if disable_source_text
+              logger.warn('Re-writing PDF source-text is an EXPERIMENTAL feature.')
+            else
+              logger.warn('Source-file contains text, but disable_source_text flag is ignored. ' \
+                          'Resulting file will not contain any embedded text.')
+            end
+          else
+            logger.warn('Source-text detected in input PDF. Aborting operation.')
+            return pdf_data
+          end
+        end
+
+        pdf_data.rewind
+        pdf = Origami::PDF.read(pdf_data)
+        pages = process_pdf_pages(pdf, quality)
+
+        output_pdf = create_output_pdf(pages, disable_source_text, pdf_data)
+
+        output_stream = StringIO.new
+        output_pdf.save(output_stream)
+        output_stream
+      end
+
+      # Processes all pages in the PDF.
+      # @param pdf [Origami::PDF] The Origami PDF object to process.
+      # @param quality [Integer] Compression quality.
+      # @return [Array<Origami::Page>] Processed pages.
+      def self.process_pdf_pages(pdf, quality)
+        pdf.pages.map.with_index do |page, index|
+          retrieved_page = Mindee::PDF::PDFProcessor.get_page(pdf, index)
+          process_pdf_page(retrieved_page, index, quality, page[:MediaBox])
+        end
+      end
+
+      # Creates the output PDF with processed pages.
+      # @param pages [Array<Origami::Page>] Processed pages.
+      # @param disable_source_text [bool] Whether to disable source text.
+      # @param pdf_data [StringIO] Original PDF data.
+      # @return [Origami::PDF] Output PDF object.
+      def self.create_output_pdf(pages, disable_source_text, pdf_data)
+        output_pdf = Origami::PDF.new
+        pages.rotate!(1) if pages.count >= 2
+
+        inject_text(pdf_data, pages) unless disable_source_text
+
+        pages.each { |page| output_pdf.append_page(page) }
+
+        output_pdf
+      end
+
+      # Extracts text from a source text PDF, and injects it into a newly-created one.
+      # @param pdf_data [StringIO] Stream representation of the PDF.
+      # @param pages [Array<Origami::Page>] Array of pages containing the rasterized version of the initial pages.
+      def self.inject_text(pdf_data, pages)
+        reader = PDFReader::Reader.new(pdf_data)
+
+        reader.pages.each_with_index do |original_page, index|
+          break if index >= pages.length
+
+          receiver = PDFReader::Reader::PageTextReceiver.new
+          original_page.walk(receiver)
+
+          receiver.runs.each do |text_run|
+            x = text_run.origin.x
+            y = text_run.origin.y
+            text = text_run.text
+            font_size = text_run.font_size
+
+            content_stream = Origami::Stream.new
+            content_stream.dictionary[:Filter] = :FlateDecode
+            content_stream.data = "BT\n/F1 #{font_size} Tf\n#{x} #{y} Td\n(#{text}) Tj\nET\n"
+
+            pages[index].Contents.data += content_stream.data
+          end
+        end
+      end
+
+      # Takes in a page stream, rasterizes it into a JPEG image, and applies the result onto a new Origami PDF page.
+      # @param page_stream [StringIO] Stream representation of a single page from the initial PDF.
+      # @param page_index [Integer] Index of the current page. Technically not needed, but left for debugging purposes.
+      # @param image_quality [Integer] Quality to apply to the rasterized page.
+      # @param media_box [Array<Integer>, nil] Extracted media box from the page. Can be nil.
+      # @return [Origami::Page]
+      def self.process_pdf_page(page_stream, page_index, image_quality, media_box)
+        new_page = Origami::Page.new
+        compressed_image = Mindee::Image::ImageUtils.pdf_to_magick_image(page_stream, image_quality)
+        width, height = Mindee::Image::ImageUtils.calculate_dimensions_from_media_box(compressed_image, media_box)
+
+        compressed_xobject = PDF::PDFTools.create_xobject(compressed_image)
+        PDF::PDFTools.set_xobject_properties(compressed_xobject, compressed_image)
+
+        xobject_name = "X#{page_index + 1}"
+        PDF::PDFTools.add_content_to_page(new_page, xobject_name, width, height)
+        new_page.add_xobject(compressed_xobject, xobject_name)
+
+        PDF::PDFTools.set_page_dimensions(new_page, width, height)
+        new_page
+      end
+    end
+  end
+end