mindee 3.16.0 → 3.18.0

data/lib/mindee/input/sources/local_input_source.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'marcel'
+
+require_relative '../../pdf'
+require_relative '../../image'
+
+module Mindee
+  module Input
+    # Document source handling.
+    module Source
+      # Mime types accepted by the server.
+      ALLOWED_MIME_TYPES = [
+        'application/pdf',
+        'image/heic',
+        'image/png',
+        'image/jpeg',
+        'image/tiff',
+        'image/webp',
+      ].freeze
+
+      # Standard error for invalid mime types
+      class MimeTypeError < StandardError
+      end
+
+      # Error sent if the file's mimetype isn't allowed
+      class InvalidMimeTypeError < MimeTypeError
+        # @return [String]
+        attr_reader :invalid_mimetype
+
+        # @param mime_type [String]
+        def initialize(mime_type)
+          @invalid_mimetype = mime_type
+          super("'#{@invalid_mimetype}' mime type not allowed, must be one of #{ALLOWED_MIME_TYPES.join(', ')}")
+        end
+      end
+
+      # Error sent if a pdf file couldn't be fixed
+      class UnfixablePDFError < MimeTypeError
+        def initialize
+          super("Corrupted PDF couldn't be repaired.")
+        end
+      end
+
+      # Base class for loading documents.
+      class LocalInputSource
+        # @return [String]
+        attr_reader :filename
+        # @return [String]
+        attr_reader :file_mimetype
+        # @return [StringIO]
+        attr_reader :io_stream
+
+        # @param io_stream [StringIO]
+        # @param filename [String]
+        # @param fix_pdf [Boolean]
+        def initialize(io_stream, filename, fix_pdf: false)
+          @io_stream = io_stream
+          @filename = filename
+          @file_mimetype = if fix_pdf
+                             Marcel::MimeType.for @io_stream
+                           else
+                             Marcel::MimeType.for @io_stream, name: @filename
+                           end
+          return if ALLOWED_MIME_TYPES.include? @file_mimetype
+
+          if filename.end_with?('.pdf') && fix_pdf
+            rescue_broken_pdf(@io_stream)
+            @file_mimetype = Marcel::MimeType.for @io_stream
+
+            return if ALLOWED_MIME_TYPES.include? @file_mimetype
+          end
+
+          raise InvalidMimeTypeError, @file_mimetype.to_s
+        end
+
+        # Attempts to fix pdf files if mimetype is rejected.
+        # "Broken PDFs" are often a result of third-party injecting invalid headers.
+        # This attempts to remove them and send the file
+        # @param stream [StringIO]
+        def rescue_broken_pdf(stream)
+          stream.gets('%PDF-')
+          raise UnfixablePDFError if stream.eof? || stream.pos > 500
+
+          stream.pos = stream.pos - 5
+          data = stream.read
+          @io_stream.close
+
+          @io_stream = StringIO.new
+          @io_stream << data
+        end
+
+        # Shorthand for pdf mimetype validation.
+        def pdf?
+          @file_mimetype.to_s == 'application/pdf'
+        end
+
+        # Parses a PDF file according to provided options.
+        # @param options [Hash, nil] Page cutting/merge options:
+        #
+        #  * `:page_indexes` Zero-based list of page indexes.
+        #  * `:operation` Operation to apply on the document, given the `page_indexes specified:
+        #      * `:KEEP_ONLY` - keep only the specified pages, and remove all others.
+        #      * `:REMOVE` - remove the specified pages, and keep all others.
+        #  * `:on_min_pages` Apply the operation only if document has at least this many pages.
+        def process_pdf(options)
+          @io_stream.seek(0)
+          @io_stream = PdfProcessor.parse(@io_stream, options)
+        end
+
+        # Reads a document.
+        # @param close [Boolean]
+        # @return [Array<String, [String, aBinaryString ], [Hash, nil] >]
+        def read_document(close: true)
+          @io_stream.seek(0)
+          # Avoids needlessly re-packing some files
+          data = @io_stream.read
+          @io_stream.close if close
+          ['document', data, { filename: Mindee::Input::Source.convert_to_unicode_escape(@filename) }]
+        end
+
+        def count_pdf_pages
+          return 1 unless pdf?
+
+          @io_stream.seek(0)
+          pdf_processor = Mindee::PDF::PdfProcessor.open_pdf(@io_stream)
+          pdf_processor.pages.size
+        end
+
+        # Compresses the file, according to the provided info.
+        # @param [Integer] quality Quality of the output file.
+        # @param [Integer, nil] max_width Maximum width (Ignored for PDFs).
+        # @param [Integer, nil] max_height Maximum height (Ignored for PDFs).
+        # @param [Boolean] force_source_text Whether to force the operation on PDFs with source text.
+        #   This will attempt to re-render PDF text over the rasterized original. If disabled, ignored the operation.
+        #   WARNING: this operation is strongly discouraged.
+        # @param [Boolean] disable_source_text If the PDF has source text, whether to re-apply it to the original or
+        #   not. Needs force_source_text to work.
+        def compress!(quality: 85, max_width: nil, max_height: nil, force_source_text: false, disable_source_text: true)
+          buffer = if pdf?
+                     Mindee::PDF::PDFCompressor.compress_pdf(
+                       @io_stream,
+                       quality: quality,
+                       force_source_text_compression: force_source_text,
+                       disable_source_text: disable_source_text
+                     )
+                   else
+                     Mindee::Image::ImageCompressor.compress_image(
+                       @io_stream,
+                       quality: quality,
+                       max_width: max_width,
+                       max_height: max_height
+                     )
+                   end
+          @io_stream = buffer
+          @io_stream.rewind
+        end
+
+        # Checks whether the file has source text if it is a pdf. False otherwise
+        # @return [Boolean] True if the file is a PDF and has source text.
+        def source_text?
+          Mindee::PDF::PDFTools.source_text?(@io_stream)
+        end
+      end
+
+      # Replaces non-ASCII characters by their UNICODE escape sequence.
+      # Keeps other characters as is.
+      # @return A clean String.
+      def self.convert_to_unicode_escape(string)
+        unicode_escape_string = ''.dup
+        string.each_char do |char|
+          unicode_escape_string << if char.bytesize > 1
+                                     "\\u#{char.unpack1('U').to_s(16).rjust(4, '0')}"
+                                   else
+                                     char
+                                   end
+        end
+        unicode_escape_string
+      end
+    end
+  end
+end

data/lib/mindee/input/sources/path_input_source.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'stringio'
+
+module Mindee
+  module Input
+    # Document source handling.
+    module Source
+      # Load a document from a path.
+      class PathInputSource < LocalInputSource
+        # @param filepath [String]
+        # @param fix_pdf [Boolean]
+        def initialize(filepath, fix_pdf: false)
+          io_stream = File.open(filepath, 'rb')
+          super(io_stream, File.basename(filepath), fix_pdf: fix_pdf)
+        end
+      end
+    end
+  end
+end

data/lib/mindee/input/sources/url_input_source.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'fileutils'
+
+module Mindee
+  module Input
+    module Source
+      # Load a remote document from a file url.
+      class UrlInputSource
+        # @return [String]
+        attr_reader :url
+
+        def initialize(url)
+          raise 'URL must be HTTPS' unless url.start_with? 'https://'
+
+          @url = url
+        end
+
+        # Downloads the file from the URL and saves it to the specified path.
+        #
+        # @param path [String] Path to save the file to.
+        # @param filename [String, nil] Optional name to give to the file.
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [String] The full path of the saved file.
+        def save_to_file(path, filename: nil, username: nil, password: nil, token: nil, max_redirects: 3)
+          response_body = fetch_file_content(username: username, password: password, token: token,
+                                             max_redirects: max_redirects)
+
+          filename = fill_filename(filename)
+
+          full_path = File.join(path.chomp('/'), filename)
+          File.write(full_path, response_body)
+
+          full_path
+        end
+
+        # Downloads the file from the url, and returns a BytesInputSource wrapper object for it.
+        #
+        # @param filename [String, nil] Optional name to give to the file.
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [BytesInputSource] The full path of the saved file.
+        def as_local_input_source(filename: nil, username: nil, password: nil, token: nil, max_redirects: 3)
+          filename = fill_filename(filename)
+          response_body = fetch_file_content(username: username, password: password, token: token,
+                                             max_redirects: max_redirects)
+          bytes = StringIO.new(response_body)
+
+          BytesInputSource.new(bytes.read, filename)
+        end
+
+        # Fetches the file content from the URL.
+        #
+        # @param username [String, nil] Optional username for authentication.
+        # @param password [String, nil] Optional password for authentication.
+        # @param token [String, nil] Optional token for JWT-based authentication.
+        # @param max_redirects [Integer] Maximum amount of redirects to follow.
+        # @return [String] The downloaded file content.
+        def fetch_file_content(username: nil, password: nil, token: nil, max_redirects: 3)
+          uri = URI.parse(@url)
+          request = Net::HTTP::Get.new(uri)
+
+          request['Authorization'] = "Bearer #{token}" if token
+          request.basic_auth(username, password) if username && password
+
+          response = make_request(uri, request, max_redirects)
+          if response.code.to_i > 299
+            raise "Failed to download file: HTTP status code #{response.code}"
+          elsif response.code.to_i < 200
+            raise "Failed to download file: Invalid response code #{response.code}."
+          end
+
+          response.body
+        end
+
+        private
+
+        def extract_filename_from_url(uri)
+          filename = File.basename(uri.path)
+          filename.empty? ? '' : filename
+        end
+
+        def fill_filename(filename)
+          filename ||= extract_filename_from_url(URI.parse(@url))
+          if filename.empty? || File.extname(filename).empty?
+            filename = generate_file_name(extension: get_file_extension(filename))
+          end
+          filename
+        end
+
+        def make_request(uri, request, max_redirects)
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+            response = http.request(request)
+            if response.is_a?(Net::HTTPRedirection) && max_redirects.positive?
+              location = response['location']
+              raise 'No location in redirection header.' if location.nil?
+
+              new_uri = URI.parse(location)
+              request = Net::HTTP::Get.new(new_uri)
+              make_request(new_uri, request, max_redirects - 1)
+            else
+              response
+            end
+          end
+        end
+
+        def get_file_extension(filename)
+          ext = File.extname(filename)
+          ext.empty? ? nil : ext.downcase
+        end
+
+        def generate_file_name(extension: nil)
+          extension ||= '.tmp'
+          random_string = Array.new(8) { rand(36).to_s(36) }.join
+          "mindee_temp_#{Time.now.strftime('%Y-%m-%d_%H-%M-%S')}_#{random_string}#{extension}"
+        end
+      end
+    end
+  end
+end

data/lib/mindee/input/sources.rb

@@ -1,250 +1,8 @@
 # frozen_string_literal: true
 
-require 'stringio'
-require 'marcel'
-
-require_relative '../pdf'
-require_relative '../image'
-
-module Mindee
-  module Input
-    # Document source handling.
-    module Source
-      # Mime types accepted by the server.
-      ALLOWED_MIME_TYPES = [
-        'application/pdf',
-        'image/heic',
-        'image/png',
-        'image/jpeg',
-        'image/tiff',
-        'image/webp',
-      ].freeze
-
-      # Standard error for invalid mime types
-      class MimeTypeError < StandardError
-      end
-
-      # Error sent if the file's mimetype isn't allowed
-      class InvalidMimeTypeError < MimeTypeError
-        # @return [String]
-        attr_reader :invalid_mimetype
-
-        # @param mime_type [String]
-        def initialize(mime_type)
-          @invalid_mimetype = mime_type
-          super("'#{@invalid_mimetype}' mime type not allowed, must be one of #{ALLOWED_MIME_TYPES.join(', ')}")
-        end
-      end
-
-      # Error sent if a pdf file couldn't be fixed
-      class UnfixablePDFError < MimeTypeError
-        def initialize
-          super("Corrupted PDF couldn't be repaired.")
-        end
-      end
-
-      # Base class for loading documents.
-      class LocalInputSource
-        # @return [String]
-        attr_reader :filename
-        # @return [String]
-        attr_reader :file_mimetype
-        # @return [StringIO]
-        attr_reader :io_stream
-
-        # @param io_stream [StringIO]
-        # @param filename [String]
-        # @param fix_pdf [Boolean]
-        def initialize(io_stream, filename, fix_pdf: false)
-          @io_stream = io_stream
-          @filename = filename
-          @file_mimetype = if fix_pdf
-                             Marcel::MimeType.for @io_stream
-                           else
-                             Marcel::MimeType.for @io_stream, name: @filename
-                           end
-          return if ALLOWED_MIME_TYPES.include? @file_mimetype
-
-          if filename.end_with?('.pdf') && fix_pdf
-            rescue_broken_pdf(@io_stream)
-            @file_mimetype = Marcel::MimeType.for @io_stream
-
-            return if ALLOWED_MIME_TYPES.include? @file_mimetype
-          end
-
-          raise InvalidMimeTypeError, @file_mimetype.to_s
-        end
-
-        # Attempts to fix pdf files if mimetype is rejected.
-        # "Broken PDFs" are often a result of third-party injecting invalid headers.
-        # This attempts to remove them and send the file
-        # @param stream [StringIO]
-        def rescue_broken_pdf(stream)
-          stream.gets('%PDF-')
-          raise UnfixablePDFError if stream.eof? || stream.pos > 500
-
-          stream.pos = stream.pos - 5
-          data = stream.read
-          @io_stream.close
-
-          @io_stream = StringIO.new
-          @io_stream << data
-        end
-
-        # Shorthand for pdf mimetype validation.
-        def pdf?
-          @file_mimetype.to_s == 'application/pdf'
-        end
-
-        # Parses a PDF file according to provided options.
-        # @param options [Hash, nil] Page cutting/merge options:
-        #
-        #  * `:page_indexes` Zero-based list of page indexes.
-        #  * `:operation` Operation to apply on the document, given the `page_indexes specified:
-        #      * `:KEEP_ONLY` - keep only the specified pages, and remove all others.
-        #      * `:REMOVE` - remove the specified pages, and keep all others.
-        #  * `:on_min_pages` Apply the operation only if document has at least this many pages.
-        def process_pdf(options)
-          @io_stream.seek(0)
-          @io_stream = PdfProcessor.parse(@io_stream, options)
-        end
-
-        # Reads a document.
-        # @param close [Boolean]
-        # @return [Array<String, [String, aBinaryString ], [Hash, nil] >]
-        def read_document(close: true)
-          @io_stream.seek(0)
-          # Avoids needlessly re-packing some files
-          data = @io_stream.read
-          @io_stream.close if close
-          ['document', data, { filename: Mindee::Input::Source.convert_to_unicode_escape(@filename) }]
-        end
-
-        def count_pdf_pages
-          return 1 unless pdf?
-
-          @io_stream.seek(0)
-          pdf_processor = Mindee::PDF::PdfProcessor.open_pdf(@io_stream)
-          pdf_processor.pages.size
-        end
-
-        # Compresses the file, according to the provided info.
-        # @param [Integer] quality Quality of the output file.
-        # @param [Integer, nil] max_width Maximum width (Ignored for PDFs).
-        # @param [Integer, nil] max_height Maximum height (Ignored for PDFs).
-        # @param [Boolean] force_source_text Whether to force the operation on PDFs with source text.
-        #   This will attempt to re-render PDF text over the rasterized original. If disabled, ignored the operation.
-        #   WARNING: this operation is strongly discouraged.
-        # @param [Boolean] disable_source_text If the PDF has source text, whether to re-apply it to the original or
-        #   not. Needs force_source_text to work.
-        def compress!(quality: 85, max_width: nil, max_height: nil, force_source_text: false, disable_source_text: true)
-          buffer = if pdf?
-                     Mindee::PDF::PDFCompressor.compress_pdf(
-                       @io_stream,
-                       quality: quality,
-                       force_source_text_compression: force_source_text,
-                       disable_source_text: disable_source_text
-                     )
-                   else
-                     Mindee::Image::ImageCompressor.compress_image(
-                       @io_stream,
-                       quality: quality,
-                       max_width: max_width,
-                       max_height: max_height
-                     )
-                   end
-          @io_stream = buffer
-          @io_stream.rewind
-        end
-
-        # Checks whether the file has source text if it is a pdf. False otherwise
-        # @return [Boolean] True if the file is a PDF and has source text.
-        def source_text?
-          Mindee::PDF::PDFTools.source_text?(@io_stream)
-        end
-      end
-
-      # Load a document from a path.
-      class PathInputSource < LocalInputSource
-        # @param filepath [String]
-        # @param fix_pdf [Boolean]
-        def initialize(filepath, fix_pdf: false)
-          io_stream = File.open(filepath, 'rb')
-          super(io_stream, File.basename(filepath), fix_pdf: fix_pdf)
-        end
-      end
-
-      # Load a document from a base64 string.
-      class Base64InputSource < LocalInputSource
-        # @param base64_string [String]
-        # @param filename [String]
-        # @param fix_pdf [Boolean]
-        def initialize(base64_string, filename, fix_pdf: false)
-          io_stream = StringIO.new(base64_string.unpack1('m*'))
-          io_stream.set_encoding Encoding::BINARY
-          super(io_stream, filename, fix_pdf: fix_pdf)
-        end
-
-        # Overload of the same function to prevent a base64 from being re-encoded.
-        # @param close [Boolean]
-        # @return [Array<String, [String, aBinaryString ], [Hash, nil] >]
-        def read_document(close: true)
-          @io_stream.seek(0)
-          data = @io_stream.read
-          @io_stream.close if close
-          ['document', [data].pack('m'), { filename: Source.convert_to_unicode_escape(@filename) }]
-        end
-      end
-
-      # Load a document from raw bytes.
-      class BytesInputSource < LocalInputSource
-        # @param raw_bytes [String]
-        # @param filename [String]
-        # @param fix_pdf [Boolean]
-        def initialize(raw_bytes, filename, fix_pdf: false)
-          io_stream = StringIO.new(raw_bytes)
-          io_stream.set_encoding Encoding::BINARY
-          super(io_stream, filename, fix_pdf: fix_pdf)
-        end
-      end
-
-      # Load a document from a file handle.
-      class FileInputSource < LocalInputSource
-        # @param input_file [File]
-        # @param filename [String]
-        # @param fix_pdf [Boolean]
-        def initialize(input_file, filename, fix_pdf: false)
-          io_stream = input_file
-          super(io_stream, filename, fix_pdf: fix_pdf)
-        end
-      end
-
-      # Load a remote document from a file url.
-      class UrlInputSource
-        # @return [String]
-        attr_reader :url
-
-        def initialize(url)
-          raise 'URL must be HTTPS' unless url.start_with? 'https://'
-
-          @url = url
-        end
-      end
-
-      # Replaces non-ASCII characters by their unicode escape sequence.
-      # Keeps other characters as is.
-      # @return A clean String.
-      def self.convert_to_unicode_escape(string)
-        unicode_escape_string = ''.dup
-        string.each_char do |char|
-          unicode_escape_string << if char.bytesize > 1
-                                     "\\u#{char.unpack1('U').to_s(16).rjust(4, '0')}"
-                                   else
-                                     char
-                                   end
-        end
-        unicode_escape_string
-      end
-    end
-  end
-end
+require_relative 'sources/local_input_source'
+require_relative 'sources/bytes_input_source'
+require_relative 'sources/base64_input_source'
+require_relative 'sources/file_input_source'
+require_relative 'sources/path_input_source'
+require_relative 'sources/url_input_source'

data/lib/mindee/parsing/common/api_response.rb

@@ -32,7 +32,7 @@ module Mindee
         attr_reader :id
         # @return [Mindee::Parsing::Standard::DateField]
         attr_reader :issued_at
-        # @return [Mindee::Parsing::Standard::DateField, nil]
+        # @return [Time, nil]
         attr_reader :available_at
         # @return [JobStatus, Symbol]
         attr_reader :status
@@ -121,6 +121,27 @@ module Mindee
           @job = Mindee::Parsing::Common::Job.new(http_response['job']) if http_response.key?('job')
         end
       end
+
+      # Represents the server response after a document is sent to a workflow.
+      class WorkflowResponse
+        # Set the prediction model used to parse the document.
+        # The response object will be instantiated based on this parameter.
+        # @return [Mindee::Parsing::Common::Execution]
+        attr_reader :execution
+        # @return [Mindee::Parsing::Common::ApiRequest]
+        attr_reader :api_request
+        # @return [String]
+        attr_reader :raw_http
+
+        # @param http_response [Hash]
+        # @param product_class [Mindee::Inference]
+        def initialize(product_class, http_response, raw_http)
+          @raw_http = raw_http.to_s
+          @api_request = Mindee::Parsing::Common::ApiRequest.new(http_response['api_request'])
+          product_class = (product_class || Product::Generated::GeneratedV1)
+          @execution = Mindee::Parsing::Common::Execution.new(product_class, http_response['execution'])
+        end
+      end
     end
   end
 end

data/lib/mindee/parsing/common/execution.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Mindee
+  module Parsing
+    module Common
+      # Identifier for the batch to which the execution belongs.
+      class Execution
+        # Identifier for the batch to which the execution belongs.
+        # @return [String]
+        attr_reader :batch_name
+        # The time at which the execution started.
+        # @return [Time, nil]
+        attr_reader :created_at
+        # File representation within a workflow execution.
+        # @return [ExecutionFile]
+        attr_reader :file
+        # Identifier for the execution.
+        # @return [String]
+        attr_reader :id
+        # Deserialized inference object.
+        # @return [Mindee::Inference]
+        attr_reader :inference
+        # Priority of the execution.
+        # @return [ExecutionPriority]
+        attr_reader :priority
+        # The time at which the file was tagged as reviewed.
+        # @return [Time, nil]
+        attr_reader :reviewed_at
+        # The time at which the file was uploaded to a workflow.
+        # @return [Time, nil]
+        attr_reader :available_at
+        # Reviewed fields and values.
+        # @return [Mindee::Product::Generated::GeneratedV1Document]
+        attr_reader :reviewed_prediction
+        # Execution Status.
+        # @return [String]
+        attr_reader :status
+        # Execution type.
+        # @return [String]
+        attr_reader :type
+        # The time at which the file was uploaded to a workflow.
+        # @return [Time, nil]
+        attr_reader :uploaded_at
+        # Identifier for the workflow.
+        # @return [String]
+        attr_reader :workflow_id
+
+        # rubocop:disable Metrics/CyclomaticComplexity
+
+        # @param product_class [Mindee::Inference]
+        # @param http_response [Hash]
+        def initialize(product_class, http_response)
+          @batch_name = http_response['batch_name']
+          @created_at = Time.iso8601(http_response['created_at']) if http_response['created_at']
+          @file = ExecutionFile.new(http_response['file']) if http_response['file']
+          @id = http_response['id']
+          @inference = product_class.new(http_response['inference']) if http_response['inference']
+          @priority = Mindee::Parsing::Common::ExecutionPriority.to_priority(http_response['priority'])
+          @reviewed_at = Time.iso8601(http_response['reviewed_at']) if http_response['reviewed_at']
+          @available_at = Time.iso8601(http_response['available_at']) if http_response['available_at']
+          if http_response['reviewed_prediction']
+            @reviewed_prediction = GeneratedV1Document.new(http_response['reviewed_prediction'])
+          end
+          @status = http_response['status']
+          @type = http_response['type']
+          @uploaded_at = Time.iso8601(http_response['uploaded_at']) if http_response['uploaded_at']
+          @workflow_id = http_response['workflow_id']
+        end
+        # rubocop:enable Metrics/CyclomaticComplexity
+      end
+    end
+  end
+end