mindee-lite 5.0.0.beta1

data/lib/mindee/v2/client.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require_relative '../input'
+require_relative '../http'
+require_relative 'product'
+require_relative 'parsing/job'
+require_relative '../logging'
+
+module Mindee
+  module V2
+    # Mindee V2 API Client.
+    class Client
+      # @return [V2::HTTP::MindeeApiV2]
+      private attr_reader :mindee_api
+
+      # @param api_key [String]
+      def initialize(api_key: '')
+        @mindee_api = Mindee::V2::HTTP::MindeeApiV2.new(api_key: api_key)
+      end
+
+      # Retrieves a result from a given queue or URL to the result.
+      # @param product [Class<Mindee::V2::Product::BaseProduct>] The return class.
+      # @param resource [String] ID of the inference or URL to the result.
+      # @return [Mindee::V2::Parsing::BaseResponse]
+      def get_result(product, resource)
+        @mindee_api.req_get_result(product, resource)
+      end
+
+      # Retrieves an inference from a given queue or URL to the job.
+      # @param job_id [String] ID of the job.
+      # @return [Mindee::V2::Parsing::JobResponse]
+      def get_job(job_id)
+        @mindee_api.req_get_job(job_id)
+      end
+
+      # Enqueue a document for async parsing.
+      # @param product [Class<Mindee::V2::Product::BaseProduct>] The return class.
+      # @param input_source [Mindee::Input::Source::LocalInputSource, Mindee::Input::Source::URLInputSource]
+      #   The source of the input document (local file or URL).
+      # @param params [Hash, Input::BaseParameters] Parameters for the inference.
+      # @return [Mindee::V2::Parsing::JobResponse]
+      def enqueue(
+        product,
+        input_source,
+        params
+      )
+        normalized_params = normalize_parameters(product.params_type, params)
+        normalized_params.validate_async_params
+        logger.debug("Enqueueing document to model '#{normalized_params.model_id}'.")
+
+        @mindee_api.req_post_enqueue(input_source, normalized_params)
+      end
+
+      # Enqueues to an asynchronous endpoint and automatically polls for a response.
+      #
+      # @param product [Class<Mindee::V2::Product::BaseProduct>] The return class.
+      # @param input_source [Mindee::Input::Source::LocalInputSource, Mindee::Input::Source::URLInputSource]
+      #   The source of the input document (local file or URL).
+      # @param params [Hash, Input::BaseParameters] Parameters for the inference.
+      # @return [Parsing::BaseResponse]
+      def enqueue_and_get_result(
+        product,
+        input_source,
+        params
+      )
+        enqueue_response = enqueue(product, input_source, params)
+        normalized_params = normalize_parameters(product.params_type, params)
+        normalized_params.validate_async_params
+
+        if enqueue_response.job.id.nil? || enqueue_response.job.id.empty?
+          logger.error("Failed enqueueing:\n#{enqueue_response.raw_http}")
+          raise Mindee::Error::MindeeError, 'Enqueueing of the document failed.'
+        end
+
+        job_id = enqueue_response.job.id
+        logger.debug("Successfully enqueued document with job id: #{job_id}.")
+
+        sleep(normalized_params.polling_options.initial_delay_sec)
+        retry_counter = 1
+        poll_results = get_job(job_id)
+
+        while retry_counter < normalized_params.polling_options.max_retries
+          if poll_results.job.status == 'Failed'
+            break
+          elsif !poll_results.job.result_url.nil?
+            return get_result(product, poll_results.job.result_url)
+          end
+
+          logger.debug(
+            "Successfully enqueued inference with job id: #{job_id}.\n" \
+            "Attempt n°#{retry_counter}/#{normalized_params.polling_options.max_retries}.\n" \
+            "Job status: #{poll_results.job.status}."
+          )
+
+          sleep(normalized_params.polling_options.delay_sec)
+          poll_results = get_job(job_id)
+          retry_counter += 1
+        end
+
+        error = poll_results.job.error
+        unless error.nil?
+          err_to_raise = Mindee::Error::MindeeHTTPErrorV2.new(error)
+          # NOTE: purposefully decoupled from the line above, otherwise rubocop thinks `error` is a `message` param.
+          raise err_to_raise
+        end
+
+        sec_count = normalized_params.polling_options.delay_sec * retry_counter
+        raise Mindee::Error::MindeeError,
+              "Asynchronous parsing request timed out after #{sec_count} seconds"
+      end
+
+      # Searches for a list of available models for the given API key.
+      # @param model_name [String]
+      # @param model_type [String]
+      # @return [Mindee::V2::Parsing::Search::SearchResponse]
+      def search_models(model_name, model_type)
+        @mindee_api.search_models(model_name, model_type)
+      end
+
+      private
+
+      # If needed, converts the parsing options provided as a hash into a proper BaseParameters subclass object.
+      # @param params [Hash, Class<BaseParameters>] Params.
+      # @return [BaseParameters]
+      def normalize_parameters(param_class, params)
+        return param_class.from_hash(params: params) if params.is_a?(Hash)
+
+        params
+      end
+    end
+  end
+end

data/lib/mindee/v2/file_operation/crop.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Mindee
+  module V2
+    module FileOperation
+      # Crop operations.
+      module Crop
+        # Extracts a single crop as complete PDFs from the document.
+        #
+        # @param input_source [LocalInputSource] Local Input Source to extract sub-receipts from.
+        # @param crop [FieldLocation] Crop to extract.
+        # @return [ExtractedImage]
+        def self.extract_single_crop(input_source, crop)
+          polygons = [crop.polygon]
+          Mindee::Image::ImageExtractor.extract_multiple_images_from_source(
+            input_source, crop.page, polygons
+          ).first
+        end
+
+        # Extracts individual receipts from multi-receipts documents.
+        #
+        # @param input_source [LocalInputSource] Local Input Source to extract sub-receipts from.
+        # @param crops [Array<CropItem>] List of crops.
+        # @return [CropFiles] Individual extracted receipts as an array of ExtractedImage.
+        # @raise [MindeeError] if the crops array is empty.
+        def self.extract_crops(input_source, crops)
+          if crops.nil? || crops.empty?
+            raise Mindee::Error::MindeeError,
+                  'No possible candidates found for Crop extraction.'
+          end
+
+          polygons = Array.new(input_source.page_count) { [] }
+
+          crops.each do |crop|
+            polygons[crop.location.page] << crop.location.polygon
+          end
+
+          images = [] # @type var images: Array[Image::ExtractedImage]
+          polygons.each_with_index do |page_polygons, page_index|
+            extracted = Mindee::Image::ImageExtractor.extract_multiple_images_from_source(
+              input_source, page_index, page_polygons
+            )
+            images.concat(extracted)
+          end
+
+          CropFiles.new(images)
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/file_operation/crop_files.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mindee
+  module V2
+    module FileOperation
+      # Collection of cropped files.
+      class CropFiles < Array
+        # Save all extracted crops to disk.
+        #
+        # @param path [String, Pathname] Path to save the extracted crops to.
+        # @param prefix [String] Prefix to add to the filename, defaults to 'crop'.
+        # @param file_format [String, nil] File format to save the crops as, defaults to jpg if nil.]
+        def save_all_to_disk(path, prefix: 'crop', file_format: nil)
+          FileUtils.mkdir_p(path)
+          each.with_index(1) do |crop, idx|
+            filename = "#{prefix}_#{format('%03d', idx)}.jpg"
+            file_path = File.join(path.to_s, filename)
+
+            crop.write_to_file(file_path, file_format)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/file_operation/split.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Mindee
+  module V2
+    module FileOperation
+      # Split operations.
+      module Split
+        # Extracts a single split as a complete PDF from the document.
+        #
+        # @param input_source [LocalInputSource] Input source to split.
+        # @param split [Array<Integer>] List of pages to keep.
+        # @return [ExtractedPDF] Extracted PDF
+        def self.extract_single_split(input_source, split)
+          extract_splits(input_source, [split]).first
+        end
+
+        # Extracts splits as complete PDFs from the document.
+        #
+        # @param input_source [LocalInputSource] Input source to split.
+        # @param splits [Array<Array<Integer>>] List of sub-lists of pages to keep.
+        # @return [SplitFiles] A list of extracted invoices.
+        # @raise [MindeeError] if no indexes are provided.
+        def self.extract_splits(input_source, splits)
+          raise Mindee::Error::MindeeError, 'No indexes provided.' if splits.nil? || splits.empty?
+
+          pdf_extractor = Mindee::PDF::PDFExtractor.new(input_source)
+
+          page_groups = splits.map do |split|
+            (split[0]..split[1]).to_a
+          end
+
+          SplitFiles.new(pdf_extractor.extract_sub_documents(page_groups))
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/file_operation/split_files.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Mindee
+  module V2
+    module FileOperation
+      # Collection of split files.
+      class SplitFiles < Array
+        # Save all extracted splits to disk.
+        #
+        # @param path [String, Pathname] Path to save the extracted splits to.
+        # @param prefix [String] Prefix to add to the filename, defaults to 'split'.
+        def save_all_to_disk(path, prefix: 'split')
+          FileUtils.mkdir_p(path)
+
+          each.with_index(1) do |split, idx|
+            filename = "#{prefix}_#{format('%03d', idx)}.pdf"
+            file_path = File.join(path.to_s, filename)
+
+            split.write_to_file(file_path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/file_operation.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require_relative 'file_operation/crop'
+require_relative 'file_operation/crop_files'
+require_relative 'file_operation/split'
+require_relative 'file_operation/split_files'

data/lib/mindee/v2/http/.rubocop.yml

@@ -0,0 +1,7 @@
+inherit_from: ../../../../.rubocop.yml
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/PerceivedComplexity:
+  Max: 10

data/lib/mindee/v2/http/api_v2_settings.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require_relative '../../version'
+
+module Mindee
+  module V2
+    module HTTP
+      # API client for version 2.
+      class ApiV2Settings
+        # V2 API key's default environment key name.
+        MINDEE_V2_API_KEY_ENV_NAME = 'MINDEE_V2_API_KEY'
+        # V2 API key's default value.
+        MINDEE_V2_API_KEY_DEFAULT = nil
+
+        # V2 base URL default environment key name.
+        MINDEE_V2_BASE_URL_ENV_NAME = 'MINDEE_V2_BASE_URL'
+        # V2 base URL default value.
+        MINDEE_V2_BASE_URL_DEFAULT = 'https://api-v2.mindee.net'
+
+        # HTTP request timeout default environment key name.
+        MINDEE_V2_REQUEST_TIMEOUT_ENV_NAME = 'MINDEE_V2_REQUEST_TIMEOUT'
+        # HTTP request timeout default value.
+        MINDEE_V2_TIMEOUT_DEFAULT = 120
+
+        # Default value for the user agent (same as V1).
+        USER_AGENT = "mindee-api-ruby@v#{Mindee::VERSION} ruby-v#{RUBY_VERSION} #{Mindee::PLATFORM}".freeze
+
+        # @return [String]
+        attr_reader :api_key
+        # @return [Integer]
+        attr_reader :request_timeout
+        # @return [String]
+        attr_reader :base_url
+        # @return [String]
+        attr_reader :user_agent
+
+        def initialize(api_key: nil)
+          @request_timeout = ENV.fetch(MINDEE_V2_REQUEST_TIMEOUT_ENV_NAME, MINDEE_V2_TIMEOUT_DEFAULT).to_i
+          if api_key.nil? && !ENV.fetch(MINDEE_V2_API_KEY_ENV_NAME, MINDEE_V2_API_KEY_DEFAULT).to_s.empty?
+            logger.debug('API key set from environment')
+          end
+          @api_key = if api_key.nil? || api_key.empty?
+                       ENV.fetch(MINDEE_V2_API_KEY_ENV_NAME,
+                                 MINDEE_V2_API_KEY_DEFAULT)
+                     else
+                       api_key
+                     end
+          @base_url = ENV.fetch(MINDEE_V2_BASE_URL_ENV_NAME, MINDEE_V2_BASE_URL_DEFAULT).chomp('/')
+          @user_agent = USER_AGENT
+        end
+
+        # Checks API key for a value.
+        # @return
+        # @raise [Error::MindeeAPIError] Raises if the api key is empty or nil.
+        def check_api_key
+          return unless @api_key.nil? || @api_key.to_s.empty?
+
+          raise Error::MindeeAPIError,
+                "Missing API key. check your Client Configuration.\nYou can set this using the " \
+                "'#{MINDEE_V2_API_KEY_ENV_NAME}' environment variable."
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/http/mindee_api_v2.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require_relative '../../input'
+require_relative '../../http'
+require_relative '../../error'
+
+module Mindee
+  module V2
+    module HTTP
+      # API client for version 2.
+      class MindeeApiV2
+        # @return [ApiV2Settings]
+        attr_reader :settings
+
+        # @param api_key [String, nil]
+        def initialize(api_key: nil)
+          @settings = ApiV2Settings.new(api_key: api_key)
+        end
+
+        # Sends a file to the queue.
+        #
+        # @param input_source [Input::Source::LocalInputSource, Input::Source::URLInputSource]
+        # @param params [Input::BaseParameters]
+        # @return [Mindee::V2::Parsing::JobResponse]
+        # @raise [Mindee::Error::MindeeHttpErrorV2]
+        def req_post_enqueue(input_source, params)
+          @settings.check_api_key
+          response = enqueue(
+            input_source,
+            params
+          )
+          Mindee::V2::Parsing::JobResponse.new(process_response(response))
+        end
+
+        # Retrieves a result from a given queue.
+        # @param product [Class<Mindee::V2::Product::BaseProduct>] The return class.
+        # @param resource [String] ID of the inference or URL to the result.
+        # @return [Mindee::V2::Parsing::BaseResponse]
+        def req_get_result(product, resource)
+          return req_get_result_url(product.response_type, resource) if uri?(resource)
+
+          @settings.check_api_key
+          response = result_req_get(
+            resource,
+            product
+          )
+          product.response_type.new(process_response(response))
+        end
+
+        # Retrieves a queued job.
+        #
+        # @param job_id [String] ID of the job or URL to the job.
+        # @return [Mindee::V2::Parsing::JobResponse]
+        def req_get_job(job_id)
+          @settings.check_api_key
+          response = poll("#{@settings.base_url}/v2/jobs/#{job_id}")
+          Mindee::V2::Parsing::JobResponse.new(process_response(response))
+        end
+
+        # Retrieves a list of models.
+        # @param model_name [String, nil]
+        # @param model_type [String, nil]
+        # @return [Mindee::V2::Parsing::Search::SearchResponse]
+        def search_models(model_name, model_type)
+          Mindee::V2::Parsing::Search::SearchResponse.new(process_response(req_get_search_models(model_name,
+                                                                                                 model_type)))
+        end
+
+        private
+
+        # Retrieves a list of models.
+        # @param model_name [String, nil]
+        # @param model_type [String, nil]
+        # @return [Net::HTTPResponse]
+        def req_get_search_models(model_name, model_type)
+          url = "#{@settings.base_url}/v2/search/models"
+          uri = URI(url)
+
+          query_params = {} # @type var query_params: Hash[Symbol, String | nil]
+          query_params[:name] = model_name if model_name
+          query_params[:model_type] = model_type if model_type
+          uri.query = URI.encode_www_form(query_params) unless query_params.empty?
+
+          headers = {
+            'Authorization' => @settings.api_key,
+            'User-Agent' => @settings.user_agent,
+          }
+          req = Net::HTTP::Get.new(uri, headers)
+          req['Transfer-Encoding'] = 'chunked'
+
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, read_timeout: @settings.request_timeout) do |http|
+            return http.request(req)
+          end
+          raise Mindee::Error::MindeeError, 'Could not resolve server response.'
+        end
+
+        # @param resource [String] Resource to check.
+        # @return [Boolean]
+        def uri?(resource)
+          uri = URI.parse(resource)
+          throw Mindee::Error::MindeeError, 'HTTP is not supported.' if uri.scheme == 'http'
+          uri.scheme == 'https'
+        rescue URI::BadURIError, URI::InvalidURIError
+          false
+        end
+
+        # Retrieves a queued job.
+        #
+        # @param url [String]
+        # @return [Mindee::V2::Parsing::JobResponse]
+        def req_get_job_url(url)
+          @settings.check_api_key
+          response = poll(url)
+          Mindee::V2::Parsing::JobResponse.new(process_response(response))
+        end
+
+        # Retrieves a queued job.
+        #
+        # @param result_class [Class<Mindee::V2::Parsing::BaseResponse>]
+        # @param url [String]
+        # @return [Mindee::V2::Parsing::BaseResponse]
+        def req_get_result_url(result_class, url)
+          @settings.check_api_key
+          response = poll(url)
+          result_class.new(process_response(response))
+        end
+
+        # Converts an HTTP response to a parsed response object.
+        #
+        # @param response [Net::HTTPResponse, nil]
+        # @return [Hash]
+        # @raise Throws if the server returned an error.
+        def process_response(response)
+          if !response.nil? && response.respond_to?(:body) &&
+             Mindee::HTTP::ResponseValidation.valid_v2_response?(response)
+            return JSON.parse(response.body, object_class: Hash)
+          end
+
+          response_body = if response.nil? || !response.respond_to?(:body)
+                            '{ "status": -1,
+                            "detail": "Empty server response." }'
+                          else
+                            response.body
+                          end
+          raise Mindee::HTTP::ErrorHandler.generate_v2_error(JSON.parse(response_body).transform_keys(&:to_sym))
+        end
+
+        # Polls a queue for either a result or a job.
+        # @param url [String] URL, passed as a string.
+        # @return [Net::HTTPResponse]
+        def poll(url)
+          uri = URI(url)
+          headers = {
+            'Authorization' => @settings.api_key,
+            'User-Agent' => @settings.user_agent,
+          }
+          req = Net::HTTP::Get.new(uri, headers)
+          req['Transfer-Encoding'] = 'chunked'
+
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, read_timeout: @settings.request_timeout) do |http|
+            return http.request(req)
+          end
+          raise Mindee::Error::MindeeError, 'Could not resolve server response.'
+        end
+
+        # Polls the API for the result of an inference.
+        #
+        # @param queue_id [String] ID of the queue.
+        # @param product [Class<Mindee::V2::Product::BaseProduct>] The return class.
+        # @return [Net::HTTPResponse]
+        def result_req_get(queue_id, product)
+          poll("#{@settings.base_url}/v2/products/#{product.slug}/results/#{queue_id}")
+        end
+
+        # Handle parameters for the enqueue form
+        # @param form_data [Array] Array of form fields
+        # @param params [V2::Product::Extraction::Params::ExtractionParameters] Inference options.
+        def enqueue_form_options(form_data, params)
+          form_data.push(['rag', params.rag.to_s]) unless params.rag.nil?
+          form_data.push(['raw_text', params.raw_text.to_s]) unless params.raw_text.nil?
+          form_data.push(['polygon', params.polygon.to_s]) unless params.polygon.nil?
+          form_data.push(['confidence', params.confidence.to_s]) unless params.confidence.nil?
+          form_data.push ['text_context', params.text_context] if params.text_context
+          form_data.push ['data_schema', params.data_schema.to_s] if params.data_schema
+          unless params.webhook_ids.nil? || params.webhook_ids.empty?
+            params.webhook_ids.each do |webhook_id|
+              form_data.push ['webhook_ids[]', webhook_id]
+            end
+          end
+          form_data
+        end
+
+        # @param input_source [Mindee::Input::Source::LocalInputSource, Mindee::Input::Source::URLInputSource]
+        # @param params [Input::BaseParameters] Inference options.
+        # @return [Net::HTTPResponse, nil]
+        def enqueue(input_source, params)
+          uri = URI("#{@settings.base_url}/v2/products/#{params.slug}/enqueue")
+
+          form_data = if input_source.is_a?(Mindee::Input::Source::URLInputSource)
+                        [['url', input_source.url]] # : Array[Array[untyped]]
+                      else
+                        file_data, file_metadata = input_source.read_contents(close: params.close_file)
+                        [['file', file_data, file_metadata]] # : Array[Array[untyped]]
+                      end
+          form_data.push(['model_id', params.model_id])
+          form_data.push ['file_alias', params.file_alias] if params.file_alias
+          if params.is_a?(V2::Product::Extraction::Params::ExtractionParameters)
+            form_data = enqueue_form_options(form_data, params)
+          end
+
+          form_data = params.append_form_data(form_data)
+
+          headers = {
+            'Authorization' => @settings.api_key,
+            'User-Agent' => @settings.user_agent,
+          }
+          req = Net::HTTP::Post.new(uri, headers)
+
+          req.set_form(form_data, 'multipart/form-data')
+          req['Transfer-Encoding'] = 'chunked'
+
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, read_timeout: @settings.request_timeout) do |http|
+            return http.request(req)
+          end
+          raise Mindee::Error::MindeeError, 'Could not resolve server response.'
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/http.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative 'http/api_v2_settings'
+require_relative 'http/mindee_api_v2'

data/lib/mindee/v2/parsing/base_inference.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative '../product/base_product'
+require_relative '../parsing'
+
+module Mindee
+  module V2
+    module Parsing
+      # Base class for V2 inference responses.
+      class BaseInference < Mindee::V2::Product::BaseProduct
+        # @return [InferenceJob] Metadata about the job.
+        attr_reader :job
+        # @return [V2::Parsing::InferenceModel] Model info for the inference.
+        attr_reader :model
+        # @return [V2::Parsing::InferenceFile] File info for the inference.
+        attr_reader :file
+        # @return [String] ID of the inference.
+        attr_reader :id
+
+        def initialize(http_response)
+          raise ArgumentError, 'Server response must be a Hash' unless http_response.is_a?(Hash)
+
+          super()
+          @model = Mindee::V2::Parsing::InferenceModel.new(http_response['model'])
+          @file = Mindee::V2::Parsing::InferenceFile.new(http_response['file'])
+          @id = http_response['id']
+          @job = Mindee::V2::Parsing::InferenceJob.new(http_response['job']) if http_response.key?('job')
+        end
+
+        # String representation.
+        # @return [String]
+        def to_s
+          [
+            'Inference',
+            '#########',
+            @job.to_s,
+            @model.to_s,
+            @file.to_s,
+          ].join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/mindee/v2/parsing/base_response.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require_relative 'common_response'
+
+module Mindee
+  module V2
+    module Parsing
+      # Base class for V2 inference responses.
+      class BaseResponse < Mindee::V2::Parsing::CommonResponse
+        # @return [BaseInference] The inference result for a split utility request
+        attr_reader :inference
+      end
+    end
+  end
+end

data/lib/mindee/v2/parsing/common_response.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Mindee
+  module V2
+    module Parsing
+      # Base class for inference and job responses on the V2 API.
+      class CommonResponse
+        # @return [String]
+        attr_reader :raw_http
+
+        # @param http_response [Hash]
+        def initialize(http_response)
+          @raw_http = JSON.generate(http_response)
+        end
+      end
+    end
+  end
+end