mindee 4.13.0 → 5.0.0.beta1

data/lib/mindee/input/data_schema.rb

@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-module Mindee
-  module Input
-    # Data Schema Field.
-    class DataSchemaField
-      # @return [String] Display name for the field, also impacts inference results.
-      attr_reader :title
-      # @return [String] Name of the field in the data schema.
-      attr_reader :name
-      # @return [Boolean] Whether this field can contain multiple values.
-      attr_reader :is_array
-      # @return [String] Data type of the field.
-      attr_reader :type
-      # @return [Array<String>, nil] Allowed values when type is `classification`. Leave empty for other types.
-      attr_reader :classification_values
-      # @return [Boolean, nil] Whether to remove duplicate values in the array.
-      # Only applicable if `is_array` is True.
-      attr_reader :unique_values
-      # @return [String, nil] Detailed description of what this field represents.
-      attr_reader :description
-      # @return [String, nil] Optional extraction guidelines.
-      attr_reader :guidelines
-      # @return [Array<Hash>, nil] Nested fields.
-      attr_reader :nested_fields
-
-      # @param field [Hash]
-      def initialize(field)
-        field.transform_keys!(&:to_sym)
-        @name = field[:name]
-        @title = field[:title]
-        @is_array = field[:is_array]
-        @type = field[:type]
-        @classification_values = field[:classification_values]
-        @unique_values = field[:unique_values]
-        @description = field[:description]
-        @guidelines = field[:guidelines]
-        @nested_fields = field[:nested_fields]
-      end
-
-      # @return [Hash]
-      def to_hash
-        out = {
-          name: @name,
-          title: @title,
-          is_array: @is_array,
-          type: @type,
-        } # @type var out: Hash[Symbol, untyped]
-        out[:classification_values] = @classification_values unless @classification_values.nil?
-        out[:unique_values] = @unique_values unless @unique_values.nil?
-        out[:description] = @description unless @description.nil?
-        out[:guidelines] = @guidelines unless @guidelines.nil?
-        out[:nested_fields] = @nested_fields unless @nested_fields.nil?
-        out
-      end
-
-      # @return [String]
-      def to_s
-        to_hash.to_json
-      end
-    end
-
-    # The structure to completely replace the data schema of the model.
-    class DataSchemaReplace
-      # @return [Array<DataSchemaField>] Subfields when type is `nested_object`. Leave empty for other types.
-      attr_reader :fields
-
-      # @param data_schema_replace [Hash]
-      def initialize(data_schema_replace)
-        data_schema_replace.transform_keys!(&:to_sym)
-        fields_list = data_schema_replace[:fields]
-        raise Mindee::Errors::MindeeError, 'Invalid Data Schema provided.' if fields_list.nil?
-        raise TypeError, 'Data Schema replacement fields cannot be empty.' if fields_list.empty?
-
-        @fields = fields_list.map { |field| DataSchemaField.new(field) }
-      end
-
-      # @return [Hash]
-      def to_hash
-        { fields: @fields.map(&:to_hash) }
-      end
-
-      # @return [String]
-      def to_s
-        to_hash.to_json
-      end
-    end
-
-    # Modify the Data Schema.
-    class DataSchema
-      # @return [Mindee::Input::DataSchemaReplace]
-      attr_reader :replace
-
-      # @param data_schema [Hash, String]
-      def initialize(data_schema)
-        case data_schema
-        when String
-          parsed = JSON.parse(data_schema.to_s, object_class: Hash)
-          parsed.transform_keys!(&:to_sym)
-          @replace = DataSchemaReplace.new(parsed[:replace])
-        when Hash
-          data_schema.transform_keys!(&:to_sym)
-          @replace = if data_schema[:replace].is_a?(DataSchemaReplace)
-                       data_schema[:replace]
-                     else
-                       DataSchemaReplace.new(data_schema[:replace])
-                     end
-        when DataSchema
-          @replace = data_schema.replace
-        else
-          raise TypeError, 'Invalid Data Schema provided.'
-        end
-      end
-
-      # @return [Hash]
-      def to_hash
-        { replace: @replace.to_hash }
-      end
-
-      # @return [String]
-      def to_s
-        to_hash.to_json
-      end
-    end
-  end
-end

data/lib/mindee/input/inference_parameters.rb

@@ -1,119 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'data_schema'
-require_relative '../input/base_parameters'
-
-module Mindee
-  module Input
-    # Parameters to set when sending a file for inference.
-    class InferenceParameters < Mindee::Input::BaseParameters
-      # @return [Boolean, nil] Enhance extraction accuracy with Retrieval-Augmented Generation.
-      attr_reader :rag
-
-      # @return [Boolean, nil] Extract the full text content from the document as strings,
-      #   and fill the raw_text` attribute.
-      attr_reader :raw_text
-
-      # @return [Boolean, nil] Calculate bounding box polygons for all fields,
-      #   and fill their `locations` attribute.
-      attr_reader :polygon
-
-      # @return [Boolean, nil] Boost the precision and accuracy of all extractions.
-      #   Calculate confidence scores for all fields, and fill their confidence attribute.
-      attr_reader :confidence
-
-      # @return [String, nil] Additional text context used by the model during inference.
-      #   Not recommended, for specific use only.
-      attr_reader :text_context
-
-      # @return [DataSchemaField]
-      attr_reader :data_schema
-
-      # @return [String] Slug for the endpoint.
-      def self.slug
-        'extraction'
-      end
-
-      # rubocop:disable Metrics/ParameterLists
-      # @param [String] model_id ID of the model
-      # @param [Boolean, nil] rag Whether to enable RAG.
-      # @param [Boolean, nil] raw_text Whether to enable rax text.
-      # @param [Boolean, nil] polygon Whether to enable polygons.
-      # @param [Boolean, nil] confidence Whether to enable confidence scores.
-      # @param [String, nil] file_alias File alias, if applicable.
-      # @param [Array<String>, nil] webhook_ids
-      # @param [String, nil] text_context
-      # @param [Hash, nil] polling_options
-      # @param [Boolean, nil] close_file
-      # @param [DataSchemaField, String, Hash nil] data_schema
-      def initialize(
-        model_id,
-        rag: nil,
-        raw_text: nil,
-        polygon: nil,
-        confidence: nil,
-        file_alias: nil,
-        webhook_ids: nil,
-        text_context: nil,
-        polling_options: nil,
-        close_file: true,
-        data_schema: nil
-      )
-        super(
-          model_id,
-          file_alias: file_alias,
-          webhook_ids: webhook_ids,
-          polling_options: polling_options,
-          close_file: close_file
-        )
-
-        @rag = rag
-        @raw_text = raw_text
-        @polygon = polygon
-        @confidence = confidence
-        @text_context = text_context
-        @data_schema = DataSchema.new(data_schema) unless data_schema.nil?
-        # rubocop:enable Metrics/ParameterLists
-      end
-
-      # Appends inference-specific form data to the provided array.
-      # @param [Array] form_data Array of form fields
-      # @return [Array]
-      def append_form_data(form_data)
-        new_form_data = super
-
-        new_form_data.push(['rag', @rag.to_s]) unless @rag.nil?
-        new_form_data.push(['raw_text', @raw_text.to_s]) unless @raw_text.nil?
-        new_form_data.push(['polygon', @polygon.to_s]) unless @polygon.nil?
-        new_form_data.push(['confidence', @confidence.to_s]) unless @confidence.nil?
-        new_form_data.push(['text_context', @text_context]) if @text_context
-        new_form_data.push(['data_schema', @data_schema.to_s]) if @data_schema
-
-        new_form_data
-      end
-
-      # Loads a prediction from a Hash.
-      # @param [Hash] params Parameters to provide as a hash.
-      # @return [InferenceParameters]
-      def self.from_hash(params: {})
-        rag = params.fetch(:rag, nil)
-        raw_text = params.fetch(:raw_text, nil)
-        polygon = params.fetch(:polygon, nil)
-        confidence = params.fetch(:confidence, nil)
-        base_params = load_from_hash(params: params)
-        new_params = base_params.merge(rag: rag, raw_text: raw_text, polygon: polygon, confidence: confidence)
-        model_id = new_params.fetch(:model_id)
-
-        InferenceParameters.new(
-          model_id, rag: rag,
-                    raw_text: raw_text,
-                    polygon: polygon,
-                    confidence: confidence,
-                    file_alias: params.fetch(:file_alias, nil),
-                    webhook_ids: params.fetch(:webhook_ids, nil),
-                    close_file: params.fetch(:close_file, true)
-        )
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/api_request.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Mindee
-  module Parsing
-    module Common
-      # HTTP request response.
-      class ApiRequest
-        # @return [Hash]
-        attr_reader :error
-        # @return [Array<String>]
-        attr_reader :resources
-        # @return [RequestStatus, Symbol]
-        attr_reader :status
-        # @return [Integer]
-        attr_reader :status_code
-        # @return [String]
-        attr_reader :url
-
-        def initialize(server_response)
-          @error = server_response['error']
-          @resources = server_response['resources']
-
-          @status = if server_response['status'] == 'failure'
-                      RequestStatus::FAILURE
-                    elsif server_response['status'] == 'success'
-                      RequestStatus::SUCCESS
-                    else
-                      server_response['status']&.to_sym
-                    end
-          @status_code = server_response['status_code']
-          @url = server_response['url']
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/api_response.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'document'
-require_relative '../../logging'
-
-module Mindee
-  module Parsing
-    module Common
-      # Potential values for queue in asynchronous calls.
-      module JobStatus
-        # Job is waiting.
-        WAITING    = :waiting
-        # Job is processing.
-        PROCESSING = :processing
-        # Job is done.
-        COMPLETED  = :completed
-        # Job failed.
-        FAILURE    = :failed
-      end
-
-      # Potential values for requests.
-      module RequestStatus
-        # Failed.
-        FAILURE = :failure
-        # Success.
-        SUCCESS = :success
-      end
-
-      # Wrapper class for all predictions (synchronous and asynchronous)
-      class ApiResponse
-        # @return [Mindee::Parsing::Common::Document, nil]
-        attr_reader :document
-        # @return [Mindee::Parsing::Common::Job, nil]
-        attr_reader :job
-        # @return [Mindee::Parsing::Common::ApiRequest]
-        attr_reader :api_request
-        # @return [String]
-        attr_reader :raw_http
-
-        # @param product_class [Mindee::Inference]
-        # @param http_response [Hash]
-        # @param raw_http [Hash]
-        def initialize(product_class, http_response, raw_http)
-          logger.debug('Handling API response')
-          @raw_http = raw_http.to_s
-          raise Errors::MindeeAPIError, 'Invalid response format.' unless http_response.key?('api_request')
-
-          @api_request = Mindee::Parsing::Common::ApiRequest.new(http_response['api_request'])
-
-          if http_response.key?('document') &&
-             (!http_response.key?('job') ||
-               http_response['job']['status'] == 'completed') &&
-             @api_request.status == RequestStatus::SUCCESS
-            @document = Mindee::Parsing::Common::Document.new(product_class, http_response['document'])
-          end
-          @job = Mindee::Parsing::Common::Job.new(http_response['job']) if http_response.key?('job')
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/document.rb

@@ -1,84 +0,0 @@
-# frozen_string_literal: true
-
-require_relative 'inference'
-require_relative 'extras'
-
-module Mindee
-  module Parsing
-    module Common
-      # Stores all response attributes.
-      class Document
-        # @return [Mindee::Inference]
-        attr_reader :inference
-        # @return [String] Filename sent to the API
-        attr_reader :name
-        # @return [String] Mindee ID of the document
-        attr_reader :id
-        # @return [Mindee::Parsing::Common::Extras::Extras] Potential Extras fields sent back along the prediction.
-        attr_reader :extras
-        # @return [Mindee::Parsing::Common::OCR::OCR, nil] OCR text results (limited availability)
-        attr_reader :ocr
-        # @return [Integer] Amount of pages of the document
-        attr_reader :n_pages
-
-        # Loads the MVision OCR response.
-        # @param http_response [Hash] Full HTTP contents of the response.
-        # @return [Mindee::Parsing::Common::OCR::OCR]
-        def self.load_ocr(http_response)
-          ocr_prediction = http_response.fetch('ocr', nil)
-          return nil if ocr_prediction.nil? || ocr_prediction.fetch('mvision-v1', nil).nil?
-
-          OCR::OCR.new(ocr_prediction)
-        end
-
-        # Loads extras into the document prediction.
-        # @param http_response [Hash] Full HTTP contents of the response.
-        # @return [Mindee::Parsing::Common::OCR::OCR]
-        def self.extract_extras(http_response)
-          extras_prediction = http_response['inference'].fetch('extras', nil)
-          return nil if extras_prediction.nil? || extras_prediction.fetch('mvision-v1', nil).nil?
-
-          Mindee::Parsing::Common::Extras::Extras.new(extras_prediction)
-        end
-
-        # @param product_class [Mindee::Inference]
-        # @param http_response [Hash]
-        def initialize(product_class, http_response)
-          @id = http_response['id']
-          @name = http_response['name']
-          @inference = product_class.new(http_response['inference'])
-          @ocr = self.class.load_ocr(http_response)
-          @extras = self.class.extract_extras(http_response)
-          inject_full_text_ocr(http_response)
-          @n_pages = http_response['n_pages']
-        end
-
-        # @return [String]
-        def to_s
-          out_str = String.new
-          out_str << "########\nDocument\n########"
-          out_str << "\n:Mindee ID: #{@id}"
-          out_str << "\n:Filename: #{@name}"
-          out_str << "\n\n#{@inference}"
-        end
-
-        private
-
-        def inject_full_text_ocr(raw_prediction)
-          return unless raw_prediction.dig('inference', 'pages', 0, 'extras', 'full_text_ocr')
-
-          full_text_ocr = String.new
-          raw_prediction.dig('inference', 'pages').each do |page|
-            full_text_ocr << page['extras']['full_text_ocr']['content']
-          end
-          artificial_text_obj = { 'content' => full_text_ocr }
-          if @extras.nil? || @extras.empty?
-            @extras = Mindee::Parsing::Common::Extras::Extras.new({ 'full_text_ocr' => artificial_text_obj })
-          else
-            @extras.add_artificial_extra({ 'full_text_ocr' => artificial_text_obj })
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/execution.rb

@@ -1,75 +0,0 @@
-# frozen_string_literal: true
-
-module Mindee
-  module Parsing
-    module Common
-      # Identifier for the batch to which the execution belongs.
-      class Execution
-        # Identifier for the batch to which the execution belongs.
-        # @return [String]
-        attr_reader :batch_name
-        # The time at which the execution started.
-        # @return [Time, nil]
-        attr_reader :created_at
-        # File representation within a workflow execution.
-        # @return [ExecutionFile]
-        attr_reader :file
-        # Identifier for the execution.
-        # @return [String]
-        attr_reader :id
-        # Deserialized inference object.
-        # @return [Mindee::Inference]
-        attr_reader :inference
-        # Priority of the execution.
-        # @return [ExecutionPriority, Symbol, nil]
-        attr_reader :priority
-        # The time at which the file was tagged as reviewed.
-        # @return [Time, nil]
-        attr_reader :reviewed_at
-        # The time at which the file was uploaded to a workflow.
-        # @return [Time, nil]
-        attr_reader :available_at
-        # Reviewed fields and values.
-        # @return [Mindee::Product::Universal::UniversalDocument]
-        attr_reader :reviewed_prediction
-        # Execution Status.
-        # @return [String]
-        attr_reader :status
-        # Execution type.
-        # @return [String]
-        attr_reader :type
-        # The time at which the file was uploaded to a workflow.
-        # @return [Time, nil]
-        attr_reader :uploaded_at
-        # Identifier for the workflow.
-        # @return [String]
-        attr_reader :workflow_id
-
-        # rubocop:disable Metrics/CyclomaticComplexity
-
-        # @param product_class [Mindee::Inference]
-        # @param http_response [Hash]
-        def initialize(product_class, http_response)
-          @batch_name = http_response['batch_name']
-          @created_at = Time.iso8601(http_response['created_at']) if http_response['created_at']
-          @file = ExecutionFile.new(http_response['file']) if http_response['file']
-          @id = http_response['id']
-          @inference = product_class.new(http_response['inference']) if http_response['inference']
-          @priority = Mindee::Parsing::Common::ExecutionPriority.to_priority(http_response['priority'])
-          @reviewed_at = Time.iso8601(http_response['reviewed_at']) if http_response['reviewed_at']
-          @available_at = Time.iso8601(http_response['available_at']) if http_response['available_at']
-          if http_response['reviewed_prediction']
-            @reviewed_prediction = Mindee::Product::Universal::UniversalDocument.new(
-              http_response['reviewed_prediction']
-            )
-          end
-          @status = http_response['status']
-          @type = http_response['type']
-          @uploaded_at = Time.iso8601(http_response['uploaded_at']) if http_response['uploaded_at']
-          @workflow_id = http_response['workflow_id']
-        end
-        # rubocop:enable Metrics/CyclomaticComplexity
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/execution_file.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-
-module Mindee
-  module Parsing
-    module Common
-      # Representation of a workflow execution's file data.
-      class ExecutionFile
-        # File name.
-        # @return [String]
-        attr_reader :name
-
-        # Optional alias for the file.
-        # @return [String]
-        attr_reader :alias
-
-        # @param http_response [Hash]
-        def initialize(http_response)
-          @name = http_response['name']
-          @alias = http_response['alias']
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/execution_priority.rb

@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-module Mindee
-  module Parsing
-    module Common
-      # Execution policy priority values.
-      module ExecutionPriority
-        # Low priority execution.
-        # @return [Symbol] :low
-        LOW = :low
-        # Medium priority execution.
-        # @return [Symbol] :low
-        MEDIUM = :medium
-        # High priority execution.
-        # @return [Symbol] :low
-        HIGH = :high
-
-        # Sets the priority to one of its possibly values, defaults to nil otherwise.
-        # @param [String, nil] priority_str
-        # @return [Symbol, nil]
-        def self.to_priority(priority_str)
-          return nil if priority_str.nil?
-
-          case priority_str.downcase
-          when 'low'
-            :low
-          when 'high'
-            :high
-          else
-            :medium
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/extras/cropper_extra.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../standard/position_field'
-
-module Mindee
-  module Parsing
-    module Common
-      module Extras
-        # Contains information on the cropping of a prediction.
-        class CropperExtra
-          # Cropper extra initialization.
-          # @return [Array<Mindee::Parsing::Standard::PositionField>]
-          attr_reader :croppings
-
-          def initialize(raw_prediction, page_id = nil)
-            @croppings = [] # : Array[Mindee::Parsing::Standard::PositionField]
-            raw_prediction['cropping']&.each do |crop|
-              @croppings.push(Mindee::Parsing::Standard::PositionField.new(crop, page_id))
-            end
-          end
-
-          # @return [String]
-          def to_s
-            @croppings.map(&:to_s).join("\n           ")
-          end
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/extras/extras.rb

@@ -1,58 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../standard/position_field'
-
-module Mindee
-  module Parsing
-    module Common
-      # Extras namespace.
-      module Extras
-        # Extra information added to the prediction.
-        class Extras
-          # @return [CropperExtra, nil]
-          attr_reader :cropper
-          # @return [Mindee::Parsing::Common::Extras::FullTextOCRExtra]
-          attr_reader :full_text_ocr
-          # @return [RAGExtra, nil]
-          attr_reader :rag
-
-          def initialize(raw_prediction)
-            if raw_prediction['cropper']
-              @cropper = Mindee::Parsing::Common::Extras::CropperExtra.new(raw_prediction['cropper'])
-            end
-            if raw_prediction['full_text_ocr']
-              @full_text_ocr = Mindee::Parsing::Common::Extras::FullTextOCRExtra.new(raw_prediction['full_text_ocr'])
-            end
-            @rag = Mindee::Parsing::Common::Extras::RAGExtra.new(raw_prediction['rag']) if raw_prediction['rag']
-
-            raw_prediction.each do |key, value|
-              instance_variable_set("@#{key}", value) unless ['cropper', 'full_text_ocr', 'rag'].include?(key.to_s)
-            end
-          end
-
-          # @return [String]
-          def to_s
-            out_str = String.new
-            instance_variables.each do |var|
-              out_str << "#{var}: #{instance_variable_get(var)}"
-            end
-            out_str
-          end
-
-          # Adds artificial extra data for reconstructed extras. Currently only used for full_text_ocr.
-          #
-          # @param [Hash] raw_prediction Raw prediction used by the document.
-          def add_artificial_extra(raw_prediction)
-            return unless raw_prediction['full_text_ocr']
-
-            @full_text_ocr << Mindee::Parsing::Common::Extras::FullTextOCRExtra.new(raw_prediction)
-          end
-        end
-
-        def empty?
-          instance_variables.all? { |var| instance_variable_get(var).nil? }
-        end
-      end
-    end
-  end
-end

data/lib/mindee/parsing/common/extras/full_text_ocr_extra.rb

@@ -1,33 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../standard/position_field'
-
-module Mindee
-  module Parsing
-    module Common
-      module Extras
-        # Full Text OCR result.
-        class FullTextOCRExtra
-          # Contents of the full text OCR result.
-          # @return [String, nil]
-          attr_reader :contents
-          # Language used on the page.
-          # @return [String, nil]
-          attr_reader :language
-
-          def initialize(raw_prediction)
-            @contents = raw_prediction['content'] if raw_prediction['content']
-            return unless raw_prediction['language']
-
-            @language = raw_prediction['language']
-          end
-
-          # @return [String]
-          def to_s
-            @contents || ''
-          end
-        end
-      end
-    end
-  end
-end