esse 0.2.0 → 0.2.2

data/lib/esse/client_proxy/search.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Esse
+  class ClientProxy
+    module InstanceMethods
+      # Returns results matching a query.
+      def search(index:, **options)
+        definition = options.merge(
+          index: index,
+        )
+
+        Esse::Events.instrument('elasticsearch.search') do |payload|
+          payload[:request] = definition
+          payload[:response] = coerce_exception { client.search(definition) }
+        end
+      end
+
+      # Allows to retrieve a large numbers of results from a single search request.
+      #
+      # @param [Time] :scroll Specify how long a consistent view of the index should be maintained for scrolled search
+      # @param [Boolean] :rest_total_hits_as_int Indicates whether hits.total should be rendered as an integer or an object in the rest search response
+      # @param [Hash] :body The scroll ID
+      def scroll(scroll:, **definition)
+        unless definition[:body]
+          raise ArgumentError, 'scroll search must have a :body with the :scroll_id'
+        end
+        Esse::Events.instrument('elasticsearch.search') do |payload|
+          payload[:request] = definition
+          payload[:response] = coerce_exception { client.scroll(scroll: scroll, **definition) }
+        end
+      end
+
+      # Explicitly clears the search context for a scroll.
+      #
+      # @param [Hash] :body Body with the "scroll_id" (string or array of strings) Scroll IDs to clear.
+      #   To clear all scroll IDs, use _all.
+      def clear_scroll(body:, **options)
+        coerce_exception { client.clear_scroll(body: body, **options) }
+      end
+    end
+
+    include InstanceMethods
+  end
+end

data/lib/esse/client_proxy.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Esse
+  class ClientProxy
+    require_relative './client_proxy/search'
+
+    extend Forwardable
+
+    def_delegators :@cluster, :client
+
+    attr_reader :cluster
+
+    def initialize(cluster)
+      @cluster = cluster
+    end
+
+    # Elasticsearch::Transport was renamed to Elastic::Transport in 8.0
+    # This lib should support both versions that's why we are wrapping up the transport
+    # errors to local errors.
+    def coerce_exception
+      yield
+    rescue => exception
+      name = Hstring.new(exception.class.name)
+      if /^(Elasticsearch|Elastic|OpenSearch)?::Transport::Transport::Errors/.match?(name.value) && \
+          (exception_class = Esse::Backend::ERRORS[name.demodulize.value])
+        raise exception_class.new(exception.message)
+      else
+        raise exception
+      end
+    end
+  end
+end

data/lib/esse/cluster.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
+require_relative 'cluster_engine'
+require_relative 'client_proxy'
+
 module Esse
   class Cluster
-    ATTRIBUTES = %i[index_prefix index_settings client wait_for_status].freeze
+    ATTRIBUTES = %i[index_prefix settings mappings client wait_for_status].freeze
     WAIT_FOR_STATUSES = %w[green yellow red].freeze
 
     # The index prefix. For example an index named UsersIndex.
     # With `index_prefix = 'app1'`. Final index/alias is: 'app1_users'
     attr_accessor :index_prefix
 
-    # This settings will be passed through all indices during the mapping
-    attr_accessor :index_settings
+    # This global settings will be passed through all indices
+    attr_accessor :settings
+
+    # This global mappings will be applied to all indices
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+    attr_accessor :mappings
 
     # if this option set, actions such as creating or deleting index,
     # importing data will wait for the status specified. Extremely useful
@@ -25,7 +32,8 @@ module Esse
 
     def initialize(id:, **options)
       @id = id.to_sym
-      @index_settings = {}
+      @settings = {}
+      @mappings = {}
       assign(options)
     end
 
@@ -41,16 +49,28 @@ module Esse
     end
 
     def client
-      @client ||= Elasticsearch::Client.new
+      @client ||= if defined? Elasticsearch::Client
+        Elasticsearch::Client.new
+      elsif defined? OpenSearch::Client
+        OpenSearch::Client.new
+      else
+        raise Esse::Error, <<~ERROR
+          Elasticsearch::Client or OpenSearch::Client is not defined.
+          Please install elasticsearch or opensearch-ruby gem.
+        ERROR
+      end
     end
 
-    # Define the elasticsearch client connectio
-    # @param es_client [Elasticsearch::Client, Hash] an instance of elasticsearch/api client or an hash
-    #   with the settings that will be used to initialize Elasticsearch::Client
+    # Define the elasticsearch client connection
+    # @param es_client [Elasticsearch::Client, OpenSearch::Client, Hash] an instance of elasticsearch/api client or an hash
+    #   with the settings that will be used to initialize the Client
     def client=(es_client)
-      @client = if es_client.is_a?(Hash)
+      @client = if es_client.is_a?(Hash) && defined?(Elasticsearch::Client)
         settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
         Elasticsearch::Client.new(settings)
+      elsif es_client.is_a?(Hash) && defined?(OpenSearch::Client)
+        settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
+        OpenSearch::Client.new(settings)
       else
         es_client
       end
@@ -70,5 +90,40 @@ module Esse
 
       client.cluster.health(wait_for_status: status.to_s)
     end
+
+    # @idea Change this to use the response from `GET /`
+    def document_type?
+      defined?(OpenSearch::VERSION) || \
+        (defined?(Elasticsearch::VERSION) && Elasticsearch::VERSION < '7')
+    end
+
+    def info
+      @info ||= begin
+        resp = client.info
+        {
+          distribution: (resp.dig('version', 'distribution') || 'elasticsearch'),
+          version: resp.dig('version', 'number'),
+        }
+      end
+    end
+
+    def engine
+      ClusterEngine.new(**info)
+    end
+
+    # Build a search query for the given indices
+    #
+    # @param indices [Array<Esse::Index, String>] The indices class or the index name
+    # @return [Esse::Search::Query] The search query instance
+    def search(*indices, **kwargs, &block)
+      Esse::Search::Query.new(api, *indices, **kwargs, &block)
+    end
+
+    # Return the proxy object used to perform low level actions on the elasticsearch cluster through the official api client
+    #
+    # @return [Esse::ClientProxy] The cluster api instance
+    def api
+      Esse::ClientProxy.new(self)
+    end
   end
 end

data/lib/esse/cluster_engine.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Esse
+  class ClusterEngine
+    OPENSEARCH_FORK_VERSION = '7.10.2'
+
+    attr_reader :version, :distribution
+
+    def initialize(distribution:, version:)
+      @distribution = distribution
+      @version = version
+    end
+
+    def engine_version
+      return @version unless opensearch?
+
+      OPENSEARCH_FORK_VERSION
+    end
+
+    def opensearch?
+      distribution == 'opensearch'
+    end
+
+    def elasticsearch?
+      distribution == 'elasticsearch'
+    end
+
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html
+    def mapping_single_type?
+      engine_version >= '6'
+    end
+
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping.html
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/6.4/mapping.html
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.1/mapping.html
+    def mapping_default_type
+      return unless engine_version.to_i == 6
+
+      engine_version >= '6.4' ? :_doc : :doc
+    end
+  end
+end

data/lib/esse/collection.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Esse
+  class Collection
+    include Enumerable
+    attr_reader :options
+
+    def initialize(**options)
+      @options = options
+    end
+
+    # @yield [<Array, Hash>] A batch of documents to be serialized and indexed.
+    # @abstract Override this method to yield each chunk of documents with optional metadata
+    def each
+      raise NotImplementedError, 'Override this method to iterate over the collection'
+    end
+  end
+end

data/lib/esse/config.rb

@@ -9,10 +9,14 @@ module Esse
   #     conf.cluster(:v1) do |cluster|
   #       cluster.index_prefix = 'backend'
   #       cluster.client = Elasticsearch::Client.new
-  #       cluster.index_settings = {
+  #       cluster.settings = {
   #         number_of_shards: 2,
   #         number_of_replicas: 0
   #       }
+  #       cluster.mappings = {
+  #         dynamic_templates: [...]
+  #         properties: { ... }
+  #       }
   #     end
   #   end
   #
@@ -36,13 +40,17 @@ module Esse
   #   end
   class Config
     DEFAULT_CLUSTER_ID = :default
-    ATTRIBUTES = %i[indices_directory].freeze
+    ATTRIBUTES = %i[indices_directory bulk_wait_interval].freeze
 
     # The location of the indices. Defaults to the `app/indices`
     attr_reader :indices_directory
 
+    # wait a given period between posting pages to give Elasticsearch time to catch up.
+    attr_reader :bulk_wait_interval
+
     def initialize
       self.indices_directory = 'app/indices'
+      self.bulk_wait_interval = 0.1
       @clusters = {}
       cluster(DEFAULT_CLUSTER_ID) # initialize the :default client
     end
@@ -66,6 +74,10 @@ module Esse
       @indices_directory = value.is_a?(Pathname) ? value : Pathname.new(value)
     end
 
+    def bulk_wait_interval=(value)
+      @bulk_wait_interval = value.to_f
+    end
+
     def load(arg)
       case arg
       when Hash

data/lib/esse/core.rb

@@ -1,22 +1,28 @@
 # frozen_string_literal: true
 
-
-require 'multi_json'
-require 'elasticsearch'
-
 module Esse
   require_relative 'config'
   require_relative 'cluster'
   require_relative 'primitives'
-  require_relative 'index_type'
+  require_relative 'collection'
+  require_relative 'serializer'
+  require_relative 'hash_document'
+  require_relative 'null_document'
+  require_relative 'repository'
   require_relative 'index_setting'
+  require_relative 'dynamic_template'
   require_relative 'index_mapping'
   require_relative 'template_loader'
+  require_relative 'import/request_body'
+  require_relative 'import/bulk'
   require_relative 'backend/index'
-  require_relative 'backend/index_type'
+  require_relative 'backend/repository_backend'
   require_relative 'version'
   require_relative 'logging'
   require_relative 'events'
+  require_relative 'search/query'
+  require_relative 'search/response'
+  require_relative 'deprecations' # Should be last
   include Logging
 
   @single_threaded = false
@@ -68,4 +74,15 @@ module Esse
     end
     [id, modified]
   end
+
+  def self.eager_load_indices!
+    return false unless Esse.config.indices_directory.exist?
+
+    Dir[Esse.config.indices_directory.join('**/*_index.rb')].map { |path| Pathname.new(path) }.each do |path|
+      next unless path.extname == '.rb'
+
+      require(path.expand_path.to_s)
+    end
+    true
+  end
 end

data/lib/esse/deprecations/cluster.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Esse
+  class Cluster
+    extend Gem::Deprecate
+
+    def index_settings
+      settings
+    end
+    deprecate :index_settings, :settings, 2022, 10
+
+    def index_settings=(value)
+      self.settings = value
+    end
+    deprecate :index_settings=, :settings=, 2022, 10
+
+    def index_mappings
+      mappings
+    end
+    deprecate :index_mappings, :mappings, 2022, 10
+
+    def index_mappings=(value)
+      self.mappings = value
+    end
+    deprecate :index_mappings=, :mappings=, 2022, 10
+  end
+end

data/lib/esse/deprecations/index.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    class << self
+      extend Gem::Deprecate
+
+      def define_type(name, *args, **kwargs, &block)
+        repository(name, *args, **kwargs, &block)
+      end
+      deprecate :define_type, :repository, 2022, 8
+
+      def type_hash
+        repo_hash
+      end
+      deprecate :type_hash, :repo_hash, 2022, 8
+    end
+  end
+end

data/lib/esse/deprecations/repository.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Esse
+  class Repository
+    class << self
+      extend Gem::Deprecate
+
+      def type_name
+        document_type
+      end
+      deprecate :type_name, :document_type, 2022, 10
+
+      def mappings(*args, &block)
+        index.mappings(*args, &block)
+      end
+      deprecate :mappings, "Esse::Index.mappings", 2022, 10
+    end
+  end
+end

data/lib/esse/deprecations.rb

@@ -0,0 +1,3 @@
+require_relative 'deprecations/index'
+require_relative 'deprecations/cluster'
+require_relative 'deprecations/repository'

data/lib/esse/dynamic_template.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Esse
+  class DynamicTemplate
+    # @param [Array, Hash] value The list of dynamic_templates for mapping
+    def initialize(value)
+      @hash = normalize(value)
+    end
+
+    def merge!(value)
+      @hash = HashUtils.deep_merge(@hash, normalize(value))
+    end
+
+    def to_a
+      @hash.map do |name, value|
+        { name => value }
+      end
+    end
+
+    def any?
+      @hash.any?
+    end
+
+    def dup
+      self.class.new(@hash.dup)
+    end
+
+    private
+
+    def normalize(value)
+      case value
+      when Array
+        value.map { |v| normalize(v) }.reduce(&:merge)
+      when Hash
+        HashUtils.deep_transform_keys(value, &:to_sym)
+      end || {}
+    end
+  end
+end

data/lib/esse/errors.rb

@@ -4,6 +4,59 @@ module Esse
   class Error < StandardError
   end
 
+  # @todo Rename this
+  module Backend
+    class ServerError < ::Esse::Error; end
+
+    ES_TRANSPORT_ERRORS = {
+      'MultipleChoices' => 'MultipleChoicesError', # 300
+      'MovedPermanently' => 'MovedPermanentlyError', # 301
+      'Found' => 'FoundError', # 302
+      'SeeOther' => 'SeeOtherError', # 303
+      'NotModified' => 'NotModifiedError', # 304
+      'UseProxy' => 'UseProxyError', # 305
+      'TemporaryRedirect' => 'TemporaryRedirectError', # 307
+      'PermanentRedirect' => 'PermanentRedirectError', # 308
+      'BadRequest' => 'BadRequestError', # 400
+      'Unauthorized' => 'UnauthorizedError', # 401
+      'PaymentRequired' => 'PaymentRequiredError', # 402
+      'Forbidden' => 'ForbiddenError', # 403
+      'NotFound' => 'NotFoundError', # 404
+      'MethodNotAllowed' => 'MethodNotAllowedError', # 405
+      'NotAcceptable' => 'NotAcceptableError', # 406
+      'ProxyAuthenticationRequired' => 'ProxyAuthenticationRequiredError', # 407
+      'RequestTimeout' => 'RequestTimeoutError', # 408
+      'Conflict' => 'ConflictError', # 409
+      'Gone' => 'GoneError', # 410
+      'LengthRequired' => 'LengthRequiredError', # 411
+      'PreconditionFailed' => 'PreconditionFailedError', # 412
+      'RequestEntityTooLarge' => 'RequestEntityTooLargeError', # 413
+      'RequestURITooLong' => 'RequestURITooLongError', # 414
+      'UnsupportedMediaType' => 'UnsupportedMediaTypeError', # 415
+      'RequestedRangeNotSatisfiable' => 'RequestedRangeNotSatisfiableError', # 416
+      'ExpectationFailed' => 'ExpectationFailedError', # 417
+      'ImATeapot' => 'ImATeapotError', # 418
+      'TooManyConnectionsFromThisIP' => 'TooManyConnectionsFromThisIPError', # 421
+      'UpgradeRequired' => 'UpgradeRequiredError', # 426
+      'BlockedByWindowsParentalControls' => 'BlockedByWindowsParentalControlsError', # 450
+      'RequestHeaderTooLarge' => 'RequestHeaderTooLargeError', # 494
+      'HTTPToHTTPS' => 'HTTPToHTTPSError', # 497
+      'ClientClosedRequest' => 'ClientClosedRequestError', # 499
+      'InternalServerError' => 'InternalServerError', # 500
+      'NotImplemented' => 'NotImplementedError', # 501
+      'BadGateway' => 'BadGatewayError', # 502
+      'ServiceUnavailable' => 'ServiceUnavailableError', # 503
+      'GatewayTimeout' => 'GatewayTimeoutError', # 504
+      'HTTPVersionNotSupported' => 'HTTPVersionNotSupportedError', # 505
+      'VariantAlsoNegotiates' => 'VariantAlsoNegotiatesError', # 506
+      'NotExtended' => 'NotExtendedError', # 510
+    }
+
+    ERRORS = ES_TRANSPORT_ERRORS.each_with_object({}) do |(transport_name, esse_name), memo|
+      memo[transport_name] = const_set esse_name, Class.new(ServerError)
+    end
+  end
+
   module Events
     class UnregisteredEventError < ::Esse::Error
       def initialize(object_or_event_id)
@@ -42,6 +95,4 @@ module Esse
     class InvalidOption < Error
     end
   end
-
-  # Elasticsearch::Transport::Transport::Errors::NotFound
 end

data/lib/esse/events/event.rb

@@ -3,6 +3,10 @@
 module Esse
   module Events
     class Event
+      extend Forwardable
+      def_delegators :@payload, :[], :fetch, :to_h, :key?
+      alias_method :to_hash, :to_h
+
       attr_reader :id
 
       # Initialize a new event
@@ -18,25 +22,6 @@ module Esse
         @payload = payload
       end
 
-      # Get data from the payload
-      #
-      # @param [String,Symbol] name
-      #
-      # @api public
-      def [](name)
-        @payload.fetch(name)
-      end
-
-      # Coerce an event to a hash
-      #
-      # @return [Hash]
-      #
-      # @api public
-      def to_h
-        @payload
-      end
-      alias_method :to_hash, :to_h
-
       # Get or set a payload
       #
       # @overload

data/lib/esse/events.rb

@@ -45,5 +45,8 @@ module Esse
     register_event 'elasticsearch.update_mapping'
     register_event 'elasticsearch.update_settings'
     register_event 'elasticsearch.update_aliases'
+    register_event 'elasticsearch.bulk'
+    register_event 'elasticsearch.search'
+    register_event 'elasticsearch.execute_search_query'
   end
 end

data/lib/esse/hash_document.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Esse
+  class HashDocument < Esse::Serializer
+    META_KEYS = %i[_id _type _routing routing].freeze
+
+    def initialize(object)
+      @object = object
+      @options = {}
+    end
+
+    # @return [String, Number] the document ID
+    def id
+      object['_id'] || object[:_id] || object['id'] || object[:id]
+    end
+
+    # @return [String, nil] the document type
+    def type
+      object['_type'] || object[:_type]
+    end
+
+    # @return [String, nil] the document routing
+    def routing
+      object['_routing'] || object[:_routing] || object['routing'] || object[:routing]
+    end
+
+    # @return [Hash] the document meta
+    def meta
+      {}
+    end
+
+    # @return [Hash] the document source
+    # @abstract Override this method to return the document source
+    def source
+      object.reject { |key, _| META_KEYS.include?(key.to_sym) }
+    end
+  end
+end