esse 0.2.0 → 0.2.3

data/lib/esse/cli/templates/settings.json

@@ -0,0 +1,62 @@
+{
+  "index": {
+    "max_ngram_diff": 5,
+    "max_shingle_diff": 3,
+    "analysis": {
+      "filter": {
+        "esse_search_shingle": {
+          "token_separator": "",
+          "output_unigrams_if_no_shingles": "true",
+          "output_unigrams": "false",
+          "type": "shingle"
+        },
+        "esse_index_shingle": {
+          "token_separator": "",
+          "type": "shingle"
+        },
+        "esse_stemmer": {
+          "type": "stemmer",
+          "language": "English"
+        }
+      },
+      "analyzer": {
+        "esse_index": {
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "english_stop",
+            "esse_index_shingle",
+            "esse_stemmer"
+          ],
+          "char_filter": [
+            "ampersand"
+          ],
+          "type": "custom",
+          "tokenizer": "standard"
+        },
+        "esse_search": {
+          "filter": [
+            "lowercase",
+            "asciifolding",
+            "english_stop",
+            "esse_search_shingle",
+            "esse_stemmer"
+          ],
+          "char_filter": [
+            "ampersand"
+          ],
+          "type": "custom",
+          "tokenizer": "standard"
+        }
+      },
+      "char_filter": {
+        "ampersand": {
+          "type": "mapping",
+          "mappings": [
+            "&=> and "
+          ]
+        }
+      }
+    }
+  }
+}

data/lib/esse/cli.rb

@@ -32,6 +32,7 @@ module Esse
       def initialize(*)
         super
 
+        after_initialize
         load_app_config(options[:require])
         setup_listeners if !options[:silent] && Esse.config.cli_event_listeners?
       end
@@ -71,6 +72,10 @@ module Esse
 
       private
 
+      def after_initialize
+        # esse plugins may override this method
+      end
+
       def setup_listeners
         Esse::Events.__bus__.events.keys.grep(/^elasticsearch/).each do |event_name|
           Esse::Events.subscribe(event_name) do |event|

data/lib/esse/cluster.rb

@@ -1,16 +1,23 @@
 # frozen_string_literal: true
 
+require_relative 'cluster_engine'
+require_relative 'transport'
+
 module Esse
   class Cluster
-    ATTRIBUTES = %i[index_prefix index_settings client wait_for_status].freeze
+    ATTRIBUTES = %i[index_prefix settings mappings client wait_for_status readonly].freeze
     WAIT_FOR_STATUSES = %w[green yellow red].freeze
 
     # The index prefix. For example an index named UsersIndex.
     # With `index_prefix = 'app1'`. Final index/alias is: 'app1_users'
     attr_accessor :index_prefix
 
-    # This settings will be passed through all indices during the mapping
-    attr_accessor :index_settings
+    # This global settings will be passed through all indices
+    attr_accessor :settings
+
+    # This global mappings will be applied to all indices
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html
+    attr_accessor :mappings
 
     # if this option set, actions such as creating or deleting index,
     # importing data will wait for the status specified. Extremely useful
@@ -21,11 +28,17 @@ module Esse
     #   wait_for_status: green
     attr_accessor :wait_for_status
 
+    # Disable all writes from the application to the underlying Elasticsearch instance while keeping the
+    # application running and handling search requests.
+    attr_writer :readonly
+
     attr_reader :id
 
     def initialize(id:, **options)
       @id = id.to_sym
-      @index_settings = {}
+      @settings = {}
+      @mappings = {}
+      @readonly = false
       assign(options)
     end
 
@@ -41,34 +54,102 @@ module Esse
     end
 
     def client
-      @client ||= Elasticsearch::Client.new
+      @client ||= if defined? Elasticsearch::Client
+        Elasticsearch::Client.new
+      elsif defined? OpenSearch::Client
+        OpenSearch::Client.new
+      else
+        raise Esse::Error, <<~ERROR
+          Elasticsearch::Client or OpenSearch::Client is not defined.
+          Please install elasticsearch or opensearch-ruby gem.
+        ERROR
+      end
+    end
+
+    # @return [Boolean] Return true if the cluster is readonly
+    def readonly?
+      !!@readonly
     end
 
-    # Define the elasticsearch client connectio
-    # @param es_client [Elasticsearch::Client, Hash] an instance of elasticsearch/api client or an hash
-    #   with the settings that will be used to initialize Elasticsearch::Client
+    # @raise [Esse::Transport::ReadonlyClusterError] if the cluster is readonly
+    # @return [void]
+    def throw_error_when_readonly!
+      raise Esse::Transport::ReadonlyClusterError if readonly?
+    end
+
+    # Define the elasticsearch client connection
+    # @param es_client [Elasticsearch::Client, OpenSearch::Client, Hash] an instance of elasticsearch/api client or an hash
+    #   with the settings that will be used to initialize the Client
     def client=(es_client)
-      @client = if es_client.is_a?(Hash)
+      @client = if es_client.is_a?(Hash) && defined?(Elasticsearch::Client)
         settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
         Elasticsearch::Client.new(settings)
+      elsif es_client.is_a?(Hash) && defined?(OpenSearch::Client)
+        settings = es_client.each_with_object({}) { |(k, v), r| r[k.to_sym] = v }
+        OpenSearch::Client.new(settings)
       else
         es_client
       end
     end
 
     def inspect
-      attrs = ([:id] + ATTRIBUTES - [:client]).map do |method|
+      attrs = ([:id] + ATTRIBUTES - [:client, :readonly]).map do |method|
         value = public_send(method)
         format('%<k>s=%<v>p', k: method, v: value) if value
       end.compact
+      attrs << 'readonly=true' if readonly?
       attrs << format('client=%p', @client)
       format('#<Esse::Cluster %<attrs>s>', attrs: attrs.join(' '))
     end
 
-    def wait_for_status!(status: wait_for_status)
+    # Wait until cluster is in a specific state
+    #
+    # @option [String] :status Wait until cluster is in a specific state (options: green, yellow, red)
+    # @option [String] :index Limit the information returned to a specific index
+    def wait_for_status!(status: nil, **kwargs)
+      status ||= wait_for_status
       return unless WAIT_FOR_STATUSES.include?(status.to_s)
 
-      client.cluster.health(wait_for_status: status.to_s)
+      api.health(**kwargs, wait_for_status: status.to_s)
+    end
+
+    # @return [void]
+    def may_update_type!(hash, key: :type)
+      if (single_type = engine.mapping_default_type)
+        hash[key] = single_type
+        return
+      end
+      hash.delete(key) if engine.mapping_single_type?
+    end
+
+    def info
+      @info ||= begin
+        resp = client.info
+        {
+          distribution: (resp.dig('version', 'distribution') || 'elasticsearch'),
+          version: resp.dig('version', 'number'),
+        }
+      end
+    end
+
+    def engine
+      @engine ||=ClusterEngine.new(**info)
+    end
+    alias_method :warm_up!, :engine
+
+    # Build a search query for the given indices
+    #
+    # @param indices [Array<Esse::Index, String>] The indices class or the index name
+    # @return [Esse::Search::Query] The search query instance
+    def search(*indices, **kwargs, &block)
+      Esse::Search::Query.new(api, *indices, **kwargs, &block)
+    end
+
+    # Return the proxy object used to perform low level actions on the elasticsearch cluster through the official api client
+    #
+    # @return [Esse::Transport] The cluster api instance
+    def api
+      Esse::Transport.new(self)
     end
   end
 end

data/lib/esse/cluster_engine.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Esse
+  class ClusterEngine
+    OPENSEARCH_FORK_VERSION = '7.10.2'
+
+    attr_reader :version, :distribution
+
+    def initialize(distribution:, version:)
+      @distribution = distribution
+      @version = version
+    end
+
+    def engine_version
+      return @version unless opensearch?
+
+      OPENSEARCH_FORK_VERSION
+    end
+
+    def opensearch?
+      distribution == 'opensearch'
+    end
+
+    def elasticsearch?
+      distribution == 'elasticsearch'
+    end
+
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.17/removal-of-types.html
+    def mapping_single_type?
+      engine_version >= '6'
+    end
+
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/6.3/mapping.html
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/6.4/mapping.html
+    # @see https://www.elastic.co/guide/en/elasticsearch/reference/7.1/mapping.html
+    def mapping_default_type
+      return unless engine_version.to_i == 6
+
+      engine_version >= '6.4' ? :_doc : :doc
+    end
+  end
+end

data/lib/esse/collection.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Esse
+  class Collection
+    include Enumerable
+    attr_reader :options
+
+    def initialize(**options)
+      @options = options
+    end
+
+    # @yield [<Array, Hash>] A batch of documents to be serialized and indexed.
+    # @abstract Override this method to yield each chunk of documents with optional metadata
+    def each
+      raise NotImplementedError, 'Override this method to iterate over the collection'
+    end
+  end
+end

data/lib/esse/config.rb

@@ -9,10 +9,14 @@ module Esse
   #     conf.cluster(:v1) do |cluster|
   #       cluster.index_prefix = 'backend'
   #       cluster.client = Elasticsearch::Client.new
-  #       cluster.index_settings = {
+  #       cluster.settings = {
   #         number_of_shards: 2,
   #         number_of_replicas: 0
   #       }
+  #       cluster.mappings = {
+  #         dynamic_templates: [...]
+  #         properties: { ... }
+  #       }
   #     end
   #   end
   #
@@ -36,13 +40,17 @@ module Esse
   #   end
   class Config
     DEFAULT_CLUSTER_ID = :default
-    ATTRIBUTES = %i[indices_directory].freeze
+    ATTRIBUTES = %i[indices_directory bulk_wait_interval].freeze
 
     # The location of the indices. Defaults to the `app/indices`
     attr_reader :indices_directory
 
+    # wait a given period between posting pages to give Elasticsearch time to catch up.
+    attr_reader :bulk_wait_interval
+
     def initialize
       self.indices_directory = 'app/indices'
+      self.bulk_wait_interval = 0.1
       @clusters = {}
       cluster(DEFAULT_CLUSTER_ID) # initialize the :default client
     end
@@ -66,6 +74,10 @@ module Esse
       @indices_directory = value.is_a?(Pathname) ? value : Pathname.new(value)
     end
 
+    def bulk_wait_interval=(value)
+      @bulk_wait_interval = value.to_f
+    end
+
     def load(arg)
       case arg
       when Hash

data/lib/esse/core.rb

@@ -1,22 +1,26 @@
 # frozen_string_literal: true
 
-
-require 'multi_json'
-require 'elasticsearch'
-
 module Esse
   require_relative 'config'
   require_relative 'cluster'
   require_relative 'primitives'
-  require_relative 'index_type'
+  require_relative 'collection'
+  require_relative 'document'
+  require_relative 'hash_document'
+  require_relative 'null_document'
+  require_relative 'repository'
   require_relative 'index_setting'
+  require_relative 'dynamic_template'
   require_relative 'index_mapping'
   require_relative 'template_loader'
-  require_relative 'backend/index'
-  require_relative 'backend/index_type'
+  require_relative 'import/request_body'
+  require_relative 'import/bulk'
   require_relative 'version'
   require_relative 'logging'
   require_relative 'events'
+  require_relative 'search/query'
+  require_relative 'search/response'
+  require_relative 'deprecations' # Should be last
   include Logging
 
   @single_threaded = false
@@ -68,4 +72,21 @@ module Esse
     end
     [id, modified]
   end
+
+  def self.eager_load_indices!
+    return false unless Esse.config.indices_directory.exist?
+
+    Dir[Esse.config.indices_directory.join('**/*_index.rb')].map { |path| Pathname.new(path) }.each do |path|
+      next unless path.extname == '.rb'
+
+      require(path.expand_path.to_s)
+    end
+    true
+  end
+
+  def self.document?(object)
+    return false unless object
+
+    !!(object.is_a?(Esse::Document) && object.id)
+  end
 end

data/lib/esse/deprecations/cluster.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Esse
+  class Cluster
+    extend Esse::Deprecations::Deprecate
+
+    def index_settings
+      settings
+    end
+    deprecate :index_settings, :settings, 2023, 12
+
+    def index_settings=(value)
+      self.settings = value
+    end
+    deprecate :index_settings=, :settings=, 2023, 12
+
+    def index_mappings
+      mappings
+    end
+    deprecate :index_mappings, :mappings, 2023, 12
+
+    def index_mappings=(value)
+      self.mappings = value
+    end
+    deprecate :index_mappings=, :mappings=, 2023, 12
+  end
+end

data/lib/esse/deprecations/deprecate.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Esse
+  module Deprecations
+    module Deprecate
+      def self.extended(base)
+        base.extend Gem::Deprecate
+        base.include InstanceMethods
+      end
+
+      module InstanceMethods
+        def warning(method, repl, year, month)
+          msg = ["NOTE: #{method} is deprecated"]
+          msg << if repl == :none
+            ' with no replacement'
+          elsif repl.respond_to?(:call)
+            "; use #{repl.call} instead"
+          else
+            "; use #{repl} instead"
+          end
+          msg << '. It will be removed on or after %4d-%02d-01.' % [year, month]
+          msg << "\n#{method} called from #{Gem.location_of_caller(2).join(':')}"
+
+          warn "#{msg.join}." unless Gem::Deprecate.skip
+        end
+      end
+    end
+  end
+end

data/lib/esse/deprecations/index.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Esse
+  class Index
+    class << self
+      extend Esse::Deprecations::Deprecate
+
+      def define_type(name, *args, **kwargs, &block)
+        repository(name, *args, **kwargs, &block)
+      end
+      deprecate :define_type, :repository, 2023, 12
+
+      def type_hash
+        repo_hash
+      end
+      deprecate :type_hash, :repo_hash, 2023, 12
+
+      def index_version
+        index_suffix
+      end
+      deprecate :index_version, :index_suffix, 2023, 12
+
+      def index_version=(value)
+        self.index_suffix = value
+      end
+      deprecate :index_version=, :index_suffix=, 2023, 12
+
+      def elasticsearch
+        Esse::Deprecations::IndexBackendDelegator.new(:elasticsearch, self)
+      end
+
+      def backend
+        Esse::Deprecations::IndexBackendDelegator.new(:backend, self)
+      end
+    end
+  end
+end