elastic-rails 0.6.4 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66b9bacfe2ed32b87930002d60d7f84d8c996c05
-  data.tar.gz: e2d4f108fb27db7ec85afdb861f35a009db2db0f
+  metadata.gz: f9ab5fe73898a276646cbf7c3f9a564d3f7419e4
+  data.tar.gz: a90c322ea5f8226affff52f5646974ac42aebe5a
 SHA512:
-  metadata.gz: 425e22896f8f062c17dd12ead7ec8f052cf3e6a0151c71c788cc55636f3a3db2351e1a4af8bf6af87b3a7441f13c65ebecef0da7925baac0ddc709aeceffa632
-  data.tar.gz: 9e95d2d6e6ba9d85ae68ad88e241d61ba2558d45e20e1c939006eae783c8c7041f095eb18e2f9dc561f29a6271e7a5430188c252eaa9d81166f770338799212e
+  metadata.gz: 54b6f86610924e06e58690fa99e45d74dd2c18f5fd902985efe5ff6ab827117a44ceb6be50a8637f3dee085591f677514d706556907af15e2c2b2925ebc45980
+  data.tar.gz: 346536b41ffcaef70416b0afd92f2c54ee4b4658077a500cb71cda8c24fd572636d619eae0f4456873b8115c84dbb3db6a1ad12110755780c3663ba5709db417

data/README.md

@@ -60,9 +60,19 @@ BikeIndex.must(brand: 'Trek', size: 'M').should(year: { gte: 2015 }).avg(:price)
 BikeIndex.must(origin: 'China').segment(:brand).each { |brand, bikes|   }
 ```
 
+migrate: remaps if necessary 
+
+reindex: attempts to rotate if index already exist
 
 TODO: Write usage instructions here
 
+## Missing Features
+
+These are some features that will be added in the future:
+
+* Support for record deletion
+
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/elastic/commands/build_agg_from_params.rb

@@ -66,11 +66,20 @@ module Elastic::Commands
     end
 
     def build_date_histogram(_options)
-      Elastic::Nodes::Agg::DateHistogram.build(agg_name, @field, interval: _options[:interval])
+      Elastic::Nodes::Agg::DateHistogram.build(
+        agg_name,
+        @field,
+        interval: _options[:interval],
+        time_zone: _options[:time_zone]
+      )
     end
 
     def build_terms(_options)
-      Elastic::Nodes::Agg::Terms.build(agg_name, @field, size: _options[:size])
+      Elastic::Nodes::Agg::Terms.build(
+        agg_name,
+        @field,
+        size: _options[:size]
+      )
     end
 
     def agg_name

data/lib/elastic/commands/compare_mappings.rb

@@ -0,0 +1,44 @@
+module Elastic::Commands
+  class CompareMappings < Elastic::Support::Command.new(:current, :user)
+    def perform
+      user_properties.select do |field, property|
+        !compare_field_properties(current_properties[field], property)
+      end.map { |f| f[0] }
+    end
+
+    private
+
+    def current_properties
+      @current_properties ||= Hash[flatten(current)]
+    end
+
+    def user_properties
+      @user_properties ||= Hash[flatten(user)]
+    end
+
+    def flatten(_raw, _prefix = '')
+      _raw['properties'].flat_map do |name, raw_field|
+        if raw_field['type'] == 'nested'
+          childs = flatten(raw_field, name + '.')
+          childs << [
+            _prefix + name,
+            raw_field.slice(*(raw_field.keys - ['properties']))
+          ]
+        else
+          [[_prefix + name, raw_field.dup]]
+        end
+      end
+    end
+
+    def compare_field_properties(_current, _user)
+      return false if _current.nil?
+
+      case _current['type']
+      when 'date'
+        return _current == { 'format' => 'dateOptionalTime' }.merge(_user)
+      else
+        return _current == _user
+      end
+    end
+  end
+end

data/lib/elastic/commands/import_index_documents.rb

@@ -3,14 +3,12 @@ module Elastic::Commands
     :index, collection: nil, batch_size: 10000, verbose: false
   )
     def perform
-      index.adaptor.with_settings(refresh_interval: -1) do
-        if collection.present?
-          import_collection
-        else
-          targets.each { |target| import_target(target) }
-        end
-        flush
+      if collection.present?
+        import_collection
+      else
+        targets.each { |target| import_target(target) }
       end
+      flush
     end
 
     private
@@ -34,7 +32,7 @@ module Elastic::Commands
 
     def flush
       unless cache.empty?
-        index.adaptor.bulk_index(cache)
+        index.connector.bulk_index(cache)
         log_flush(cache.size) if verbose
         cache.clear
       end
@@ -43,11 +41,11 @@ module Elastic::Commands
     def log_flush(_size)
       @total ||= 0
       @total += _size
-      Elastic::Configuration.logger.info "Imported #{@total} documents"
+      Elastic.logger.info "Imported #{@total} documents"
     end
 
     def render_for_es(_object)
-      index.new(_object).as_es_document
+      index.new(_object).as_elastic_document
     end
 
     def main_target

data/lib/elastic/configuration.rb

@@ -1,69 +1,62 @@
 module Elastic
-  module Configuration
-    DEFAULT = {
+  class Configuration
+    DEFAULTS = {
       host: '127.0.0.1',
       port: 9200,
       page_size: 20,
       coord_similarity: true,
-      import_batch_size: 10_000
+      import_batch_size: 10_000,
+      whiny_indices: false,
+      api_client: nil, # set by method
+      logger: nil, # set by method
+      time_zone: nil # set by method
     }
 
-    extend self
+    attr_accessor :host, :port, :api_client, :index, :page_size, :coord_similarity, :logger,
+      :import_batch_size, :whiny_indices, :time_zone
+
+    def initialize
+      assign_attributes DEFAULTS
+    end
 
     def reset
-      @config = nil
-      self
+      assign_attributes DEFAULTS
     end
 
-    def configure(_options = nil, &_block)
-      if _options.nil?
-        _block.call self
-      else
-        @config = config.merge _options.symbolize_keys
-      end
+    def assign_attributes(_options)
+      _options.each { |k, v| public_send("#{k}=", v) }
       self
     end
 
     def api_client
-      config[:client] ||= load_api_client
+      @api_client || default_api_client
     end
 
-    def index_name
-      config[:index]
-    end
-
-    def indices_path
-      'app/indices'
-    end
-
-    def page_size
-      @config[:page_size]
-    end
-
-    def coord_similarity
-      @config[:coord_similarity]
+    def logger
+      @logger || default_logger
     end
 
-    def logger
-      @config[:logger] || default_logger
+    def time_zone
+      @time_zone || default_time_zone
     end
 
-    def import_batch_size
-      @config[:import_batch_size]
+    def time_zone=(_value)
+      _value = ActiveSupport::TimeZone.new(_value) if _value.is_a? String
+      @time_zone = _value
     end
 
     private
 
-    def config
-      @config ||= DEFAULT
+    def default_api_client
+      @default_api_client ||= Elasticsearch::Client.new host: @host, port: @port
     end
 
     def default_logger
       @default_logger ||= Logger.new(STDOUT)
     end
 
-    def load_api_client
-      Elasticsearch::Client.new host: config[:host], port: config[:port]
+    def default_time_zone
+      @default_time_zone ||= ActiveSupport::TimeZone.new('UTC')
     end
   end
 end

data/lib/elastic/core/connector.rb

@@ -0,0 +1,253 @@
+module Elastic::Core
+  class Connector
+    def initialize(_name, _types, _mapping)
+      @name = _name
+      @types = _types
+      @mapping = _mapping
+    end
+
+    def index_name
+      @index_name ||= "#{Elastic.config.index}_#{@name}"
+    end
+
+    def read_index_name
+      index_name
+    end
+
+    def write_index_name
+      Thread.current[write_index_thread_override] || write_index_alias
+    end
+
+    def status
+      actual_name = resolve_actual_index_name
+      return :not_available if actual_name.nil?
+      return :not_synchronized unless mapping_synchronized? actual_name
+      :ready
+    end
+
+    def drop
+      api.indices.delete index: "#{index_name}:*"
+      nil
+    end
+
+    def remap
+      case status
+      when :not_available
+        create_from_scratch
+      when :not_synchronized
+        begin
+          setup_index_types resolve_actual_index_name
+        rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+          return false
+        end
+      end
+
+      true
+    end
+
+    def migrate(batch_size: nil)
+      unless remap
+        rollover do
+          copy_documents(read_index_name, write_index_name, batch_size || default_batch_size)
+        end
+      end
+
+      nil
+    end
+
+    def index(_document)
+      # TODO: validate document type
+
+      api.index(
+        index: write_index_name,
+        id: _document['_id'],
+        type: _document['_type'],
+        body: _document['data']
+      )
+    end
+
+    def bulk_index(_documents)
+      # TODO: validate documents type
+
+      body = _documents.map { |doc| { 'index' => doc } }
+
+      retry_on_temporary_error('bulk indexing') do
+        api.bulk(index: write_index_name, body: body)
+      end
+    end
+
+    def refresh
+      api.indices.refresh index: read_index_name
+    end
+
+    def find(_type, _id)
+      api.get(index: write_index_name, type: _type, id: _id)
+    end
+
+    def delete(_type, _id)
+      api.delete(index: write_index_name, type: _type, id: _id)
+    end
+
+    def count(query: nil, type: nil)
+      api.count(index: read_index_name, type: type, body: query)['count']
+    end
+
+    def query(query: nil, type: nil)
+      api.search(index: read_index_name, type: type, body: query)
+    end
+
+    def rollover(&_block) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      new_index = create_index_w_mapping
+      tmp_index = create_index_w_mapping('tmp')
+      actual_index = resolve_actual_index_name
+
+      begin
+        transfer_alias(write_index_alias, from: actual_index, to: tmp_index)
+
+        perform_optimized_write_on(new_index, &_block)
+
+        transfer_alias(index_name, from: actual_index, to: new_index)
+        transfer_alias(write_index_alias, from: tmp_index, to: new_index)
+        api.indices.delete index: actual_index if actual_index
+      rescue
+        transfer_alias(write_index_alias, from: tmp_index, to: actual_index)
+        api.indices.delete index: new_index
+      ensure
+        # rollback
+        # TODO: what would happen if the following fails? O.O
+        copy_documents(tmp_index, write_index_name, small_batch_size)
+        api.indices.delete index: tmp_index
+        api.indices.refresh index: index_name
+      end
+    end
+
+    private
+
+    def api
+      Elastic.config.api_client
+    end
+
+    def perform_optimized_write_on(_index)
+      old_index = Thread.current[write_index_thread_override]
+      Thread.current[write_index_thread_override] = _index
+      configure_index(_index, refresh_interval: -1)
+      yield
+    ensure
+      configure_index(_index, refresh_interval: '1s')
+      Thread.current[write_index_thread_override] = old_index
+    end
+
+    def write_index_thread_override
+      "_elastic_#{index_name}_write_index"
+    end
+
+    def write_index_alias
+      @write_index_alias = "#{index_name}.w"
+    end
+
+    def resolve_actual_index_name
+      result = api.indices.get_alias(name: index_name)
+      result.keys.first
+    rescue Elasticsearch::Transport::Transport::Errors::NotFound
+      nil
+    end
+
+    def create_index_w_mapping(_role = 'main')
+      new_name = "#{index_name}:#{_role}:#{Time.now.to_i}"
+      api.indices.create index: new_name
+      api.cluster.health wait_for_status: 'yellow'
+      setup_index_types new_name
+      new_name
+    end
+
+    def create_from_scratch
+      new_index = create_index_w_mapping
+      api.indices.update_aliases(
+        body: {
+          actions: [
+            { add: { index: new_index, alias: index_name } },
+            { add: { index: new_index, alias: write_index_alias } }
+          ]
+        }
+      )
+    end
+
+    def mapping_synchronized?(_index)
+      type_mappings = api.indices.get_mapping(index: _index)
+      return false if type_mappings[_index].nil?
+      type_mappings = type_mappings[_index]['mappings']
+
+      @types.all? do |type|
+        next false if type_mappings[type].nil?
+
+        diff = Elastic::Commands::CompareMappings.for(
+          current: type_mappings[type],
+          user: @mapping
+        )
+        diff.empty?
+      end
+    end
+
+    def setup_index_types(_index)
+      @types.each do |type|
+        api.indices.put_mapping(index: _index, type: type, body: @mapping)
+      end
+    end
+
+    def transfer_alias(_alias, from: nil, to: nil)
+      actions = []
+      actions << { remove: { index: from, alias: _alias } } if from
+      actions << { add: { index: to, alias: _alias } } if to
+      api.indices.update_aliases body: { actions: actions }
+    end
+
+    def copy_documents(_from, _to, _batch_size)
+      api.indices.refresh index: _from
+
+      r = api.search(
+        index: _from,
+        body: { sort: ['_doc'] },
+        scroll: '5m',
+        size: _batch_size
+      )
+
+      count = 0
+      while !r['hits']['hits'].empty?
+        count += r['hits']['hits'].count
+        Elastic.logger.info "Copied #{count} docs"
+
+        body = r['hits']['hits'].map { |h| { 'index' => transform_hit_to_doc(h) } }
+        api.bulk(index: _to, body: body)
+
+        r = api.scroll scroll: '5m', scroll_id: r['_scroll_id']
+      end
+    end
+
+    def configure_index(_index, _settings)
+      api.indices.put_settings index: _index, body: { index: _settings }
+    end
+
+    def transform_hit_to_doc(_hit)
+      { '_id' => _hit['_id'], '_type' => _hit['_type'], 'data' => _hit['_source'] }
+    end
+
+    def default_batch_size
+      1_000
+    end
+
+    def small_batch_size
+      500
+    end
+
+    def retry_on_temporary_error(_action, retries: 3)
+      return yield
+    rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable,
+           Elasticsearch::Transport::Transport::Errors::GatewayTimeout => exc
+      raise if retries <= 0
+
+      Elastic.logger.warn("#{exc.class} error during '#{_action}', retrying!")
+      retries -= 1
+      retry
+    end
+  end
+end

data/lib/elastic/core/definition.rb

@@ -38,10 +38,6 @@ module Elastic::Core
       @field_map.each_value
     end
 
-    def expanded_field_names
-      @field_map.map { |_, field| field.expanded_names }.flatten
-    end
-
     def freeze
       return if frozen?
       cache_targets

data/lib/elastic/core/query_assembler.rb

@@ -55,7 +55,7 @@ module Elastic::Core
       query = build_base_query
 
       if !grouped?
-        query.size = (@config.limit || Elastic::Configuration.page_size)
+        query.size = (@config.limit || Elastic.config.page_size)
         query.offset = @config.offset
         query = sort_node(query)
       else

data/lib/elastic/core/serializer.rb

@@ -16,7 +16,17 @@ module Elastic::Core
       @definition.fields
     end
 
-    def as_es_document(only_data: false)
+    def read_elastic_type
+      object.class.to_s
+    end
+
+    def read_elastic_id
+      if has_attribute_for_indexing?(:id)
+        read_attribute_for_indexing(:id)
+      end
+    end
+
+    def as_elastic_document(only_data: false)
       data = {}.tap do |hash|
         fields.each do |field|
           value = read_attribute_for_indexing(field.name)
@@ -27,8 +37,12 @@ module Elastic::Core
 
       return data if only_data
 
-      result = { '_type' => object.class.to_s, 'data' => data }
-      result['_id'] = read_attribute_for_indexing(:id) if has_attribute_for_indexing?(:id)
+      result = {
+        '_type' => read_elastic_type,
+        'data' => data
+      }
+
+      read_elastic_id.tap { |id| result['_id'] = id unless id.nil? }
       result
     end
 

data/lib/elastic/datatypes/date.rb

@@ -13,9 +13,9 @@ module Elastic::Datatypes
     def prepare_value_for_result(_value)
       case _value
       when ::String
-        ::Time.parse(_value).utc.to_date
+        time_zone.parse(_value).to_date
       when ::Integer
-        ::Time.at(_value / 1000).utc.to_date
+        time_zone.at(_value / 1000).to_date
       else
         _value
       end
@@ -26,7 +26,13 @@ module Elastic::Datatypes
     end
 
     def date_histogram_aggregation_defaults
-      { interval: '1w' }
+      { interval: '1w', time_zone: time_zone }
+    end
+
+    private
+
+    def time_zone
+      @time_zone ||= ActiveSupport::TimeZone.new('UTC') # dates are always UTC
     end
   end
 end

data/lib/elastic/datatypes/time.rb

@@ -10,9 +10,9 @@ module Elastic::Datatypes
       # TODO: set timezone
       case _value
       when ::String
-        ::Time.parse(_value)
+        time_zone.parse(_value)
       when ::Integer
-        ::Time.at(_value / 1000)
+        time_zone.at(_value / 1000)
       else
         _value
       end
@@ -23,7 +23,14 @@ module Elastic::Datatypes
     end
 
     def date_histogram_aggregation_defaults
-      { interval: '1h' }
+      { interval: '1h', time_zone: time_zone }
+    end
+
+    private
+
+    def time_zone
+      # TODO: user_options[:timezone]
+      Elastic.config.time_zone
     end
   end
 end

data/lib/elastic/errors.rb

@@ -0,0 +1,4 @@
+module Elastic
+  class Error < StandardError
+  end
+end

data/lib/elastic/fields/nested.rb

@@ -11,10 +11,6 @@ module Elastic::Fields
       # does nothing
     end
 
-    def expanded_names
-      [@name] + @index.definition.expanded_field_names.map { |n| @name + '.' + n }
-    end
-
     def validate
       nil
     end
@@ -45,7 +41,7 @@ module Elastic::Fields
     end
 
     def prepare_value_for_index(_values)
-      _values.map { |v| @index.new(v).as_es_document(only_data: true) }
+      _values.map { |v| @index.new(v).as_elastic_document(only_data: true) }
     end
 
     def prepare_value_for_result(_values)

data/lib/elastic/fields/value.rb

@@ -35,10 +35,6 @@ module Elastic::Fields
       nil
     end
 
-    def expanded_names
-      [@name]
-    end
-
     def needs_inference?
       mapping_inference_enabled? && !@options.key?(:type)
     end

data/lib/elastic/nodes/agg/date_histogram.rb

@@ -3,21 +3,27 @@ module Elastic::Nodes::Agg
     include Elastic::Nodes::Concerns::Aggregable
     include Elastic::Nodes::Concerns::Bucketed
 
-    def self.build(_name, _field, interval: nil)
+    def self.build(_name, _field, interval: nil, time_zone: nil)
       super(_name).tap do |node|
         node.field = _field
         node.interval = interval
+        node.time_zone = time_zone
       end
     end
 
     attr_accessor :field
-    attr_reader :interval
+    attr_reader :interval, :time_zone
 
     def interval=(_value)
       raise ArgumentError, 'invalid interval' if _value && !valid_interval?(_value)
       @interval = _value
     end
 
+    def time_zone=(_value)
+      raise ArgumentError, 'invalid time_zone' if _value && !_value.is_a?(ActiveSupport::TimeZone)
+      @time_zone = _value
+    end
+
     def clone
       prepare_clone(super)
     end
@@ -29,6 +35,7 @@ module Elastic::Nodes::Agg
     def render(_options = {})
       hash = { 'field' => @field.to_s }
       hash['interval'] = @interval if @interval
+      hash['time_zone'] = @time_zone.formatted_offset if @time_zone
 
       render_aggs({ 'date_histogram' => hash }, _options)
     end
@@ -38,6 +45,7 @@ module Elastic::Nodes::Agg
     def prepare_clone(_clone)
       _clone.field = @field
       _clone.interval = @interval
+      _clone.time_zone = @time_zone
       _clone
     end