elastic-rails 0.6.4 → 0.7.0

data/lib/elastic/nodes/boolean.rb

@@ -18,7 +18,7 @@ module Elastic::Nodes
       @must_nots = []
       @shoulds = []
       @filters = []
-      @disable_coord = !Elastic::Configuration.coord_similarity
+      @disable_coord = !Elastic.config.coord_similarity
     end
 
     def must(_node)

data/lib/elastic/query.rb

@@ -86,7 +86,7 @@ module Elastic
     end
 
     def execute(_query)
-      raw = @index.adaptor.query(
+      raw = @index.connector.query(
         type: @index.definition.types,
         query: _query.render
       )

data/lib/elastic/railtie.rb

@@ -4,13 +4,16 @@ require "elastic/railties/ar_middleware"
 require "elastic/railties/configuration_extensions"
 require "elastic/railties/type_extensions"
 require "elastic/railties/query_extensions"
-require "elastic/railties/indexing_job"
+# disabled for now: require "elastic/railties/indexing_job"
 require "elastic/railties/indexable_record"
 
 module Elastic
   class Railtie < Rails::Railtie
     initializer "elastic.configure_rails_initialization" do
-      Elastic.configure Rails.application.config_for(:elastic).merge(logger: Rails.logger)
+      Elastic.configure Rails.application.config_for(:elastic).merge(
+        time_zone: Rails.application.config.time_zone,
+        logger: Rails.logger
+      )
 
       # Make every activerecord model indexable
       ActiveRecord::Base.send(:include, Elastic::Railties::IndexableRecord)
@@ -30,7 +33,7 @@ module Elastic
 end
 
 # Add activerecord related configuration parameters
-module Elastic::Configuration
+class Elastic::Configuration
   include Elastic::Railties::ConfigurationExtensions
 end
 

data/lib/elastic/railties/configuration_extensions.rb

@@ -1,13 +1,10 @@
 module Elastic::Railties
   module ConfigurationExtensions
     def self.included(_klass)
-      _klass.extend ClassMethods
+      _klass::DEFAULTS[:active_job_queue] = :default
+      _klass::DEFAULTS[:indices_path] = 'app/indices'
     end
 
-    module ClassMethods
-      def active_job_queue
-        config[:active_job_queue] || :default
-      end
-    end
+    attr_accessor :active_job_queue, :indices_path
   end
 end

data/lib/elastic/railties/indexable_record.rb

@@ -18,21 +18,31 @@ module Elastic::Railties
         @constantized_index_class ||= index_class.constantize
       end
 
-      def index(_options)
-        on = _options.delete(:on)
+      def index(on: nil, unindex: true, delayed: false)
+        raise NotImplementedError, 'delayed indexing not implemented' if delayed
+
         if on == :create
-          index_on_create _options
+          index_on_create
         elsif on == :save
-          index_on_save _options
+          index_on_save
+        else
+          raise ArgumentError, 'must provide an indexing target when calling index \
+(ie: `index on: :save`)'
         end
+
+        unindex_on_destroy if unindex
       end
 
       def index_on_create(_options = {})
-        after_create(_options) { index_later }
+        after_create(_options) { index_now }
       end
 
       def index_on_save(_options = {})
-        after_save(_options) { index_later }
+        after_save(_options) { index_now }
+      end
+
+      def unindex_on_destroy(_options = {})
+        before_destroy(_options) { unindex_now }
       end
     end
 
@@ -40,8 +50,8 @@ module Elastic::Railties
       self.class.constantized_index_class.index self
     end
 
-    def index_later
-      IndexingJob.set(queue: Elastic::Configuration.active_job_queue).perform_later(self)
+    def unindex_now
+      self.class.constantized_index_class.delete self
     end
   end
 end

data/lib/elastic/railties/tasks/es.rake

@@ -1,19 +1,25 @@
 namespace :es do
   desc "Elastic: Updates indices mappings"
-  task migrate: :environment do
+  task :remap, [:index] => :environment do |_, args|
     Elastic.configure logger: Logger.new(STDOUT)
-    Elastic.migrate
+    Elastic.remap args.index
+  end
+
+  desc "Elastic: Updates indices mappings, rebuilding index if necessary"
+  task :migrate, [:index] => :environment do |_, args|
+    Elastic.configure logger: Logger.new(STDOUT)
+    Elastic.migrate args.index
   end
 
   desc "Elastic: Rebuilds indices from source data"
-  task reindex: :environment do
+  task :reindex, [:index] => :environment do |_, args|
     Elastic.configure logger: Logger.new(STDOUT)
-    Elastic.reindex
+    Elastic.reindex args.index
   end
 
   desc "Elastic: Lists indices stats"
-  task stats: :environment do
+  task :stats, [:index] => :environment do |_, args|
     Elastic.configure logger: Logger.new(STDOUT)
-    Elastic.stats
+    Elastic.stats args.index
   end
 end

data/lib/elastic/railties/utils.rb

@@ -8,11 +8,23 @@ module Elastic::Railties
       end
     end
 
+    def remap(_index = nil)
+      logger.info "Remapping all indices" if _index.nil?
+      indices(_index).each do |index|
+        logger.info "Remapping index #{index.suffix}"
+        handle_errors do
+          unless index.connector.remap
+            logger.info 'Mapping couldnt be changed, make sure you call migrate'
+          end
+        end
+      end
+    end
+
     def migrate(_index = nil)
       logger.info "Migrating all indices" if _index.nil?
       indices(_index).each do |index|
         logger.info "Migrating index #{index.suffix}"
-        handle_errors { index.mapping.migrate }
+        handle_errors { index.migrate }
       end
     end
 
@@ -45,7 +57,7 @@ module Elastic::Railties
     end
 
     def indices_paths
-      Rails.root.join(Elastic::Configuration.indices_path)
+      Rails.root.join(Elastic.config.indices_path)
     end
 
     def handle_errors
@@ -56,7 +68,7 @@ module Elastic::Railties
     end
 
     def logger
-      Elastic::Configuration.logger
+      Elastic.logger
     end
   end
 end

data/lib/elastic/type.rb

@@ -10,6 +10,10 @@ module Elastic
         :coord_similarity, :limit, :offset, :pluck, :ids, :total
     end
 
+    def self.default_suffix
+      to_s.underscore
+    end
+
     def self.suffix
       @suffix || default_suffix
     end
@@ -19,38 +23,49 @@ module Elastic
     end
 
     def self.import_batch_size
-      @import_batch_size || Configuration.import_batch_size
+      @import_batch_size || Elastic.config.import_batch_size
     end
 
     def self.import_batch_size=(_value)
       @import_batch_size = _value
     end
 
-    def self.adaptor
-      @adaptor ||= Elastic::Core::Adaptor.new(suffix)
+    def self.connector
+      @connector ||= begin
+        Elastic::Core::Connector.new(
+          suffix,
+          definition.types,
+          definition.as_es_mapping
+        ).tap do |conn|
+          if Elastic.config.whiny_indices && conn.status != :ready
+            raise 'elastic index out of sync, try migrating'
+          end
+        end
+      end
     end
 
-    def self.mapping
-      @mapping ||= load_mapping
+    def self.index_name
+      connector.index_name
     end
 
-    def self.reindex(verbose: true, batch_size: nil)
-      drop
-      mapping.migrate
-      batch_size = batch_size || import_batch_size
+    def self.migrate
+      connector.migrate(batch_size: import_batch_size)
+      self
+    end
 
-      Commands::ImportIndexDocuments.for(
-        index: self,
-        verbose: verbose,
-        batch_size: batch_size
-      )
+    def self.reindex(verbose: true)
+      connector.rollover do
+        Commands::ImportIndexDocuments.for(
+          index: self,
+          verbose: verbose,
+          batch_size: import_batch_size
+        )
+      end
 
-      ensure_full_mapping
       self
     end
 
     def self.import(_collection, batch_size: nil)
-      enforce_mapping!
       batch_size = batch_size || import_batch_size
 
       Commands::ImportIndexDocuments.for(
@@ -59,7 +74,6 @@ module Elastic
         batch_size: batch_size
       )
 
-      ensure_full_mapping
       self
     end
 
@@ -67,52 +81,37 @@ module Elastic
       new(_object).save
     end
 
+    def self.delete(_object)
+      wrapper = new(_object)
+      id = wrapper.read_elastic_id
+      raise ArgumentError, 'index does not provide an id' if id.nil?
+
+      connector.delete(
+        wrapper.read_elastic_type,
+        wrapper.read_elastic_id
+      )
+
+      nil
+    end
+
     def self.query
-      enforce_mapping!
-      ensure_full_mapping
       Query.new self
     end
 
     def self.drop
-      adaptor.drop if adaptor.exists?
+      connector.drop
       self
     end
 
     def self.refresh
-      adaptor.refresh
+      connector.refresh
       self
     end
 
-    def self.enforce_mapping!
-      if mapping.out_of_sync?
-        raise 'elastic mapping out of sync, run `rake es:migrate`'
-      end
-    end
-
-    def self.ensure_full_mapping
-      if mapping.incomplete?
-        mapping.fetch
-      end
-    end
-
     def save
       self.class.tap do |klass|
-        klass.enforce_mapping!
-        klass.adaptor.index as_es_document
-        klass.ensure_full_mapping
+        klass.connector.index as_elastic_document
       end
     end
-
-    def self.load_mapping
-      Elastic::Core::MappingManager.new(adaptor, definition).tap(&:fetch)
-    end
-
-    private_class_method :load_mapping
-
-    def self.default_suffix
-      to_s.underscore
-    end
-
-    private_class_method :default_suffix
   end
 end

data/lib/elastic/version.rb

@@ -1,3 +1,3 @@
 module Elastic
-  VERSION = "0.6.4"
+  VERSION = "0.7.0"
 end

data/lib/elastic.rb

@@ -2,6 +2,7 @@ require "elasticsearch"
 
 require "elastic/version"
 require "elastic/configuration"
+require "elastic/errors"
 
 require "elastic/support/command"
 require "elastic/support/transform"
@@ -11,6 +12,7 @@ require "elastic/commands/import_index_documents"
 require "elastic/commands/build_query_from_params"
 require "elastic/commands/build_agg_from_params"
 require "elastic/commands/build_sort_from_params"
+require "elastic/commands/compare_mappings"
 
 require "elastic/results/base"
 require "elastic/results/scored_item"
@@ -64,9 +66,8 @@ require "elastic/datatypes/time"
 require "elastic/fields/value"
 require "elastic/fields/nested"
 
+require "elastic/core/connector"
 require "elastic/core/definition"
-require "elastic/core/adaptor"
-require "elastic/core/mapping_manager"
 require "elastic/core/serializer"
 require "elastic/core/middleware"
 require "elastic/core/base_middleware"
@@ -89,8 +90,17 @@ require "elastic/query"
 require "elastic/nested_query"
 
 module Elastic
-  def self.configure(*_args)
-    Configuration.configure(*_args)
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  def self.logger
+    config.logger
+  end
+
+  def self.configure(_options = nil, &_block)
+    config.assign_attributes(_options) unless _options.nil?
+    _block.call(config) unless _block.nil?
   end
 
   def self.register_middleware(_middleware)

data/lib/generators/elastic/templates/elastic.yml

@@ -2,6 +2,7 @@ development:
   host: '127.0.0.1'
   port: 9200
   index: '<%= Rails.application.class.parent_name.to_s.underscore %>_develop'
+  whiny_indices: true
 
 test:
   host: '127.0.0.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elastic-rails
 version: !ruby/object:Gem::Version
-  version: 0.6.4
+  version: 0.7.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2016-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -228,13 +228,13 @@ files:
 - lib/elastic/commands/build_agg_from_params.rb
 - lib/elastic/commands/build_query_from_params.rb
 - lib/elastic/commands/build_sort_from_params.rb
+- lib/elastic/commands/compare_mappings.rb
 - lib/elastic/commands/import_index_documents.rb
 - lib/elastic/configuration.rb
-- lib/elastic/core/adaptor.rb
 - lib/elastic/core/base_middleware.rb
+- lib/elastic/core/connector.rb
 - lib/elastic/core/default_middleware.rb
 - lib/elastic/core/definition.rb
-- lib/elastic/core/mapping_manager.rb
 - lib/elastic/core/middleware.rb
 - lib/elastic/core/query_assembler.rb
 - lib/elastic/core/query_config.rb
@@ -248,6 +248,7 @@ files:
 - lib/elastic/dsl/bool_query_builder.rb
 - lib/elastic/dsl/bool_query_context.rb
 - lib/elastic/dsl/metric_builder.rb
+- lib/elastic/errors.rb
 - lib/elastic/fields/nested.rb
 - lib/elastic/fields/value.rb
 - lib/elastic/nested_query.rb

data/lib/elastic/core/adaptor.rb

@@ -1,126 +0,0 @@
-module Elastic::Core
-  class Adaptor
-    DEFAULT_SETTINGS = {
-      refresh_interval: '1s',
-      number_of_replicas: 1
-    }
-
-    def initialize(_suffix)
-      @suffix = _suffix
-    end
-
-    def index_name
-      @index_name ||= "#{Elastic::Configuration.index_name}_#{@suffix}"
-    end
-
-    def remap(_type, _mapping)
-      # TODO
-    end
-
-    def exists?
-      api_client.indices.exists? build_options
-    end
-
-    def ensure_index
-      create unless exists?
-      self
-    end
-
-    def create
-      api_client.indices.create build_options
-      api_client.cluster.health wait_for_status: 'yellow'
-      self
-    end
-
-    def drop
-      api_client.indices.delete build_options
-      self
-    end
-
-    def exists_type?(_type)
-      api_client.indices.exists_type build_options(type: _type)
-    end
-
-    def exists_mapping?(_type)
-      !api_client.indices.get_mapping(build_options(type: _type)).empty?
-    end
-
-    def get_mappings(type: nil)
-      mappings = api_client.indices.get_mapping build_options(type: type)
-      mappings[index_name]['mappings']
-    end
-
-    def set_mapping(_type, _mapping)
-      api_client.indices.put_mapping build_options(
-        type: _type,
-        body: _mapping
-      )
-      self
-    end
-
-    def index(_document)
-      api_client.index build_options(
-        id: _document['_id'],
-        type: _document['_type'],
-        body: _document['data']
-      )
-      self
-    end
-
-    def bulk_index(_documents)
-      body = _documents.map { |doc| { 'index' => doc } }
-
-      retry_on_temporary_error('bulk indexing') do
-        api_client.bulk build_options(body: body)
-      end
-
-      self
-    end
-
-    def with_settings(_options)
-      _options = DEFAULT_SETTINGS.merge _options
-      api_client.indices.put_settings build_options(body: { index: _options })
-      yield
-    ensure
-      api_client.indices.put_settings build_options(body: { index: DEFAULT_SETTINGS })
-    end
-
-    def refresh
-      api_client.indices.refresh build_options
-      self
-    end
-
-    def find(_id, type: '_all')
-      api_client.get build_options(type: type, id: _id)
-    end
-
-    def count(type: nil, query: nil)
-      api_client.count(build_options(type: type, body: query))['count']
-    end
-
-    def query(type: nil, query: nil)
-      api_client.search build_options(type: type, body: query)
-    end
-
-    private
-
-    def retry_on_temporary_error(_action, retries: 3)
-      return yield
-    rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable,
-           Elasticsearch::Transport::Transport::Errors::GatewayTimeout => exc
-      raise if retries <= 0
-
-      Configuration.logger.warn("#{exc.class} error during '#{_action}', retrying!")
-      retries -= 1
-      retry
-    end
-
-    def api_client
-      Elastic::Configuration.api_client
-    end
-
-    def build_options(_options = {})
-      { index: index_name }.merge! _options
-    end
-  end
-end

data/lib/elastic/core/mapping_manager.rb

@@ -1,120 +0,0 @@
-module Elastic::Core
-  class MappingManager
-    attr_reader :adaptor, :definition
-
-    def initialize(_adaptor, _definition)
-      @adaptor = _adaptor
-      @definition = _definition
-      @status = :pending
-    end
-
-    def out_of_sync?
-      @status == :out_of_sync
-    end
-
-    def incomplete?
-      @status == :incomplete
-    end
-
-    def fetch
-      begin
-        mappings = @adaptor.get_mappings
-        mappings = @definition.types.map { |t| mappings[t] }.reject(&:nil?)
-        @index = merge_mappings_into_index mappings
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound
-        # ignore not-found errors when fetching mappings
-        @index = nil
-      end
-
-      @status = compute_status
-      self
-    end
-
-    def unmapped_fields
-      @definition.expanded_field_names.reject { |f| has_field? f }
-    end
-
-    def has_field?(_name)
-      @index.key? _name
-    end
-
-    def get_field_options(_name)
-      @index[_name]
-    end
-
-    def migrate
-      # TODO: make this a command
-      @adaptor.create unless @adaptor.exists?
-      begin
-        @definition.types.each { |t| @adaptor.set_mapping(t, user_mapping) }
-      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
-        # TODO: https://www.elastic.co/guide/en/elasticsearch/guide/current/reindex.html
-      end
-      fetch
-    end
-
-    private
-
-    def compute_status
-      if !synchronized?
-        :out_of_sync
-      elsif unmapped_fields.count > 0
-        :incomplete
-      else
-        :ready
-      end
-    end
-
-    def synchronized?
-      return false if @index.nil?
-      flatten(user_mapping).all? do |field, properties|
-        compare_field_properties(@index[field], properties)
-      end
-    end
-
-    def compare_field_properties(_current, _user)
-      return false if _current.nil?
-
-      case _current['type']
-      when 'date'
-        _current = { 'format' => 'dateOptionalTime' }.merge(_user)
-      else
-        _current == _user
-      end
-    end
-
-    def user_mapping
-      @user_mapping ||= definition.as_es_mapping
-    end
-
-    def flatten(_raw, _prefix = '')
-      _raw['properties'].flat_map do |name, raw_field|
-        if raw_field['type'] == 'nested'
-          childs = flatten(raw_field, name + '.')
-          childs << [
-            _prefix + name,
-            raw_field.slice(*(raw_field.keys - ['properties']))
-          ]
-        else
-          [[_prefix + name, raw_field.dup]]
-        end
-      end
-    end
-
-    def merge_mappings_into_index(_mappings)
-      {}.tap do |result|
-        _mappings.each do |mapping|
-          index = flatten(mapping)
-          index.each do |field, properties|
-            if result.key? field
-              result[field].merge! properties
-            else
-              result[field] = properties
-            end
-          end
-        end
-        result.each_value(&:freeze)
-      end
-    end
-  end
-end