elastic-rails 0.1.0 → 0.5.0

data/lib/elastic/nodes/nested.rb

@@ -0,0 +1,42 @@
+module Elastic::Nodes
+  class Nested < Base
+    def self.build(_path, _child)
+      new.tap do |node|
+        node.path = _path
+        node.child = _child
+      end
+    end
+
+    attr_accessor :path, :child
+
+    def traverse(&_block)
+      super
+      @child.traverse(&_block)
+    end
+
+    def clone
+      prepare_clone super, @child.clone
+    end
+
+    def simplify
+      prepare_clone super, @child.simplify
+    end
+
+    def render
+      {
+        "nested" => {
+          "path" => @path,
+          "query" => @child.render
+        }
+      }
+    end
+
+    private
+
+    def prepare_clone(_clone, _child)
+      _clone.path = @path
+      _clone.child = _child
+      _clone
+    end
+  end
+end

data/lib/elastic/nodes/or.rb

@@ -0,0 +1,9 @@
+module Elastic::Nodes
+  class Or < And
+    private
+
+    def operation
+      'or'
+    end
+  end
+end

data/lib/elastic/nodes/range.rb

@@ -0,0 +1,36 @@
+module Elastic::Nodes
+  class Range < Base
+    include Concerns::Boostable
+
+    attr_accessor :field, :gte, :gt, :lte, :lt
+
+    def clone
+      prepare_clone(super)
+    end
+
+    def simplify
+      prepare_clone(super)
+    end
+
+    def render
+      options = {}
+      options['gte'] = @gte unless @gte.nil?
+      options['gt'] = @gt unless @gt.nil?
+      options['lte'] = @lte unless @lte.nil?
+      options['lt'] = @lt unless @lt.nil?
+
+      { "range" => { @field.to_s => render_boost(options) } }
+    end
+
+    private
+
+    def prepare_clone(_clone)
+      _clone.field = @field
+      _clone.gte = @gte
+      _clone.gt = @gt
+      _clone.lte = @lte
+      _clone.lt = @lt
+      _clone
+    end
+  end
+end

data/lib/elastic/nodes/search.rb

@@ -0,0 +1,48 @@
+module Elastic::Nodes
+  class Search < Base
+    include Concerns::Aggregable
+    include Concerns::HitProvider
+
+    attr_accessor :query, :offset
+
+    def self.build(_query)
+      new.tap { |n| n.query = _query }
+    end
+
+    def traverse(&_block)
+      super
+      @query.traverse(&_block)
+    end
+
+    def render
+      { "query" => @query.render }.tap do |options|
+        options["from"] = @offset if offset && offset > 0
+        render_hit_options(options)
+        render_aggs(options)
+      end
+    end
+
+    def clone
+      prepare_clone(super, @query.clone)
+    end
+
+    def simplify
+      prepare_clone(super, @query.simplify)
+    end
+
+    def handle_result(_raw)
+      hits = _raw['hits'] ? _raw['hits']['hits'].map { |h| Elastic::Results::Hit.new h } : []
+      aggs = _raw['aggregations'] ? load_aggs_results(_raw['aggregations']) : {}
+
+      Elastic::Results::Root.new(hits, aggs)
+    end
+
+    private
+
+    def prepare_clone(_clone, _query)
+      _clone.query = _query
+      _clone.offset = @offset
+      _clone
+    end
+  end
+end

data/lib/elastic/nodes/term.rb

@@ -0,0 +1,58 @@
+module Elastic::Nodes
+  class Term < Base
+    include Concerns::Boostable
+
+    BOOLEAN_MODE = [:any, :all]
+
+    attr_accessor :field, :mode
+
+    def terms=(_terms)
+      @terms = _terms.dup.to_a
+    end
+
+    def mode=(_value)
+      if !_value.nil? && !BOOLEAN_MODE.include?(_value)
+        raise ArgumentError, "invalid mode #{_value}"
+      end
+
+      @mode = _value
+    end
+
+    def terms
+      @terms.each
+    end
+
+    def clone
+      prepare_clone(super)
+    end
+
+    def simplify
+      prepare_clone(super)
+    end
+
+    def render
+      raise ArgumentError, 'must provide at least one term' if !@terms || @terms.empty?
+
+      if @terms.length == 1
+        { 'term' => { @field.to_s => render_boost('value' => @terms.first) } }
+      elsif @mode == :all
+        {
+          'bool' => render_boost(
+            'must' => @terms.map { |t| { 'term' => { @field.to_s => t } } }
+          )
+        }
+      else
+        { 'terms' => render_boost(@field.to_s => @terms) }
+      end
+    end
+
+    private
+
+    def prepare_clone(_clone)
+      _clone.field = @field
+      _clone.terms = @terms
+      _clone.mode = @mode
+      _clone
+    end
+  end
+end

data/lib/elastic/query.rb

@@ -1,39 +1,101 @@
 module Elastic
   class Query
-    include Capabilities::ContextHandler
-    include Capabilities::BoolQueryBuilder
-    include Capabilities::AggregationBuilder
+    extend Forwardable
+    include Enumerable
+    include Dsl::BoolQueryBuilder
+    include Dsl::MetricBuilder
 
-    attr_accessor :type, :size, :input_transform
+    def_delegators :result, :[], :each, :each_with_score, :count, :first, :last
 
-    def initialize(_type, minimum_should_match: 1, size: nil)
-      @type = _type
-      @size = size
-      self.minimum_should_match = minimum_should_match
+    attr_reader :index
+
+    def initialize(_index, _query_config = nil)
+      @index = _index
+      @config = _query_config || build_base_config
+    end
+
+    def coord_similarity(_enable)
+      with_clone { |config| config.root.query.disable_coord = !_enable }
+    end
+
+    def limit(_size)
+      with_clone { |config| config.limit = _size }
+    end
+    alias :size :limit
+
+    def offset(_offset)
+      with_clone { |config| config.offset = _offset }
+    end
+
+    def segment(*_params)
+      with_clone do |config|
+        config.groups << Commands::BuildAggFromParams.for(index: index, params: _params)
+      end
+    end
+
+    def ids(_type = nil)
+      execute assembler.assemble_ids
+    end
+
+    def pluck(_field)
+      execute assembler.assemble_pluck(_field.to_s)
+    end
+
+    def aggregate(_name = nil, _node = nil, &_block)
+      # TODO
+    end
+
+    def total
+      execute assembler.assemble_total
     end
 
-    def render
-      search = {}
-      search['size'] = @size unless @size.nil?
-      render_query_to search
-      render_aggregations_to search
-      search
+    def result(_reset = false)
+      @result = nil if _reset
+      @result ||= execute(assembler.assemble)
     end
 
-    def run
-      type.query(render)
+    def as_es_query
+      assembler.assemble.render
     end
 
     private
 
-    def render_query_to(_search)
-      query = {}
-      render_bool_query_to query
-      _search['query'] = query if query.length > 0
+    def with_clone(&_block)
+      new_config = @config.clone
+      _block.call(new_config)
+      self.class.new(@index, new_config)
+    end
+
+    def with_bool_query(&_block)
+      with_clone { |config| _block.call(config.root.query) }
+    end
+
+    def with_aggregable_for_metric(&_block)
+      adaptor = AggregableAdaptor.new.tap(&_block)
+      execute assembler.assemble_metric(adaptor.agg)
     end
 
-    def transform_input(_name, _value)
-      type.prepare_field_for_query _name, _value
+    def build_base_config
+      Core::QueryConfig.initial_config
+    end
+
+    def assembler
+      @assembler ||= Core::QueryAssembler.new(@index, @config)
+    end
+
+    def execute(_query)
+      _query.handle_result @index.adaptor.query(
+        type: @index.definition.types,
+        query: _query.render
+      )
+    end
+
+    class AggregableAdaptor
+      attr_accessor :agg
+
+      def aggregate(_node)
+        self.agg = _node
+      end
     end
   end
 end

data/lib/elastic/railtie.rb

@@ -0,0 +1,41 @@
+require "elastic/railties/utils"
+require "elastic/railties/ar_helpers"
+require "elastic/railties/ar_middleware"
+require "elastic/railties/type_extensions"
+require "elastic/railties/query_extensions"
+require "elastic/railties/indexable_record"
+
+module Elastic
+  class Railtie < Rails::Railtie
+    initializer "elastic.configure_rails_initialization" do
+      Elastic.configure Rails.application.config_for(:elastic).merge(logger: Rails.logger)
+
+      # Make every activerecord model indexable
+      ActiveRecord::Base.send(:include, Elastic::Railties::IndexableRecord)
+    end
+
+    rake_tasks do
+      load File.expand_path('../railties/tasks/es.rake', __FILE__)
+    end
+
+    # TODO: configure generators here too
+  end
+end
+
+# Expose railties utils at Elastic namespace
+module Elastic
+  extend Elastic::Railties::Utils
+end
+
+# Add activerecord related index helpers
+class Elastic::Type
+  include Elastic::Railties::TypeExtensions
+end
+
+# Add activerecord related query helpers
+class Elastic::Query
+  include Elastic::Railties::QueryExtensions
+end
+
+# Register active record middleware
+Elastic.register_middleware Elastic::Railties::ARMiddleware

data/lib/elastic/railties/ar_helpers.rb

@@ -0,0 +1,51 @@
+module Elastic::Railties
+  module ARHelpers
+    extend self
+
+    def find_each_with_options(_collection, includes: nil, scope: nil, &_block)
+      if _collection.respond_to? :find_each
+        _collection = _collection.includes(*includes) if includes
+        _collection = _collection.send(scope) if scope
+        _collection.find_each(&_block)
+      elsif _collection.respond_to? :each
+        ActiveRecord::Associations::Preloader.new.preload(_collection, *includes) if includes
+        _collection.each(&_block)
+      else
+        raise 'Elastic ActiveRecord importing is only supported for collection types'
+      end
+    end
+
+    def infer_ar4_field_options(_klass, _field)
+      # TODO: consider methods occluded by an override:
+      # AR defines methods, this wont work: return nil if _klass.method_defined? _field
+
+      return nil unless _klass.serialized_attributes[_field].nil? # occluded by serializer
+      return nil if _klass.columns_hash[_field].nil?
+      ar_type_to_options _klass.columns_hash[_field].type
+    end
+
+    def infer_ar5_field_options(_klass, _field)
+      # TODO: consider methods occluded by an override:
+      # AR defines methods, this wont work: return nil if _klass.method_defined? _field
+
+      meta = _klass.type_for_attribute _field
+      return nil if meta.to_s == 'ActiveRecord::Type::Serialized' # occluded by serializer
+      ar_type_to_options meta.type
+    end
+
+    private
+
+    # rubocop:disable Metrics/CyclomaticComplexity
+    def ar_type_to_options(_type)
+      case _type.try(:to_sym)
+      when :text              then { type: :string }
+      when :string            then { type: :string, index: 'not_analyzed' }
+      when :integer           then { type: :long } # not sure..
+      when :float, :decimal   then { type: :double } # not sure..
+      when :datetime, :date   then { type: :date }
+      when :boolean           then { type: :boolean }
+      end
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity
+  end
+end

data/lib/elastic/railties/ar_middleware.rb

@@ -0,0 +1,45 @@
+module Elastic::Railties
+  class ARMiddleware < Elastic::Core::BaseMiddleware
+    def self.accepts?(_target)
+      _target < ::ActiveRecord::Base
+    end
+
+    def mode
+      :index # storage mode not supported for AR
+    end
+
+    def field_options_for(_field, _options)
+      if Rails.version.to_f >= 4.2
+        ARHelpers.infer_ar5_field_options(target, _field)
+      else
+        ARHelpers.infer_ar4_field_options(target, _field)
+      end
+    end
+
+    def collect_all(_options, &_block)
+      collect_from(target, _options, &_block)
+    end
+
+    def collect_from(_collection, _options, &_block)
+      ARHelpers.find_each_with_options(
+        _collection,
+        includes: _options[:ar_collect_includes],
+        scope: _options[:ar_collect_scope],
+        &_block
+      )
+    end
+
+    def find_by_ids(_ids, _options)
+      results = target.where(id: _ids).order('id ASC')
+      results = results.includes(_options[:ar_includes]) if _options.key? :ar_includes
+      order_results _ids, results
+    end
+
+    private
+
+    def order_results(_ordered_ids, _results)
+      hash = _results.each_with_object({}) { |o, h| h[o.id] = o }
+      _ordered_ids.map { |id| hash[id.to_i] }
+    end
+  end
+end

data/lib/elastic/{indexable_record.rb → railties/indexable_record.rb}

@@ -1,15 +1,25 @@
-module Elastic
+module Elastic::Railties
   module IndexableRecord
     def self.included(_base)
-      _base.include Indexable
       _base.extend ClassMethods
     end
 
     module ClassMethods
+      def index_class
+        @index_class ||= to_s + 'Index'
+      end
+
+      def index_class=(_class)
+        @constantized_index_class = nil
+        @index_class = _class
+      end
+
+      def constantized_index_class
+        @constantized_index_class ||= index_class.constantize
+      end
+
       def index(_options)
-        index_depends(_options.delete(:depends))
         on = _options.delete(:on)
-
         if on == :create
           index_on_create _options
         elsif on == :save
@@ -24,22 +34,15 @@ module Elastic
       def index_on_save(_options = {})
         after_save(_options) { index_later }
       end
+    end
 
-      def index_depends(_depends)
-        @index_depends = _depends
-      end
-
-      def reindex
-        index_class.clear
-
-        scope = self
-        scope = self.includes(@index_depends) if @index_depends
-        scope.find_each { |r| index_class.store(r) } # TODO: index_many
-      end
+    def index_now
+      self.class.constantized_index_class.index self
+    end
 
-      def search
-        # TODO: index_class.query.decorate blabla
-      end
+    def index_later
+      # TODO: ActiveJob support
+      index_now
     end
   end
 end

data/lib/elastic/railties/query_extensions.rb

@@ -0,0 +1,9 @@
+module Elastic::Railties
+  module QueryExtensions
+    def includes(*_includes)
+      with_clone do |config|
+        config.middleware_options[:ar_includes] = _includes
+      end
+    end
+  end
+end

data/lib/elastic/railties/rspec.rb

@@ -0,0 +1,6 @@
+RSpec.configure do |config|
+  config.before(:example, elasticsearch: true) do
+    Elastic.drop
+    Elastic.migrate
+  end
+end

data/lib/elastic/railties/tasks/es.rake

@@ -0,0 +1,19 @@
+namespace :es do
+  desc "Elastic: Updates indices mappings"
+  task migrate: :environment do
+    Elastic.configure logger: Logger.new(STDOUT)
+    Elastic.migrate
+  end
+
+  desc "Elastic: Rebuilds indices from source data"
+  task reindex: :environment do
+    Elastic.configure logger: Logger.new(STDOUT)
+    Elastic.reindex
+  end
+
+  desc "Elastic: Lists indices stats"
+  task stats: :environment do
+    Elastic.configure logger: Logger.new(STDOUT)
+    Elastic.stats
+  end
+end

data/lib/elastic/railties/type_extensions.rb

@@ -0,0 +1,14 @@
+module Elastic::Railties
+  module TypeExtensions
+    def self.included(_klass)
+      _klass.extend ClassMethods
+    end
+
+    module ClassMethods
+      def references(*_includes)
+        # TODO: check target allows options
+        definition.middleware_options[:ar_collect_includes] = _includes
+      end
+    end
+  end
+end

data/lib/elastic/railties/utils.rb

@@ -0,0 +1,62 @@
+module Elastic::Railties
+  module Utils
+    def reindex(_index = nil)
+      logger.info "Reindexing all indices" if _index.nil?
+      indices(_index).each do |index|
+        logger.info "Reindexing index #{index.suffix}"
+        handle_errors { index.reindex }
+      end
+    end
+
+    def migrate(_index = nil)
+      logger.info "Migrating all indices" if _index.nil?
+      indices(_index).each do |index|
+        logger.info "Migrating index #{index.suffix}"
+        handle_errors { index.mapping.migrate }
+      end
+    end
+
+    def drop(_index = nil)
+      logger.info "Dropping all indices" if _index.nil?
+      indices(_index).each &:drop
+    end
+
+    def stats(_index = nil)
+      logger.info "Indices stats" if _index.nil?
+      indices(_index).each do |index|
+        logger.info "Stats for #{index.suffix}:"
+        # TODO.
+      end
+    end
+
+    private
+
+    def indices(_index = nil)
+      Dir.glob(indices_paths.join('**/*.rb')).map do |path|
+        path = Pathname.new path
+        path = path.relative_path_from indices_paths
+        path = path.dirname.join(path.basename(path.extname)).to_s
+        next nil if _index && (path != _index && path.camelize != _index)
+
+        klass = path.camelize.constantize
+        next nil unless klass < Elastic::Type
+        klass
+      end.reject(&:nil?)
+    end
+
+    def indices_paths
+      Rails.root.join(Elastic::Configuration.indices_path)
+    end
+
+    def handle_errors
+      yield
+    rescue => exc
+      logger.error exc.message
+      logger.error exc.backtrace.join("\n")
+    end
+
+    def logger
+      Elastic::Configuration.logger
+    end
+  end
+end

data/lib/elastic/results/aggregations.rb

@@ -0,0 +1,29 @@
+module Elastic::Results
+  class Aggregations < Base
+    include Enumerable
+
+    attr_reader :key
+
+    def initialize(_aggs)
+      @aggs = _aggs
+    end
+
+    def [](_key)
+      @aggs[_key.to_s].try(:as_value)
+    end
+
+    def each(&_block)
+      @aggs.each(&_block)
+    end
+
+    def as_value
+      # TODO: return aggregation value if configured as single bucket
+      self
+    end
+
+    def traverse(&_block)
+      super
+      @aggs.each_value { |a| a.traverse(&_block) }
+    end
+  end
+end

data/lib/elastic/results/base.rb

@@ -0,0 +1,13 @@
+module Elastic::Results
+  class Base
+    include Elastic::Support::Traversable
+
+    def as_value
+      self
+    end
+
+    def traverse(&_block)
+      _block.call(self)
+    end
+  end
+end

data/lib/elastic/results/bucket.rb

@@ -0,0 +1,15 @@
+module Elastic::Results
+  class Bucket < Aggregations
+    attr_reader :key
+
+    def initialize(_key, _aggs)
+      @key = _key
+      super _aggs
+    end
+
+    def as_value
+      # TODO: return aggregation value if configured as single bucket
+      self
+    end
+  end
+end