elastic-rails 0.5.0 → 0.6.0

data/lib/elastic/core/query_assembler.rb

@@ -6,33 +6,27 @@ module Elastic::Core
     end
 
     def assemble
-      query = build_base_query
+      populated_query build_hit_query
+    end
 
-      if !grouped?
-        query.size = (@config.limit || Elastic::Configuration.page_size)
-        query.offset = @config.offset
-      else
-        query.size = 0
-        last = attach_groups query
-        last.aggregate(Elastic::Nodes::TopHits.build('default'))
+    def assemble_total
+      query = build_base_query
+      query.size = 0
 
+      if grouped?
+        attach_groups(query)
         query = grouped_query query
-        query = reduced_query query
       end
 
-      populated_query query
+      pick_query_totals query
     end
 
     def assemble_ids
-      raise NotImplementedError, 'ids retrieval not yet implemented'
-    end
-
-    def assemble_total
-      raise NotImplementedError, 'total not yet implemented'
+      pick_query_ids build_hit_query
     end
 
-    def assemble_pluck(_field)
-      raise NotImplementedError, 'pluck not yet implemented'
+    def assemble_pick(_field)
+      pick_query_fields build_hit_query, _field
     end
 
     def assemble_metric(_node)
@@ -45,7 +39,7 @@ module Elastic::Core
       query.size = 0
 
       last = attach_groups(query)
-      last.aggs = _aggs
+      last.aggregations = _aggs
 
       query = grouped_query(query) if grouped?
       query
@@ -54,7 +48,33 @@ module Elastic::Core
     private
 
     def build_base_query
-      @config.root.simplify
+      Elastic::Nodes::Search.build @config.query.simplify
+    end
+
+    def build_hit_query
+      query = build_base_query
+
+      if !grouped?
+        query.size = (@config.limit || Elastic::Configuration.page_size)
+        query.offset = @config.offset
+        query = sort_node(query)
+      else
+        query.size = 0
+        last = attach_groups query
+        last.aggregate sort_node Elastic::Nodes::TopHits.build('default')
+
+        query = grouped_query query
+        query = reduced_query query
+      end
+
+      query
+    end
+
+    def sort_node(_node)
+      return _node unless @config.sort
+      sort_node = @config.sort.clone
+      sort_node.child = _node
+      sort_node
     end
 
     def grouped?
@@ -80,5 +100,17 @@ module Elastic::Core
     def populated_query(_query)
       Elastic::Shims::Populating.new(@index, @config, _query)
     end
+
+    def pick_query_ids(_query)
+      Elastic::Shims::IdPicking.new(_query)
+    end
+
+    def pick_query_fields(_query, _field)
+      Elastic::Shims::FieldPicking.new(_query, _field.to_s)
+    end
+
+    def pick_query_totals(_query)
+      Elastic::Shims::TotalPicking.new(_query)
+    end
   end
 end

data/lib/elastic/core/query_config.rb

@@ -1,12 +1,10 @@
 module Elastic::Core
   class QueryConfig
-    attr_accessor :root, :groups, :limit, :offset, :middleware_options
+    attr_accessor :query, :groups, :limit, :offset, :sort, :middleware_options
 
     def self.initial_config
       new.tap do |config|
-        config.root = Elastic::Nodes::Search.new
-        config.root.query = Elastic::Nodes::Boolean.new
-        config.root.query.disable_coord = true unless Elastic::Configuration.coord_similarity
+        config.query = Elastic::Nodes::Boolean.new
         config.groups = []
         config.middleware_options = HashWithIndifferentAccess.new
       end
@@ -14,10 +12,11 @@ module Elastic::Core
 
     def clone
       self.class.new.tap do |clone|
-        clone.root = @root.clone
+        clone.query = @query.clone
         clone.groups = @groups.dup
         clone.limit = @limit
         clone.offset = @offset
+        clone.sort = @sort.try(:clone)
         clone.middleware_options = @middleware_options.dup
       end
     end

data/lib/elastic/core/source_formatter.rb

@@ -1,40 +1,21 @@
 module Elastic::Core
   class SourceFormatter
-    def initialize(_mapping)
-      @mapping = _mapping
-      @treatment_cache = {}
+    def initialize(_definition)
+      @definition = _definition
     end
 
-    def format(_source, _prefix = nil)
-      _source.each do |field, value|
-        field_name = _prefix ? "#{_prefix}.#{field}" : field
-
-        treatment = treatment_for field_name
-        if treatment == :nested
-          value.each { |v| format(v, field_name) }
-        else
-          _source[field] = send(treatment, value) unless treatment == :none
-        end
-      end
-    end
-
-    private
-
-    def treatment_for(_field)
-      treatment = @treatment_cache[_field]
-      treatment = @treatment_cache[_field] = get_treatment(_field) if treatment.nil?
-      treatment
-    end
-
-    def get_treatment(_field)
-      field_options = @mapping.get_field_options(_field) || {}
-      return :nested if field_options['type'] == 'nested'
-      return :parse_date if field_options['type'] == 'date'
-      :none
+    def format_field(_field, _value)
+      field = @definition.get_field _field
+      return _value if field.nil?
+      field.prepare_value_for_result _value
     end
 
-    def parse_date(_value)
-      Time.parse _value
+    def format(_source)
+      _source.each do |key, value|
+        field = @definition.get_field key
+        next if field.nil?
+        _source[key] = field.prepare_value_for_result(value)
+      end
     end
   end
 end

data/lib/elastic/datatypes/date.rb

@@ -0,0 +1,32 @@
+module Elastic::Datatypes
+  class Date < Default
+    def prepare_for_index(_value)
+      if !_value.nil? && !_value.is_a?(::Date)
+        raise ArgumentError, "expected a date for field #{name}"
+      end
+
+      # date is stored as the corresponding utc timestamp in elastic search,
+      # no need to convert it here
+      _value
+    end
+
+    def prepare_value_for_result(_value)
+      case _value
+      when ::String
+        ::Time.parse(_value).utc.to_date
+      when ::Integer
+        ::Time.at(_value / 1000).utc.to_date
+      else
+        _value
+      end
+    end
+
+    def supported_aggregations
+      [:date_histogram] + super
+    end
+
+    def date_histogram_aggregation_defaults
+      { interval: '1w' }
+    end
+  end
+end

data/lib/elastic/datatypes/default.rb

@@ -0,0 +1,74 @@
+module Elastic::Datatypes
+  class Default
+    MAPPING_OPTIONS = [
+      :type, :analyzer, :boost, :coerce, :copy_to, :doc_values, :dynamic,
+      :enabled, :fielddata, :geohash, :geohash_precision, :geohash_prefix, :format, :ignore_above,
+      :ignore_malformed, :include_in_all, :index_options, :lat_lon, :index, :fields, :norms,
+      :null_value, :position_increment_gap, :properties, :search_analyzer, :similarity, :store,
+      :term_vector
+    ]
+
+    def initialize(_name, _options)
+      @name = _name
+      @user_options = _options
+    end
+
+    def mapping_options
+      @user_options.slice(*MAPPING_OPTIONS)
+    end
+
+    def prepare_for_query(_value)
+      prepare_for_index _value
+    end
+
+    def prepare_for_index(_value)
+      _value
+    end
+
+    def prepare_value_for_result(_value)
+      _value
+    end
+
+    def supported_aggregations
+      [:terms, :histogram, :range]
+    end
+
+    def supported_queries
+      [:term, :range]
+    end
+
+    # default configurations
+
+    def term_query_defaults
+      {}
+    end
+
+    def match_query_defaults
+      {}
+    end
+
+    def range_query_defaults
+      {}
+    end
+
+    def terms_aggregation_defaults
+      { size: 0 }
+    end
+
+    def date_histogram_aggregation_defaults
+      {}
+    end
+
+    def histogram_aggregation_defaults
+      {}
+    end
+
+    def range_aggregation_defaults
+      {}
+    end
+
+    private
+
+    attr_reader :name, :user_options
+  end
+end

data/lib/elastic/datatypes/string.rb

@@ -0,0 +1,7 @@
+module Elastic::Datatypes
+  class String < Default
+    def supported_queries
+      [:match, :term, :range]
+    end
+  end
+end

data/lib/elastic/datatypes/term.rb

@@ -0,0 +1,10 @@
+module Elastic::Datatypes
+  class Term < Default
+    def mapping_options
+      options = super
+      options[:type] = 'string'
+      options[:index] = 'not_analyzed'
+      options
+    end
+  end
+end

data/lib/elastic/datatypes/time.rb

@@ -0,0 +1,29 @@
+module Elastic::Datatypes
+  class Time < Default
+    def mapping_options
+      options = super
+      options[:type] = 'date'
+      options
+    end
+
+    def prepare_value_for_result(_value)
+      # TODO: set timezone
+      case _value
+      when ::String
+        ::Time.parse(_value)
+      when ::Integer
+        ::Time.at(_value / 1000)
+      else
+        _value
+      end
+    end
+
+    def supported_aggregations
+      [:date_histogram] + super
+    end
+
+    def date_histogram_aggregation_defaults
+      { interval: '1h' }
+    end
+  end
+end

data/lib/elastic/dsl/bool_query_builder.rb

@@ -1,5 +1,9 @@
 module Elastic::Dsl
   module BoolQueryBuilder
+    def coord_similarity(_enable)
+      with_bool_query { |query| query.disable_coord = !_enable }
+    end
+
     def must(*_queries)
       with_bool_query do |query|
         query.must build_query_from_params(_queries)

data/lib/elastic/fields/nested.rb

@@ -7,19 +7,32 @@ module Elastic::Fields
       @index = _index
     end
 
+    def merge!(_options)
+      # does nothing
+    end
+
     def expanded_names
       [@name] + @index.definition.expanded_field_names.map { |n| @name + '.' + n }
     end
 
-    def mapping_inference_enabled?
+    def validate
+      nil
+    end
+
+    def needs_inference?
       false
     end
 
+    def nested?
+      true
+    end
+
     def disable_mapping_inference
+      # does nothing, inference is always disabled
     end
 
     def freeze
-      @index.freeze_index_definition
+      @index.freeze_definition
       super
     end
 
@@ -31,12 +44,17 @@ module Elastic::Fields
       @index.definition.get_field _name
     end
 
-    def prepare_value_for_query(_value)
-      _value
-    end
-
     def prepare_value_for_index(_values)
       _values.map { |v| @index.new(v).as_es_document(only_data: true) }
     end
+
+    def prepare_value_for_result(_values)
+      formatter = Elastic::Core::SourceFormatter.new @index.definition
+      _values.each { |v| formatter.format(v) }
+    end
+
+    def select_aggregation(_from)
+      nil
+    end
   end
 end

data/lib/elastic/fields/value.rb

@@ -1,60 +1,104 @@
 module Elastic::Fields
   class Value
-    MAPPING_OPTIONS = [
-      :type, :analyzer, :boost, :coerce, :copy_to, :doc_values, :dynamic,
-      :enabled, :fielddata, :geohash, :geohash_precision, :geohash_prefix, :format, :ignore_above,
-      :ignore_malformed, :include_in_all, :index_options, :lat_lon, :index, :fields, :norms,
-      :null_value, :position_increment_gap, :properties, :search_analyzer, :similarity, :store,
-      :term_vector
-    ]
+    extend Forwardable
 
     attr_reader :name
 
+    def_delegators :@datatype, :mapping_options, :prepare_value_for_result, :supported_queries,
+      :supported_aggregations
+
+    def_delegators :@datatype, :term_query_defaults, :match_query_defaults, :range_query_defaults,
+      :terms_aggregation_defaults, :date_histogram_aggregation_defaults,
+      :histogram_aggregation_defaults, :range_aggregation_defaults
+
     def initialize(_name, _options)
       @name = _name.to_s
-      @options = _options
+      @options = _options.symbolize_keys
       @mapping_inference = true
-      @transform = Elastic::Support::Transform.new @options[:transform]
+    end
+
+    def nested?
+      false
+    end
+
+    def merge!(_options)
+      return if _options.nil?
+      @options.merge! _options.symbolize_keys
+    end
+
+    def get_field(_name)
+      nil
+    end
+
+    def validate
+      return "explicit field type for #{@name} required" unless @options.key? :type
+      nil
     end
 
     def expanded_names
       [@name]
     end
 
-    def mapping_inference_enabled?
-      @mapping_inference && !@options.key?(:transform)
+    def needs_inference?
+      mapping_inference_enabled? && !@options.key?(:type)
     end
 
     def disable_mapping_inference
       @mapping_inference = false
     end
 
-    def mapping_options
-      process_special_types @options.symbolize_keys.slice(*MAPPING_OPTIONS)
-    end
-
-    def has_field?(_name)
-      false
+    def freeze
+      @name.freeze
+      @options.freeze
+      load_transform_and_datatype
     end
 
     def prepare_value_for_query(_value)
-      prepare_value_for_index(_value)
+      _value = @transform.apply _value if @transform
+      @datatype.prepare_for_query _value
     end
 
     def prepare_value_for_index(_value)
-      @transform.apply _value
+      _value = @transform.apply _value if @transform
+      @datatype.prepare_for_index _value
     end
 
     private
 
-    def process_special_types(_definition)
-      case _definition[:type].try(:to_sym)
-      when :term
-        _definition[:type] = 'string'
-        _definition[:index] = 'not_analyzed'
+    def load_transform_and_datatype
+      @datatype = datatype_class.new(@name, @options)
+      @transform = Elastic::Support::Transform.new @options[:transform] if @options.key? :transform
+    end
+
+    def mapping_inference_enabled?
+      @mapping_inference && !@options.key?(:transform)
+    end
+
+    def datatype_class
+      case @options[:type]
+      when Symbol, String
+        load_registered_datatype @options[:type].to_sym
+      when nil
+        Elastic::Datatypes::Default
+      else
+        @options[:type]
       end
+    end
 
-      _definition
+    def load_registered_datatype(_name)
+      # TODO: replace this with a datatype registry
+      case _name
+      when :term
+        Elastic::Datatypes::Term
+      when :string
+        Elastic::Datatypes::String
+      when :date
+        Elastic::Datatypes::Date
+      when :time
+        Elastic::Datatypes::Time
+      else
+        Elastic::Datatypes::Default
+      end
     end
   end
 end

data/lib/elastic/nested_query.rb

@@ -0,0 +1,34 @@
+module Elastic
+  class NestedQuery
+    include Elastic::Dsl::BoolQueryBuilder
+
+    attr_reader :index
+
+    def initialize(_index, _root = nil)
+      @index = _index
+      @root = _root || build_root_node
+    end
+
+    def score_mode(_mode)
+      with_clone { |root| root.score_mode = _mode }
+    end
+
+    def as_node
+      @root.clone
+    end
+
+    private
+
+    def with_clone(&_block)
+      self.class.new @index, @root.clone.tap(&_block)
+    end
+
+    def with_bool_query(&_block)
+      with_clone { |root| root.child.tap(&_block) }
+    end
+
+    def build_root_node
+      Elastic::Nodes::Nested.build(nil, Elastic::Nodes::Boolean.new)
+    end
+  end
+end

data/lib/elastic/nested_type.rb

@@ -1,5 +1,15 @@
 module Elastic
   class NestedType < Types::BaseType
     extend Types::FacetedType
+
+    class << self
+      extend Forwardable
+
+      def_delegators :query, :must, :should, :segment, :coord_similarity
+    end
+
+    def self.query
+      NestedQuery.new self
+    end
   end
 end

data/lib/elastic/nodes/agg/average.rb

@@ -1,6 +1,8 @@
 module Elastic::Nodes::Agg
   class Average < BaseMetric
-    private def metric
+    private
+
+    def metric
       'avg'
     end
   end

data/lib/elastic/nodes/agg/base_metric.rb

@@ -17,14 +17,15 @@ module Elastic::Nodes::Agg
       prepare_clone super
     end
 
-    def render
-      options = { 'field' => @field.to_s }
-      options['missing'] = @missing if @missing
+    def render(_options = {})
+      hash = { 'field' => @field.to_s }
+      hash['missing'] = @missing if @missing
 
-      { metric => options }
+      { metric => hash }
     end
 
-    def handle_result(_raw)
+    def handle_result(_raw, _formatter)
+      # TODO: apply formatter to value
       Elastic::Results::Metric.new _raw['value']
     end
 

data/lib/elastic/nodes/agg/date_histogram.rb

@@ -26,11 +26,11 @@ module Elastic::Nodes::Agg
       prepare_clone(super)
     end
 
-    def render
-      options = { 'field' => @field.to_s }
-      options['interval'] = @interval if @interval
+    def render(_options = {})
+      hash = { 'field' => @field.to_s }
+      hash['interval'] = @interval if @interval
 
-      render_aggs 'date_histogram' => options
+      render_aggs({ 'date_histogram' => hash }, _options)
     end
 
     private

data/lib/elastic/nodes/agg/maximum.rb

@@ -1,6 +1,8 @@
 module Elastic::Nodes::Agg
   class Maximum < BaseMetric
-    private def metric
+    private
+
+    def metric
       'max'
     end
   end

data/lib/elastic/nodes/agg/minimum.rb

@@ -1,6 +1,8 @@
 module Elastic::Nodes::Agg
   class Minimum < BaseMetric
-    private def metric
+    private
+
+    def metric
       'min'
     end
   end

data/lib/elastic/nodes/agg/stats.rb

@@ -1,6 +1,8 @@
 module Elastic::Nodes::Agg
   class Stats < BaseMetric
-    private def metric
+    private
+
+    def metric
       'stats'
     end
   end

data/lib/elastic/nodes/agg/sum.rb

@@ -1,6 +1,8 @@
 module Elastic::Nodes::Agg
   class Sum < BaseMetric
-    private def metric
+    private
+
+    def metric
       'sum'
     end
   end