elastic-rails 0.1.0 → 0.5.0

data/lib/elastic/core/definition.rb

@@ -0,0 +1,118 @@
+module Elastic::Core
+  class Definition
+    attr_reader :middleware_options
+
+    def main_target
+      targets.first
+    end
+
+    def targets
+      @target_cache ||= load_targets.freeze
+    end
+
+    def targets=(_values)
+      @target_cache = nil
+      @targets = _values
+    end
+
+    def types
+      targets.map(&:type_name)
+    end
+
+    def mode
+      main_target.mode
+    end
+
+    def initialize
+      @targets = []
+      @field_map = {}
+      @frozen = false
+      @middleware_options = HashWithIndifferentAccess.new
+    end
+
+    def register_field(_field)
+      raise 'definition has been frozen' if @frozen
+      @field_map[_field.name] = _field
+    end
+
+    def fields
+      @field_map.each_value
+    end
+
+    def expanded_field_names
+      @expanded_field_names ||= @field_map.map { |_, field| field.expanded_names }.flatten
+    end
+
+    def get_field(_name)
+      _name = _name.to_s
+      separator = _name.index '.'
+      if separator.nil?
+        @field_map[_name]
+      else
+        parent = @field_map[_name[0...separator]]
+        parent.try(:get_field, _name[separator + 1..-1])
+      end
+    end
+
+    def has_field?(_name)
+      !get_field(_name).nil?
+    end
+
+    def as_es_mapping
+      # TODO: Make this a command
+      properties = {}
+      @field_map.each_value do |field|
+        field_def = field.mapping_options
+
+        if !field_def.key?(:type) && field.mapping_inference_enabled?
+          inferred = infer_mapping_options(field.name)
+          field_def.merge! inferred.symbolize_keys unless inferred.nil?
+        end
+
+        if Elastic::Configuration.strict_mode && !field_def.key?(:type)
+          raise "explicit field type for #{field.name} required"
+        end
+
+        properties[field.name] = field_def if field_def.key? :type
+      end
+
+      { 'properties' => properties.as_json }
+    end
+
+    def freeze
+      unless @frozen
+        @field_map.each_value(&:freeze)
+        @frozen = true
+        @middleware_options.freeze
+      end
+    end
+
+    def frozen?
+      !!@frozen
+    end
+
+    private
+
+    def load_targets
+      mode = nil
+      @targets.map do |target|
+        target = target.to_s.camelize.constantize if target.is_a?(Symbol) || target.is_a?(String)
+
+        target = load_target_middleware(target) unless target.class < BaseMiddleware
+        raise 'index target is not indexable' if target.nil?
+        raise 'mistmatching indexable mode' if mode && mode != target.mode
+        mode = target.mode
+
+        target
+      end
+    end
+
+    def load_target_middleware(_target)
+      Middleware.wrap(_target)
+    end
+
+    def infer_mapping_options(_name)
+      main_target.field_options_for(_name, middleware_options)
+    end
+  end
+end

data/lib/elastic/core/mapping_manager.rb

@@ -0,0 +1,120 @@
+module Elastic::Core
+  class MappingManager
+    attr_reader :adaptor, :definition
+
+    def initialize(_adaptor, _definition)
+      @adaptor = _adaptor
+      @definition = _definition
+      @status = :pending
+    end
+
+    def out_of_sync?
+      @status == :out_of_sync
+    end
+
+    def incomplete?
+      @status == :incomplete
+    end
+
+    def fetch
+      begin
+        mappings = @adaptor.get_mappings
+        mappings = @definition.types.map { |t| mappings[t] }.reject(&:nil?)
+        @index = merge_mappings_into_index mappings
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        # ignore not-found errors when fetching mappings
+        @index = nil
+      end
+
+      @status = compute_status
+      self
+    end
+
+    def unmapped_fields
+      @definition.expanded_field_names.reject { |f| has_field? f }
+    end
+
+    def has_field?(_name)
+      @index.key? _name
+    end
+
+    def get_field_options(_name)
+      @index[_name]
+    end
+
+    def migrate
+      # TODO: make this a command
+      @adaptor.create unless @adaptor.exists?
+      begin
+        @definition.types.each { |t| @adaptor.set_mapping(t, user_mapping) }
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        # TODO: https://www.elastic.co/guide/en/elasticsearch/guide/current/reindex.html
+      end
+      fetch
+    end
+
+    private
+
+    def compute_status
+      if !synchronized?
+        :out_of_sync
+      elsif unmapped_fields.count > 0
+        :incomplete
+      else
+        :ready
+      end
+    end
+
+    def synchronized?
+      return false if @index.nil?
+      flatten(user_mapping).all? do |field, properties|
+        compare_field_properties(@index[field], properties)
+      end
+    end
+
+    def compare_field_properties(_current, _user)
+      return false if _current.nil?
+
+      case _current['type']
+      when 'date'
+        _current = { 'format' => 'dateOptionalTime' }.merge(_user)
+      else
+        _current == _user
+      end
+    end
+
+    def user_mapping
+      @user_mapping ||= definition.as_es_mapping
+    end
+
+    def flatten(_raw, _prefix = '')
+      _raw['properties'].flat_map do |name, raw_field|
+        if raw_field['type'] == 'nested'
+          childs = flatten(raw_field, name + '.')
+          childs << [
+            _prefix + name,
+            raw_field.slice(*(raw_field.keys - ['properties']))
+          ]
+        else
+          [[_prefix + name, raw_field.dup]]
+        end
+      end
+    end
+
+    def merge_mappings_into_index(_mappings)
+      {}.tap do |result|
+        _mappings.each do |mapping|
+          index = flatten(mapping)
+          index.each do |field, properties|
+            if result.key? field
+              result[field].merge! properties
+            else
+              result[field] = properties
+            end
+          end
+        end
+        result.each_value(&:freeze)
+      end
+    end
+  end
+end

data/lib/elastic/core/middleware.rb

@@ -0,0 +1,26 @@
+module Elastic::Core
+  module Middleware
+    extend self
+
+    def register(_middleware_class)
+      middlewares << _middleware_class
+    end
+
+    def wrap(_target)
+      middleware_for(_target).new _target
+    end
+
+    def middleware_for(_target)
+      # TODO: improve matching logic
+      middleware = middlewares.reverse_each.find { |m| m.accepts?(_target) }
+      middleware = DefaultMiddleware if middleware.nil?
+      middleware
+    end
+
+    private
+
+    def middlewares
+      @middlewares ||= []
+    end
+  end
+end

data/lib/elastic/core/query_assembler.rb

@@ -0,0 +1,84 @@
+module Elastic::Core
+  class QueryAssembler
+    def initialize(_index, _config)
+      @index = _index
+      @config = _config
+    end
+
+    def assemble
+      query = build_base_query
+
+      if !grouped?
+        query.size = (@config.limit || Elastic::Configuration.page_size)
+        query.offset = @config.offset
+      else
+        query.size = 0
+        last = attach_groups query
+        last.aggregate(Elastic::Nodes::TopHits.build('default'))
+
+        query = grouped_query query
+        query = reduced_query query
+      end
+
+      populated_query query
+    end
+
+    def assemble_ids
+      raise NotImplementedError, 'ids retrieval not yet implemented'
+    end
+
+    def assemble_total
+      raise NotImplementedError, 'total not yet implemented'
+    end
+
+    def assemble_pluck(_field)
+      raise NotImplementedError, 'pluck not yet implemented'
+    end
+
+    def assemble_metric(_node)
+      query = assemble_metrics([_node])
+      reduced_query query
+    end
+
+    def assemble_metrics(_aggs)
+      query = build_base_query
+      query.size = 0
+
+      last = attach_groups(query)
+      last.aggs = _aggs
+
+      query = grouped_query(query) if grouped?
+      query
+    end
+
+    private
+
+    def build_base_query
+      @config.root.simplify
+    end
+
+    def grouped?
+      !@config.groups.empty?
+    end
+
+    def attach_groups(_query)
+      @config.groups.inject(_query) do |last, node|
+        node = node.simplify
+        last.aggregate node
+        node
+      end
+    end
+
+    def grouped_query(_query)
+      Elastic::Shims::Grouping.new(_query)
+    end
+
+    def reduced_query(_query)
+      Elastic::Shims::Reducing.new(_query)
+    end
+
+    def populated_query(_query)
+      Elastic::Shims::Populating.new(@index, @config, _query)
+    end
+  end
+end

data/lib/elastic/core/query_config.rb

@@ -0,0 +1,25 @@
+module Elastic::Core
+  class QueryConfig
+    attr_accessor :root, :groups, :limit, :offset, :middleware_options
+
+    def self.initial_config
+      new.tap do |config|
+        config.root = Elastic::Nodes::Search.new
+        config.root.query = Elastic::Nodes::Boolean.new
+        config.root.query.disable_coord = true unless Elastic::Configuration.coord_similarity
+        config.groups = []
+        config.middleware_options = HashWithIndifferentAccess.new
+      end
+    end
+
+    def clone
+      self.class.new.tap do |clone|
+        clone.root = @root.clone
+        clone.groups = @groups.dup
+        clone.limit = @limit
+        clone.offset = @offset
+        clone.middleware_options = @middleware_options.dup
+      end
+    end
+  end
+end

data/lib/elastic/core/serializer.rb

@@ -0,0 +1,45 @@
+module Elastic::Core
+  class Serializer
+    attr_reader :object
+
+    def self.original_value_occluded?(_field)
+      public_method_defined? _field
+    end
+
+    def initialize(_definition, _object)
+      # TODO: validate that object is of type <target>?
+      @definition = _definition
+      @object = _object
+    end
+
+    def fields
+      @definition.fields
+    end
+
+    def as_es_document(only_data: false)
+      data = {}.tap do |hash|
+        fields.each do |field|
+          value = read_attribute_for_indexing(field.name)
+          value = field.prepare_value_for_index(value)
+          hash[field.name] = value
+        end
+      end
+
+      return data if only_data
+
+      result = { '_type' => object.class.to_s, 'data' => data }
+      result['_id'] = read_attribute_for_indexing(:id) if has_attribute_for_indexing?(:id)
+      result
+    end
+
+    private
+
+    def has_attribute_for_indexing?(_name)
+      respond_to?(_name) || @object.respond_to?(_name)
+    end
+
+    def read_attribute_for_indexing(_name)
+      respond_to?(_name) ? public_send(_name) : @object.public_send(_name)
+    end
+  end
+end

data/lib/elastic/core/source_formatter.rb

@@ -0,0 +1,40 @@
+module Elastic::Core
+  class SourceFormatter
+    def initialize(_mapping)
+      @mapping = _mapping
+      @treatment_cache = {}
+    end
+
+    def format(_source, _prefix = nil)
+      _source.each do |field, value|
+        field_name = _prefix ? "#{_prefix}.#{field}" : field
+
+        treatment = treatment_for field_name
+        if treatment == :nested
+          value.each { |v| format(v, field_name) }
+        else
+          _source[field] = send(treatment, value) unless treatment == :none
+        end
+      end
+    end
+
+    private
+
+    def treatment_for(_field)
+      treatment = @treatment_cache[_field]
+      treatment = @treatment_cache[_field] = get_treatment(_field) if treatment.nil?
+      treatment
+    end
+
+    def get_treatment(_field)
+      field_options = @mapping.get_field_options(_field) || {}
+      return :nested if field_options['type'] == 'nested'
+      return :parse_date if field_options['type'] == 'date'
+      :none
+    end
+
+    def parse_date(_value)
+      Time.parse _value
+    end
+  end
+end

data/lib/elastic/dsl/bool_query_builder.rb

@@ -0,0 +1,52 @@
+module Elastic::Dsl
+  module BoolQueryBuilder
+    def must(*_queries)
+      with_bool_query do |query|
+        query.must build_query_from_params(_queries)
+      end
+    end
+
+    def should(*_queries)
+      with_bool_query do |query|
+        query.should build_query_from_params(_queries)
+      end
+    end
+
+    def with(_modifier, &_block)
+      raise ArgumentError, 'block missing' if _block.nil?
+      raise ArgumentError, 'node is not a modifier' unless _modifier.respond_to? :clone_with_query
+
+      with_bool_query do |query|
+        ctx = BoolQueryContext.new index, query, _modifier
+        ctx.instance_exec(&_block)
+        ctx
+      end
+    end
+
+    def boost(_amount = nil, field: nil, fixed: false, factor: 1, modifier: :none, missing: 1,
+      &_block)
+      raise ArgumentError, 'must provide at least a boost amount' if _amount.nil? && field.nil?
+
+      node = Elastic::Nodes::FunctionScore.new
+      node.boost_mode = :replace if fixed
+
+      if !field.nil?
+        node.add_field_function(field, factor: factor, modifier: modifier, missing: missing)
+      elsif fixed
+        node.add_weight_function(_amount)
+      else
+        node.boost = _amount
+      end
+
+      # TODO: add decay function support.
+
+      with(node, &_block)
+    end
+
+    private
+
+    def build_query_from_params(_params)
+      Elastic::Commands::BuildQueryFromParams.for(index: index, params: _params)
+    end
+  end
+end

data/lib/elastic/dsl/bool_query_context.rb

@@ -0,0 +1,42 @@
+module Elastic::Dsl
+  class BoolQueryContext
+    include BoolQueryBuilder
+
+    attr_reader :index
+
+    def initialize(_index, _query, _modifier)
+      @index = _index
+      @wrapper = BoolQueryWrapper.new(_query, _modifier)
+    end
+
+    private
+
+    def with_bool_query
+      yield @wrapper
+      self
+    end
+
+    class BoolQueryWrapper
+      def initialize(_query, _modifier)
+        @query = _query
+        @modifier = _modifier
+      end
+
+      def must(_node)
+        @query.must wrap(_node)
+        self
+      end
+
+      def should(_node)
+        @query.should wrap(_node)
+        self
+      end
+
+      private
+
+      def wrap(_query)
+        @modifier.clone_with_query _query
+      end
+    end
+  end
+end

data/lib/elastic/dsl/metric_builder.rb

@@ -0,0 +1,34 @@
+module Elastic::Dsl
+  module MetricBuilder
+    def average(_field, _options = {})
+      aggregate_metric(Elastic::Nodes::Agg::Average, _field, _options, 'avg_%s')
+    end
+
+    def sum(_field, _options = {})
+      aggregate_metric(Elastic::Nodes::Agg::Sum, _field, _options, 'sum_%s')
+    end
+
+    def minimum(_field, _options = {})
+      aggregate_metric(Elastic::Nodes::Agg::Minimum, _field, _options, 'min_%s')
+    end
+
+    def maximum(_field, _options = {})
+      aggregate_metric(Elastic::Nodes::Agg::Maximum, _field, _options, 'max_%s')
+    end
+
+    def stats(_field, _options = {})
+      aggregate_metric(Elastic::Nodes::Agg::Stats, _field, _options, '%s_stats')
+    end
+
+    private
+
+    def aggregate_metric(_klass, _field, _options, _default_name)
+      with_aggregable_for_metric do |agg|
+        # TODO: detect nested name and wrap node
+        name = _options[:as] || sprintf(_default_name, _field)
+        node = _klass.build(name, _field, missing: _options[:missing])
+        agg.aggregate node
+      end
+    end
+  end
+end

data/lib/elastic/fields/nested.rb

@@ -0,0 +1,42 @@
+module Elastic::Fields
+  class Nested
+    attr_reader :name, :index
+
+    def initialize(_name, _index)
+      @name = _name.to_s
+      @index = _index
+    end
+
+    def expanded_names
+      [@name] + @index.definition.expanded_field_names.map { |n| @name + '.' + n }
+    end
+
+    def mapping_inference_enabled?
+      false
+    end
+
+    def disable_mapping_inference
+    end
+
+    def freeze
+      @index.freeze_index_definition
+      super
+    end
+
+    def mapping_options
+      @index.definition.as_es_mapping.merge!(type: :nested)
+    end
+
+    def get_field(_name)
+      @index.definition.get_field _name
+    end
+
+    def prepare_value_for_query(_value)
+      _value
+    end
+
+    def prepare_value_for_index(_values)
+      _values.map { |v| @index.new(v).as_es_document(only_data: true) }
+    end
+  end
+end

data/lib/elastic/fields/value.rb

@@ -0,0 +1,60 @@
+module Elastic::Fields
+  class Value
+    MAPPING_OPTIONS = [
+      :type, :analyzer, :boost, :coerce, :copy_to, :doc_values, :dynamic,
+      :enabled, :fielddata, :geohash, :geohash_precision, :geohash_prefix, :format, :ignore_above,
+      :ignore_malformed, :include_in_all, :index_options, :lat_lon, :index, :fields, :norms,
+      :null_value, :position_increment_gap, :properties, :search_analyzer, :similarity, :store,
+      :term_vector
+    ]
+
+    attr_reader :name
+
+    def initialize(_name, _options)
+      @name = _name.to_s
+      @options = _options
+      @mapping_inference = true
+      @transform = Elastic::Support::Transform.new @options[:transform]
+    end
+
+    def expanded_names
+      [@name]
+    end
+
+    def mapping_inference_enabled?
+      @mapping_inference && !@options.key?(:transform)
+    end
+
+    def disable_mapping_inference
+      @mapping_inference = false
+    end
+
+    def mapping_options
+      process_special_types @options.symbolize_keys.slice(*MAPPING_OPTIONS)
+    end
+
+    def has_field?(_name)
+      false
+    end
+
+    def prepare_value_for_query(_value)
+      prepare_value_for_index(_value)
+    end
+
+    def prepare_value_for_index(_value)
+      @transform.apply _value
+    end
+
+    private
+
+    def process_special_types(_definition)
+      case _definition[:type].try(:to_sym)
+      when :term
+        _definition[:type] = 'string'
+        _definition[:index] = 'not_analyzed'
+      end
+
+      _definition
+    end
+  end
+end

data/lib/elastic/nested_type.rb

@@ -0,0 +1,5 @@
+module Elastic
+  class NestedType < Types::BaseType
+    extend Types::FacetedType
+  end
+end

data/lib/elastic/nodes/agg/average.rb

@@ -0,0 +1,7 @@
+module Elastic::Nodes::Agg
+  class Average < BaseMetric
+    private def metric
+      'avg'
+    end
+  end
+end