elasticated 1.2.1 → 2.0.0

data/lib/elasticated/aggregations/single_value_aggregation.rb

@@ -15,7 +15,13 @@ module Elasticated
     end
 
     def parse(response)
-      response['value'] || 0
+      response[result_key] || 0
+    end
+
+    protected
+
+    def result_key
+      'value'
     end
 
   end

data/lib/elasticated/aggregations/stats_aggregation.rb

@@ -0,0 +1,13 @@
+module Elasticated
+  class StatsAggregation < Aggregation
+
+    def build
+      { stats: { field: field } }
+    end
+
+    def parse(response)
+      response
+    end
+
+  end
+end

data/lib/elasticated/aggregations/subaggregated.rb

@@ -20,7 +20,16 @@ module Elasticated
     end
 
     def parse_subaggregations(response)
-      _subaggregations.parse response
+      if compact and _subaggregations.one?
+        single_subaggregation = _subaggregations.first
+        single_subaggregation.parse(response[single_subaggregation.name.to_s])
+      else
+        _subaggregations.parse response
+      end
+    end
+
+    def compact
+      false
     end
 
   end

data/lib/elasticated/aggregations/top_hits_aggregation.rb

@@ -16,9 +16,9 @@ module Elasticated
     def parse(response)
       # total = response['hits']['total']
       # max_score = response['hits']['max_score']
-      # hits = response['hits']['hits'].map{ |hit| Document.from_elasticsearch_hit hit }
+      # hits = response['hits']['hits'].map{ |hit| Document.parse hit }
       # HitsInfo.new total, max_score, hits
-      response['hits']['hits'].map{ |hit| Document.from_elasticsearch_hit hit }
+      response['hits']['hits'].map{ |hit| Document.parse hit }
     end
 
   end

data/lib/elasticated/aggregations/{count_distinct_aggregation.rb → value_count_aggregation.rb}

@@ -1,5 +1,5 @@
 module Elasticated
-  class CountDistinctAggregation < SingleValueAggregation
+  class ValueCountAggregation < SingleValueAggregation
 
     # override
     def default_name

data/lib/elasticated/configurable.rb

@@ -1,9 +1,20 @@
 module Elasticated
   module Configurable
 
-    def log
-      Configuration.logger ||= DefaultLogger.new
+    def self.delegated(attr_name)
+      define_method attr_name do
+        instance_variable_get("@#{attr_name}") || Configuration.send(attr_name)
+      end
+      define_method "#{attr_name}=" do |value|
+        instance_variable_set "@#{attr_name}", value
+      end
     end
 
+    delegated :logger
+    alias_method :log, :logger
+    delegated :scroll_expiration_time
+    delegated :scroll_page_size
+    delegated :search_page_size
+
   end
 end

data/lib/elasticated/configuration.rb

@@ -6,5 +6,11 @@ module Elasticated
       attr_accessor :scroll_expiration_time, :scroll_page_size, :search_page_size
     end
 
+    # defaults
+    self.logger = Loggers::SilentLogger.new
+    self.scroll_expiration_time = '3m'
+    self.scroll_page_size = 500
+    self.search_page_size = 1000
+
   end
 end

data/lib/elasticated/document.rb

@@ -14,7 +14,7 @@ module Elasticated
         ret
       end
 
-      def from_elasticsearch_hit(hit)
+      def parse(hit)
         document = new hit['_source']
         document.id = hit['_id']
         document.type = hit['_type']
@@ -23,6 +23,7 @@ module Elasticated
         document.version = hit['_version']
         document
       end
+      alias_method :from_elasticsearch_hit, :parse
 
     end
 

data/lib/elasticated/helpers.rb

@@ -24,5 +24,23 @@ module Elasticated
       duplicate
     end
 
+    def self.unscoped_class_name(klass)
+      klass.name.gsub /^.*\:\:/, ''
+    end
+
+    def self.hash_sum(*hashes)
+      hashes = hashes.compact
+      return 0 if hashes.empty?
+      hashes.map(&:keys).reduce(:|).each_with_object({}) do |key, result|
+        result[key] =
+          if hashes.any? { |h| h[key].is_a?(Hash) }
+            new_args = hashes.map { |h| h[key] }
+            hash_sum(*new_args)
+          else
+            hashes.map { |h| h.fetch(key, 0) }.reduce(:+)
+          end
+      end
+    end
+
   end
 end

data/lib/elasticated/loggers/default_logger.rb

@@ -0,0 +1,27 @@
+module Elasticated
+  module Loggers
+    class DefaultLogger
+
+      def trace(message)
+        puts "TRACE #{message}"
+      end
+
+      def debug(message)
+        puts "DEBUG #{message}"
+      end
+
+      def info(message)
+        puts "INFO #{message}"
+      end
+
+      def warn(message)
+        puts "WARN #{message}"
+      end
+
+      def error(message)
+        puts "ERROR #{message}"
+      end
+
+    end
+  end
+end

data/lib/elasticated/loggers/silent_logger.rb

@@ -0,0 +1,27 @@
+module Elasticated
+  module Loggers
+    class SilentLogger
+
+      def trace(message)
+        # nothing to do
+      end
+
+      def debug(message)
+        # nothing to do
+      end
+
+      def info(message)
+        # nothing to do
+      end
+
+      def warn(message)
+        # nothing to do
+      end
+
+      def error(message)
+        # nothing to do
+      end
+
+    end
+  end
+end

data/lib/elasticated/query.rb

@@ -76,6 +76,14 @@ module Elasticated
       !!_size
     end
 
+    def heavy_for?(repository)
+      !_size || _size > repository.search_page_size
+    end
+
+    def has_offset?
+      _from && _from > 0
+    end
+
     def sorted?
       !!_sort
     end

data/lib/elasticated/query_aggregations.rb

@@ -1,6 +1,9 @@
 module Elasticated
   class QueryAggregations
 
+    extend Forwardable
+    def_delegators :@_aggregations, :first, :one?, :empty?
+
     include Clonable
     include BlockEvaluation
 
@@ -30,10 +33,6 @@ module Elasticated
       end
     end
 
-    def empty?
-      _aggregations.empty?
-    end
-
     private
 
     def get_aggregation_class(agg_name)

data/lib/elasticated/repository.rb

@@ -1,5 +1,6 @@
 module Elasticated
   class Repository
+    include Configurable
 
     # child can implement 'execute(action, query, opts)'
     # child can implement 'prepare(action, document, opts)'
@@ -42,6 +43,14 @@ module Elasticated
       prepare :update, document, opts
     end
 
+    def prepare_search(query, opts={})
+      execute :prepare_search, query, opts
+    end
+
+    def restore_search(scroll_id)
+      ResumableSearch.from_scroll_id self, scroll_id
+    end
+
     protected
 
     # write actions
@@ -90,38 +99,30 @@ module Elasticated
     end
 
     def _exec_paginated_search(query, aggregated, opts={})
-      body = aggregated ? query.build_for_aggregated_search : query.build_for_search
-      if query.limited?
-        response = client.search body, opts
-        Results.from_elasticsearch_response response, query
-      elsif query.sorted? || query.aggregated?
-        # normal pagination
-        page_size = Configuration.scroll_page_size || 50
-        current_page = 1
-        loop do
-          offset = page_size * (current_page - 1)
-          response = client.search body, opts.merge(size: page_size, from: offset)
-          if current_page == 1
-            results = Results.from_elasticsearch_response response, query
-          else
-            results.append_results_from response
-          end
-          total_pages = (response['hits']['total'] / page_size.to_f).ceil
-          break if current_page >= total_pages
-          current_page += 1
-          body = query.build_for_search if aggregated
-        end
-        results
+      raise "The 'size' param is not allowed in opts, it must be setted in the query itself" if opts[:size]
+      raise "The 'from' param is not allowed in opts, it must be setted in the query itself" if opts[:from]
+      search_method = opts.delete :as
+      search = case search_method
+      when nil
+        IntelligentSearch.new self, query, aggregated, opts
+      when :single_page
+        SinglePageSearch.new self, query, aggregated, opts
+      when :normal
+        NormalSearch.new self, query, aggregated, opts
+      when :scroll
+        ScrollSearch.new self, query, aggregated, opts
+      when :scan_and_scroll
+        ScanScrollSearch.new self, query, aggregated, opts
       else
-        # scan & scroll
-        scroll_expiration_time = Configuration.scroll_expiration_time || '3m'
-        page_size = Configuration.scroll_page_size || 1000
-        response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_expiration_time, size: page_size)
-        while response = client.scroll(response['_scroll_id'], scroll: scroll_expiration_time) and not response['hits']['hits'].empty? do
-          results = results ? results.append_results_from(response) : Results.from_elasticsearch_response(response, query)
-        end
-        results ? results : Results.from_elasticsearch_response(response, query)
+        raise "Invalid search method: '#{search_method}'"
       end
+      search.execute
+    end
+
+    def _exec_prepare_search(query, opts={})
+      raise "The query for a resumable search should not have the parameter 'size'" if query.limited? || opts[:size]
+      raise "The query for a resumable search should not have the parameter 'from'" if query.has_offset? || opts[:from]
+      ResumableSearch.new self, query, false, opts
     end
 
     # abstract methods

data/lib/elasticated/repository/intelligent_search.rb

@@ -0,0 +1,46 @@
+module Elasticated
+  class Repository
+    class IntelligentSearch < Search
+      include Configurable
+
+      # INTELLIGENT SEARCH CASES
+      # ------------ without sorting
+      # without size, without offset  => scan_scroll
+      # with size,    without offset  => single_page or scan_scroll
+      # without size, with offset     => normal_pagination                  # TODO analizar si se puede usar scroll con offset
+      # with size,    with offset     => single_page or normal_pagination   # TODO analizar si se puede usar scroll con offset
+      # ------------ with sorting
+      # without size, without offset  => scroll
+      # with size,    without offset  => single_page or scroll
+      # without size, with offset     => normal_pagination                  # TODO analizar si se puede usar scroll con offset
+      # with size,    with offset     => single_page or normal_pagination   # TODO analizar si se puede usar scroll con offset
+
+      # if the query is aggregated and the search strategy is use an
+      # scroll-like alternative, we must do a 2-step search process
+
+      def best_search_method
+        if query.limited? && !query.heavy_for?(repository)
+          SinglePageSearch.new repository, query, aggregated, opts
+        else
+          if query.has_offset?
+            NormalSearch.new repository, query, aggregated, opts
+          else
+            if query.sorted?
+              ScrollSearch.new repository, query, aggregated, opts
+            else
+              ScanScrollSearch.new repository, query, aggregated, opts
+            end
+          end
+        end
+      end
+
+      def execute
+        search_method = best_search_method
+        search_method_name = Helpers.unscoped_class_name search_method.class
+        log.debug "#{search_method_name} selected as best search method for query #{query.build.to_json}"
+        search_method.execute
+      end
+
+    end
+  end
+end

data/lib/elasticated/repository/normal_search.rb

@@ -0,0 +1,40 @@
+module Elasticated
+  class Repository
+    class NormalSearch < Search
+
+      def execute
+        body = aggregated ? query.build_for_aggregated_search : query.build_for_search
+
+        initial_offset = offset = query._from # TODO do it nicer
+        size = repository.search_page_size
+
+        override! body, size, offset
+        response = client.search body, opts
+        results = Results.parse response, query
+
+        target_size = query.limited? ? query._size : (results.hits.total - offset)
+        total_pages = (target_size / size.to_f).ceil
+
+        current_page = 1
+        body = query.build_for_search
+
+        loop do
+          break if current_page >= total_pages
+          offset = initial_offset + size * current_page
+          size = target_size - offset + initial_offset if query.limited? && offset - initial_offset + size > target_size
+          override! body, size, offset
+
+          response = client.search body, opts
+          new_results = Results.parse response
+          results.append new_results
+          break if new_results.count < size
+
+          current_page += 1
+        end
+
+        results
+      end
+
+    end
+  end
+end

data/lib/elasticated/repository/resumable_search.rb

@@ -0,0 +1,58 @@
+module Elasticated
+  class Repository
+    class ResumableSearch < Search
+
+      def self.from_scroll_id(client, scroll_id)
+        search = new client, nil
+        search.scroll_id = scroll_id
+        search
+      end
+
+      attr_accessor :scroll_id, :completed
+
+      def start
+        # aggregations
+        aggregation_results = if aggregated
+          body = query.build_for_aggregations
+          response = client.search body, opts
+          Results.parse response, query
+        end
+        # search
+        body = query.build_for_search
+        response = client.search body, opts.merge(scroll: scroll_expiration_time, size: scroll_page_size)
+        results = Results.parse response
+        results.append aggregation_results if aggregation_results
+        self.scroll_id = results.scroll_id
+        mark_completed! if results.documents.count < scroll_page_size
+        results
+      end
+
+      def fetch
+        raise "No scroll_id present" unless scroll_id
+        raise "No more information to fetch: scroll completed" if completed?
+        response = client.scroll scroll_id, scroll: scroll_expiration_time
+        results = Results.parse response
+        self.scroll_id = results.scroll_id
+        mark_completed! if results.documents.empty?
+        results
+      end
+
+      def completed?
+        !!completed
+      end
+
+      def execute
+        results = start
+        results.append fetch while !completed?
+        results
+      end
+
+      protected
+
+      def mark_completed!
+        self.completed = true
+      end
+
+    end
+  end
+end