elasticated 1.2.1 → 2.0.0

data/lib/elasticated/repository/scan_scroll_search.rb

@@ -0,0 +1,43 @@
+module Elasticated
+  class Repository
+    class ScanScrollSearch < Search
+
+      def fetch_aggregations
+        body = query.build_for_aggregations
+        response = client.search body, opts
+        Results.parse response, query
+      end
+
+      def execute
+        # aggregations
+        results = nil
+        if aggregated
+          body = query.build_for_aggregations
+          response = client.search body, opts
+          results = Results.parse response, query
+        end
+        # search
+        body = query.build_for_search
+        response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_expiration_time, size: scroll_page_size)
+        results = Results.parse response
+        results.append fetch_aggregations if aggregated
+        doc_count = 0
+        loop do
+          response = client.scroll results.scroll_id, scroll: scroll_expiration_time
+          new_results = Results.parse response
+          hits = new_results.documents
+          break if hits.empty?
+          if query.limited? && (doc_count + hits.count > query._size)
+            new_results.documents = hits.first(query._size - doc_count)
+            hits = new_results.documents
+          end
+          doc_count += hits.count
+          results = results.append new_results
+          break if query.limited? && doc_count >= query._size
+        end
+        results
+      end
+
+    end
+  end
+end

data/lib/elasticated/repository/scroll_search.rb

@@ -0,0 +1,45 @@
+module Elasticated
+  class Repository
+    class ScrollSearch < Search
+
+      def fetch_aggregations
+        body = query.build_for_aggregations
+        response = client.search body, opts
+        Results.parse response, query
+      end
+
+      def execute
+        body = query.build_for_search
+        size = scroll_page_size
+
+        override! body
+        response = client.search body, opts.merge(scroll: scroll_expiration_time, size: scroll_page_size)
+        results = Results.parse response, query
+        results.append fetch_aggregations if aggregated
+
+        target_size = query.limited? ? query._size : results.hits.total
+        total_pages = (target_size / size.to_f).ceil
+
+        current_page = 1
+
+        loop do
+          break if current_page >= total_pages
+
+          response = client.scroll results.scroll_id, scroll: scroll_expiration_time
+          new_results = Results.parse response
+          hits = new_results.documents
+          doc_count = results.documents.count
+          new_results.documents = hits.first(target_size - doc_count) if query.limited? && (doc_count + hits.count > target_size)
+          results.append new_results
+
+          break if new_results.documents.count < size
+
+          current_page += 1
+        end
+
+        results
+      end
+
+    end
+  end
+end

data/lib/elasticated/repository/search.rb

@@ -0,0 +1,45 @@
+module Elasticated
+  class Repository
+    class Search
+
+      # abstract class
+      # child must implement 'execute()'
+
+      attr_accessor :repository, :query, :aggregated, :opts
+
+      def initialize(repository, query, aggregated=false, opts={})
+        self.repository = repository
+        self.query = query
+        self.aggregated = aggregated
+        self.opts = opts
+      end
+
+      protected
+
+      def override!(body, size=nil, from=nil)
+        size ? body.merge!(size: size) : body.delete(:size)
+        from ? body.merge!(from: from) : body.delete(:from)
+        body
+      end
+
+      # misc getters
+
+      def client
+        repository.client
+      end
+
+      def search_page_size
+        repository.search_page_size
+      end
+
+      def scroll_expiration_time
+        repository.scroll_expiration_time
+      end
+
+      def scroll_page_size
+        repository.scroll_page_size
+      end
+
+    end
+  end
+end

data/lib/elasticated/repository/single_page_search.rb

@@ -0,0 +1,13 @@
+module Elasticated
+  class Repository
+    class SinglePageSearch < Search
+
+      def execute
+        body = aggregated ? query.build_for_aggregated_search : query.build_for_search
+        response = client.search body, opts
+        Results.parse response, query
+      end
+
+    end
+  end
+end

data/lib/elasticated/results.rb

@@ -3,41 +3,59 @@ module Elasticated
   ShardsInfo = Struct.new :total, :successful, :failed
   HitsInfo = Struct.new :total, :max_score
 
-  class Results < Array
-
-    def self.from_elasticsearch_response(elasticsearch_response, query=nil)
-      documents = elasticsearch_response['hits']['hits'].map{ |hit| Document.from_elasticsearch_hit hit }
-      results = new documents
-      results.documents = documents
-      # cluster metadata
-      results.took = elasticsearch_response['took']
-      results.timed_out = elasticsearch_response['timed_out']
-      # shards metadata
-      shards = elasticsearch_response['_shards']
-      results.shards = ShardsInfo.new shards['total'], shards['successful'], shards['failed']
-      # search metadata
-      hits = elasticsearch_response['hits']
-      results.hits = HitsInfo.new hits['total'], hits['max_score']
-      # aggregations results
-      aggregations = elasticsearch_response['aggregations']
-      results.aggregations = query.parse_aggregations aggregations if query && aggregations
-      results
+  class Results
+
+    class << self
+      def parse(elasticsearch_response, query=nil)
+        documents = elasticsearch_response['hits']['hits'].map{ |hit| Document.parse hit }
+        results = new
+        results.documents = documents
+        # scroll metadata
+        results.scroll_id = elasticsearch_response['_scroll_id']
+        # cluster metadata
+        results.took = elasticsearch_response['took']
+        results.timed_out = elasticsearch_response['timed_out']
+        # shards metadata
+        shards = elasticsearch_response['_shards']
+        results.shards = ShardsInfo.new shards['total'], shards['successful'], shards['failed']
+        # search metadata
+        hits = elasticsearch_response['hits']
+        results.hits = HitsInfo.new hits['total'], hits['max_score']
+        # aggregations results
+        aggregations = elasticsearch_response['aggregations']
+        results.aggregations = query.parse_aggregations aggregations if query && aggregations
+        results
+      end
+      alias_method :from_elasticsearch_response, :parse
     end
 
+    attr_accessor :scroll_id
     attr_accessor :took, :timed_out
     attr_accessor :shards # methods: total, successful, failed
     attr_accessor :hits # methods: total, max_score
     attr_accessor :documents, :aggregations
 
-    def append_results_from(elasticsearch_response)
-      elasticsearch_response['hits']['hits'].each do |hit|
-        documents.push Document.from_elasticsearch_hit hit
-      end
+    def append(another_results)
+      self.documents = documents + another_results.documents
+      self.aggregations = another_results.aggregations if another_results.aggregations
+      self.scroll_id = another_results.scroll_id if another_results.scroll_id
       self
     end
 
-    def sources
-      documents.map &:source
+    def sources(with_ids=true)
+      documents.map do |d|
+        d.source.tap do |s|
+          s[:_id] = d.id if with_ids
+        end
+      end
+    end
+
+    def ids
+      documents.map &:id
+    end
+
+    def count
+      documents.count
     end
 
   end

data/lib/version.rb

@@ -1,9 +1,19 @@
 module Elasticated
-  VERSION = '1.2.1'
+  VERSION = '2.0.0'
 end
 
 # Changelog
 
+# 2.0.0
+# Se renombra la aggregation 'count_distinct' por 'value_count'
+# Se separan los metodos de busqueda (scroll, paginacion normal, scan & scroll) en strategies
+# El metodo de busqueda ahora puede ser especificado por el usuario
+# Se agrega ResumableSearch (utiliza un scroll)
+# Se agregan las opciones 'include_count' y 'compact' a varias aggregations
+# Se elimina la clase CountAggregation (es reemplazable por GroupAggregation)
+# Se agregan las aggregations 'stats', 'missing', 'geohash_grid' y 'safe_date_histogram'
+# Ahora el logger default no imprime por stdout
+
 # 1.2.1
 # Ahora se optimiza la construcción de QueryConditions cuando sólo tienen una cláusula should con una condición
 

data/spec/aggregation_spec.rb

@@ -144,38 +144,22 @@ module Elasticated
       expect(agg.build).to eq expected_result
     end
 
-      it "should map a group aggregation response" do
-        agg = GroupAggregation.new :field
-        response = { 'buckets' => [
-          { 'key' => 'value1', 'doc_count' => 'count1' },
-          { 'key' => 'value2', 'doc_count' => 'count2' },
-          { 'key' => 'value3', 'doc_count' => 'count3' }
-        ] }
-        expect(agg.parse(response)).to eq({
-          'value1' => { 'count' => 'count1' },
-          'value2' => { 'count' => 'count2' },
-          'value3' => { 'count' => 'count3' }
-        })
-      end
-
-  end
-
-  describe CountAggregation do
-
-    it "should build a terms aggregation over a field" do
-      agg = CountAggregation.new :user
-      expected_result = { terms: { field: :user, size: 0 } }
-      expect(agg.build).to eq expected_result
-    end
-
-    it "should build a terms aggregation over a field with custom params" do
-      agg = CountAggregation.new :user, script: 'your_script'
-      expected_result = { terms: { field: :user, size: 0, script: 'your_script' } }
-      expect(agg.build).to eq expected_result
+    it "should map a group aggregation response" do
+      agg = GroupAggregation.new :field
+      response = { 'buckets' => [
+        { 'key' => 'value1', 'doc_count' => 'count1' },
+        { 'key' => 'value2', 'doc_count' => 'count2' },
+        { 'key' => 'value3', 'doc_count' => 'count3' }
+      ] }
+      expect(agg.parse(response)).to eq({
+        'value1' => { 'count' => 'count1' },
+        'value2' => { 'count' => 'count2' },
+        'value3' => { 'count' => 'count3' }
+      })
     end
 
-    it "should map a 'count' aggregation response" do
-      agg = CountAggregation.new :field
+    it "should map a group aggregation response compacted" do
+      agg = GroupAggregation.new :field, compact: true
       response = { 'buckets' => [
         { 'key' => 'value1', 'doc_count' => 'count1' },
         { 'key' => 'value2', 'doc_count' => 'count2' },
@@ -190,16 +174,26 @@ module Elasticated
 
   end
 
-  describe CountDistinctAggregation do
+  describe ValueCountAggregation do
 
     it "should build a terms aggregation over a field" do
-      agg = CountDistinctAggregation.new :user
+      agg = ValueCountAggregation.new :user
       expected_result = { value_count: { field: :user } }
       expect(agg.build).to eq expected_result
     end
 
   end
 
+  describe MissingAggregation do
+
+    it "should build a missing aggregation over a field" do
+      agg = MissingAggregation.new :country
+      expected_result = { missing: { field: :country } }
+      expect(agg.build).to eq expected_result
+    end
+
+  end
+
   describe HistogramAggregation do
 
     it "should build a histogram aggregation" do
@@ -474,6 +468,38 @@ module Elasticated
       })
     end
 
+    it "should map a 'range' response compacted" do
+      agg = RangeAggregation.new :a_field, compact: true do |a|
+        a.ranges do |r|
+          r.greater_equal :max_value, 'high_values'
+          r.less_equal :min_value
+          r.between :min_value, :max_value
+        end
+      end
+      response = {
+        'buckets' => {
+          'high_values' => {
+            'from' => 'max_value',
+            'doc_count' => 5
+          },
+          'less_equal_min_value' => {
+            'to' => 'min_value',
+            'doc_count' => 3
+          },
+          'between_min_value_and_max_value' => {
+            'from' => 'min_value',
+            'to' => 'max_value',
+            'doc_count' => 9
+          }
+        }
+      }
+      expect(agg.parse(response)).to eq({
+        'high_values' => 5,
+        'less_equal_min_value' => 3,
+        'between_min_value_and_max_value' => 9
+      })
+    end
+
   end
 
   describe SumDistinctAggregation do

data/spec/document_spec.rb

@@ -12,7 +12,7 @@ module Elasticated
     end
 
     it "should parse an elasticsearch hit" do
-      document = Document.from_elasticsearch_hit hit
+      document = Document.parse hit
       expect(document.id).to eq 'AU-CLCguwlaKln07OA4x'
       expect(document.index).to eq 'fbinsights-v6-2015-08'
       expect(document.type).to eq 'post'
@@ -24,17 +24,17 @@ module Elasticated
     context "the document source" do
 
       it "can be accessed via methods" do
-        document = Document.from_elasticsearch_hit hit
+        document = Document.parse hit
         expect(document.source.page_id).to eq '127735990588679'
       end
 
       it "can be accessed via key" do
-        document = Document.from_elasticsearch_hit hit
+        document = Document.parse hit
         expect(document.source['page_id']).to eq '127735990588679'
       end
 
       it "can be accessed via indifferent key" do
-        document = Document.from_elasticsearch_hit hit
+        document = Document.parse hit
         expect(document.source[:page_id]).to eq '127735990588679'
       end
 

data/spec/intelligent_search_spec.rb

@@ -0,0 +1,88 @@
+require_relative 'spec_helper'
+
+module Elasticated
+  class Repository
+    describe IntelligentSearch do
+
+      let(:repository){ double }
+
+      before :each do
+        allow(repository).to receive(:search_page_size).and_return 1000
+      end
+
+      def search_for(&query_block)
+        query = Query.build &query_block
+        IntelligentSearch.new repository, query
+      end
+
+      context "on unsorted queries" do
+
+        it "should select 'scan & scroll' for an unlimited query without offset" do
+          search = search_for {}
+          expect(search.best_search_method).to be_a ScanScrollSearch
+        end
+
+        it "should select 'single page' for a limited, light-weight query without offset" do
+          search = search_for { size 10 }
+          expect(search.best_search_method).to be_a SinglePageSearch
+        end
+
+        it "should select 'scan & scroll' for a limited, heavy query without offset" do
+          search = search_for { size 9999 }
+          expect(search.best_search_method).to be_a ScanScrollSearch
+        end
+
+        it "should select 'normal pagination' for an unlimited query with offset" do
+          search = search_for { from 10 }
+          expect(search.best_search_method).to be_a NormalSearch
+        end
+
+        it "should select 'single page' for a limited, light-weight query with offset" do
+          search = search_for { size 10; from 10 }
+          expect(search.best_search_method).to be_a SinglePageSearch
+        end
+
+        it "should select 'normal pagination' for a limited, heavy query with offset" do
+          search = search_for { size 9999; from 10 }
+          expect(search.best_search_method).to be_a NormalSearch
+        end
+
+      end
+
+      context "on sorted queries" do
+
+        it "should select 'scroll' for an unlimited query without offset" do
+          search = search_for { sort :field }
+          expect(search.best_search_method).to be_a ScrollSearch
+        end
+
+        it "should select 'single page' for a limited, light-weight query without offset" do
+          search = search_for { sort :field; size 10 }
+          expect(search.best_search_method).to be_a SinglePageSearch
+        end
+
+        it "should select 'scan & scroll' for a limited, heavy query without offset" do
+          search = search_for { sort :field; size 9999 }
+          expect(search.best_search_method).to be_a ScrollSearch
+        end
+
+        it "should select 'normal pagination' for an unlimited query with offset" do
+          search = search_for { sort :field; from 10 }
+          expect(search.best_search_method).to be_a NormalSearch
+        end
+
+        it "should select 'single page' for a limited, light-weight query with offset" do
+          search = search_for { sort :field; size 10; from 10 }
+          expect(search.best_search_method).to be_a SinglePageSearch
+        end
+
+        it "should select 'normal pagination' for a limited, heavy query with offset" do
+          search = search_for { sort :field; size 9999; from 10 }
+          expect(search.best_search_method).to be_a NormalSearch
+        end
+
+      end
+
+    end
+  end
+end