elasticated 2.5.5 → 3.0.0

data/lib/elasticated/repository/intelligent_search.rb

@@ -1,7 +1,7 @@
 module Elasticated
   class Repository
     class IntelligentSearch < Search
-      include Configurable
+      include Mixins::Configurable
 
       # INTELLIGENT SEARCH CASES
       # ------------ without sorting
@@ -15,14 +15,14 @@ module Elasticated
       # without size, with offset     => normal_pagination
       # with size,    with offset     => single_page or normal_pagination
 
-      # if the query is aggregated and the search strategy is use an
+      # if the query is aggregated and the search strategy is a
       # scroll-like alternative, we must do a 2-step search process
 
       def best_search_method
         if query.limited? && !query.heavy_for?(repository)
           SinglePageSearch.new repository, query, aggregated, opts
         else
-          if query.has_offset?
+          if query.offset?
             NormalSearch.new repository, query, aggregated, opts
           else
             if query.sorted?

data/lib/elasticated/repository/normal_search.rb

@@ -10,7 +10,7 @@ module Elasticated
 
         override! body, size, offset
         response = client.search body, opts
-        results = Results.parse response, query
+        results = parse_and_prepare_results response, query
 
         target_size = query.limited? ? query._size : (results.hits.total - offset)
         total_pages = (target_size / size.to_f).ceil
@@ -25,7 +25,7 @@ module Elasticated
           override! body, size, offset
 
           response = client.search body, opts
-          new_results = Results.parse response
+          new_results = parse_and_prepare_results response
           results.append new_results
           break if new_results.count < size
 

data/lib/elasticated/repository/resumable_search.rb

@@ -2,8 +2,8 @@ module Elasticated
   class Repository
     class ResumableSearch < Search
 
-      def self.from_scroll_id(client, scroll_id)
-        search = new client, nil
+      def self.from_scroll_id(repository, scroll_id)
+        search = new repository, nil
         search.scroll_id = scroll_id
         search
       end
@@ -15,12 +15,12 @@ module Elasticated
         aggregation_results = if aggregated
           body = query.build_for_aggregations
           response = client.search body, opts
-          Results.parse response, query
+          parse_and_prepare_results response, query
         end
         # search
         body = query.build_for_search
         response = client.search body, opts.merge(scroll: scroll_expiration_time, size: scroll_page_size)
-        results = Results.parse response
+        results = parse_and_prepare_results response
         results.append aggregation_results if aggregation_results
         self.scroll_id = results.scroll_id
         mark_completed! if results.documents.count < scroll_page_size
@@ -31,7 +31,7 @@ module Elasticated
         raise "No scroll_id present" unless scroll_id
         raise "No more information to fetch: scroll completed" if completed?
         response = client.scroll scroll_id, scroll: scroll_expiration_time
-        results = Results.parse response
+        results = parse_and_prepare_results response
         self.scroll_id = results.scroll_id
         mark_completed! if results.documents.empty?
         results

data/lib/elasticated/repository/scan_scroll_search.rb

@@ -5,7 +5,7 @@ module Elasticated
       def fetch_aggregations
         body = query.build_for_aggregations
         response = client.search body, opts
-        Results.parse response, query
+        parse_and_prepare_results response, query
       end
 
       def execute
@@ -14,17 +14,17 @@ module Elasticated
         if aggregated
           body = query.build_for_aggregations
           response = client.search body, opts
-          results = Results.parse response, query
+          results = parse_and_prepare_results response, query
         end
         # search
         body = query.build_for_search
         response = client.search body, opts.merge(search_type: 'scan', scroll: scroll_expiration_time, size: scroll_page_size)
-        results = Results.parse response
+        results = parse_and_prepare_results response
         results.append fetch_aggregations if aggregated
         doc_count = 0
         loop do
           response = client.scroll results.scroll_id, scroll: scroll_expiration_time
-          new_results = Results.parse response
+          new_results = parse_and_prepare_results response
           hits = new_results.documents
           break if hits.empty?
           if query.limited? && (doc_count + hits.count > query._size)

data/lib/elasticated/repository/scroll_search.rb

@@ -5,7 +5,7 @@ module Elasticated
       def fetch_aggregations
         body = query.build_for_aggregations
         response = client.search body, opts
-        Results.parse response, query
+        parse_and_prepare_results response, query
       end
 
       def execute
@@ -14,7 +14,7 @@ module Elasticated
 
         override! body
         response = client.search body, opts.merge(scroll: scroll_expiration_time, size: scroll_page_size)
-        results = Results.parse response, query
+        results = parse_and_prepare_results response, query
         results.append fetch_aggregations if aggregated
 
         target_size = query.limited? ? query._size : results.hits.total
@@ -26,7 +26,7 @@ module Elasticated
           break if current_page >= total_pages
 
           response = client.scroll results.scroll_id, scroll: scroll_expiration_time
-          new_results = Results.parse response
+          new_results = parse_and_prepare_results response
           hits = new_results.documents
           doc_count = results.documents.count
           new_results.documents = hits.first(target_size - doc_count) if query.limited? && (doc_count + hits.count > target_size)

data/lib/elasticated/repository/search.rb

@@ -40,6 +40,13 @@ module Elasticated
         repository.scroll_page_size
       end
 
+      # results preparation
+
+      def parse_and_prepare_results(response, query=nil)
+        results = Results.parse response, query
+        repository.send :prepare_served_results, results # TODO fixme
+      end
+
     end
   end
 end

data/lib/elasticated/repository/single_page_search.rb

@@ -5,7 +5,7 @@ module Elasticated
       def execute
         body = aggregated ? query.build_for_aggregated_search : query.build_for_search
         response = client.search body, opts
-        Results.parse response, query
+        parse_and_prepare_results response, query
       end
 
     end

data/lib/elasticated/results.rb

@@ -29,6 +29,8 @@ module Elasticated
       alias_method :from_elasticsearch_response, :parse
     end
 
+    include Mixins::Inspectionable
+
     attr_accessor :scroll_id
     attr_accessor :took, :timed_out
     attr_accessor :shards # methods: total, successful, failed
@@ -58,5 +60,17 @@ module Elasticated
       documents.count
     end
 
+    def text_for_inspect
+      text = "#{hits.total} hits"
+      text = case documents.count
+      when 0; "#{text}, no documents"
+      when 1; "#{text}, 1 document"
+      else; "#{text}, #{documents.count} documents"
+      end
+      text = aggregations ? "#{text}, with aggregations" : "#{text}, no aggregations"
+      text = "#{text}, including scroll_id" if scroll_id
+      text
+    end
+
   end
 end

data/lib/version.rb

@@ -1,33 +1,26 @@
 module Elasticated
-  VERSION = '2.5.5'
+  VERSION = '3.0.0'
 end
 
 # Changelog
 
-# 2.5.5
-# Se extrae un metodo en IndexSelector para asegurar compatibilidad hacia atras
-
-# 2.5.4
-# Se corrige un bug del date delimiter
-
-# 2.5.3
-# Se corrige un bug en el delimiter visitor
-
-# 2.5.2
-# Se corrige el visitor para que los filtros de subaggregations se representen en conjuntos por aggregation y no por filtro
-
-# 2.5.1
-# Se agregaron test para el merge delimiters en term delimiter factory que ahora considera arrays de delimiters vacios
-
-# 2.5.0
-# Ahora los delimiters son inmutables
-# Se puede delimitar por aggregations
-
-# 2.4.0
-# Ahora se puede ordenar un search por un campo, script o hash custom
-
-# 2.3.0
-# Ahora DateFieldDelimiter admite multiples fechas declaradas en un TermsCondition
+# Ahora Repository#execute_aggregated_search puede definir automaticamente si ejecutar las aggregations o no
+# Se agrega el alias 'range' para la condicion 'between'
+# Se agrega el alias 'terms' para la condicion 'equal'
+# Se agregan get, delete, y mget a Repository
+# Ahora el GroupAggregation se puede invocar como 'terms' ademas de 'group'
+# Se elimina la opcion 'include_count' de todas las aggregations que la utilizaban
+# Ya no se permite pasar la opcion 'compact' a aggregations que posean subaggregations
+# Ahora se puede llamar a bloque de aggregations por medio del alias 'aggs'
+# Ahora Repository#update_document actualiza parcialmente el documento
+# Cambian los pre-action y post-action preparation methods
+# Se incluye la extension 'test cluster' y las tareas rake para administrarlo
+# ResumableSearch ahora hace 'restore' de los documentos que devuelve
+# Se agregan mejores vistas de inspeccion para Query, Results y Document
+# Ahora se pueden pedir los documentos sin source
+# Ahora el PartitionedRepository soporta :now
+# Se agrega TermCondition
+# Ahora se puede especificar si el mapping es dinamico
 
 # 2.2.0
 # Se agrega la funcionalidad para especificar un orden aleatorio en las queries

data/spec/aggregation_spec.rb

@@ -147,9 +147,9 @@ module Elasticated
     it "should map a group aggregation response" do
       agg = GroupAggregation.new :field
       response = { 'buckets' => [
-        { 'key' => 'value1', 'doc_count' => 'count1' },
-        { 'key' => 'value2', 'doc_count' => 'count2' },
-        { 'key' => 'value3', 'doc_count' => 'count3' }
+        { 'key' => 'value1', 'doc_count' => 'count1', 'key_as_string' => 'value1str' },
+        { 'key' => 'value2', 'doc_count' => 'count2', 'key_as_string' => 'value2str' },
+        { 'key' => 'value3', 'doc_count' => 'count3', 'key_as_string' => 'value3str' }
       ] }
       expect(agg.parse(response)).to eq({
         'value1' => { 'count' => 'count1' },
@@ -158,12 +158,26 @@ module Elasticated
       })
     end
 
+    it "should map a group aggregation response including the 'key_as_string' value" do
+      agg = GroupAggregation.new :field, key_as_string: true
+      response = { 'buckets' => [
+        { 'key' => 'value1', 'doc_count' => 'count1', 'key_as_string' => 'value1str' },
+        { 'key' => 'value2', 'doc_count' => 'count2', 'key_as_string' => 'value2str' },
+        { 'key' => 'value3', 'doc_count' => 'count3', 'key_as_string' => 'value3str' }
+      ] }
+      expect(agg.parse(response)).to eq({
+        'value1' => { 'count' => 'count1', 'key_as_string' => 'value1str' },
+        'value2' => { 'count' => 'count2', 'key_as_string' => 'value2str' },
+        'value3' => { 'count' => 'count3', 'key_as_string' => 'value3str' }
+      })
+    end
+
     it "should map a group aggregation response compacted" do
       agg = GroupAggregation.new :field, compact: true
       response = { 'buckets' => [
-        { 'key' => 'value1', 'doc_count' => 'count1' },
-        { 'key' => 'value2', 'doc_count' => 'count2' },
-        { 'key' => 'value3', 'doc_count' => 'count3' }
+        { 'key' => 'value1', 'doc_count' => 'count1', 'key_as_string' => 'value1str' },
+        { 'key' => 'value2', 'doc_count' => 'count2', 'key_as_string' => 'value2str' },
+        { 'key' => 'value3', 'doc_count' => 'count3', 'key_as_string' => 'value3str' }
       ] }
       expect(agg.parse(response)).to eq({
         'value1' => 'count1',
@@ -172,6 +186,16 @@ module Elasticated
       })
     end
 
+    it "should raise if a group aggregation is compacted but has subaggregations" do
+      agg = GroupAggregation.new(:field, compact: true){ max :other_field }
+      response = { 'buckets' => [
+        { 'key' => 'value1', 'doc_count' => 'count1', 'key_as_string' => 'value1str' },
+        { 'key' => 'value2', 'doc_count' => 'count2', 'key_as_string' => 'value2str' },
+        { 'key' => 'value3', 'doc_count' => 'count3', 'key_as_string' => 'value3str' }
+      ] }
+      expect{ agg.parse(response) }.to raise_error RuntimeError
+    end
+
   end
 
   describe ValueCountAggregation do
@@ -251,12 +275,6 @@ module Elasticated
       expect(agg.build).to eq expected_result
     end
 
-    it "should build a date histogram aggregation with offset for date" do
-      agg = DateHistogramAggregation.new :date, offset: '+6h'
-      expected_result = { date_histogram: { field: :date, interval: 'day', format: 'yyyy-MM-dd', offset: '+6h' } }
-      expect(agg.build).to eq expected_result
-    end
-
     it "should build a date histogram aggregation with subaggregations" do
       agg = DateHistogramAggregation.new :date do |docs|
         docs.max :numeric_field
@@ -474,6 +492,42 @@ module Elasticated
       })
     end
 
+    it "should map a 'range' response with subaggregations" do
+      agg = RangeAggregation.new :a_field do |a|
+        a.ranges do |r|
+          r.greater_equal :max_value, 'high_values'
+          r.less_equal :min_value
+          r.between :min_value, :max_value
+        end
+        a.max :other_field
+      end
+      response = {
+        'buckets' => {
+          'high_values' => {
+            'from' => 'max_value',
+            'doc_count' => 5,
+            'max_other_field' => { 'value' => 15 }
+          },
+          'less_equal_min_value' => {
+            'to' => 'min_value',
+            'doc_count' => 3,
+            'max_other_field' => { 'value' => 15 }
+          },
+          'between_min_value_and_max_value' => {
+            'from' => 'min_value',
+            'to' => 'max_value',
+            'doc_count' => 9,
+            'max_other_field' => { 'value' => 15 }
+          }
+        }
+      }
+      expect(agg.parse(response)).to eq({
+        'high_values' => { 'count' => 5, 'max_other_field' => 15 },
+        'less_equal_min_value' => { 'count' => 3, 'max_other_field' => 15 },
+        'between_min_value_and_max_value' => { 'count' => 9, 'max_other_field' => 15 }
+      })
+    end
+
     it "should map a 'range' response compacted" do
       agg = RangeAggregation.new :a_field, compact: true do |a|
         a.ranges do |r|
@@ -506,6 +560,35 @@ module Elasticated
       })
     end
 
+    it "should raise on a subaggregated and compacted 'range'" do
+      agg = RangeAggregation.new :a_field, compact: true do |a|
+        a.ranges do |r|
+          r.greater_equal :max_value, 'high_values'
+          r.less_equal :min_value
+          r.between :min_value, :max_value
+        end
+        a.max :other_field
+      end
+      response = {
+        'buckets' => {
+          'high_values' => {
+            'from' => 'max_value',
+            'doc_count' => 5
+          },
+          'less_equal_min_value' => {
+            'to' => 'min_value',
+            'doc_count' => 3
+          },
+          'between_min_value_and_max_value' => {
+            'from' => 'min_value',
+            'to' => 'max_value',
+            'doc_count' => 9
+          }
+        }
+      }
+      expect{ agg.parse(response) }.to raise_error RuntimeError
+    end
+
   end
 
   describe SumDistinctAggregation do
@@ -550,10 +633,6 @@ module Elasticated
 
   describe TopHitsAggregation do
 
-    def open_response(name)
-      JSON.parse File.read "spec/#{name}.json"
-    end
-
     it "should build a 'top hits' aggregation" do
       agg = TopHitsAggregation.new :some_name do |q|
         q.size 5

data/spec/bulk_request_spec.rb

@@ -0,0 +1,158 @@
+require_relative 'spec_helper'
+
+module Elasticated
+  describe BulkRequest do
+
+    let :repository do
+      Repository.new
+    end
+
+    context "in the index-like actions" do
+
+      let :document do
+        Document.create do |doc|
+          doc.id = '2'
+          doc.type = 'post'
+          doc.index = 'insights'
+          doc.source = { some_field: 'some_value' }
+        end
+      end
+
+      it "should build an index action (with only a doc)" do
+        action = BulkActions::IndexAction.new repository, document
+        expect(action.build).to eq [
+          { index: { _index: "insights", _type: "post", _id: "2" } },
+          { some_field: "some_value" }
+        ]
+      end
+
+      it "should build an index action (with a doc and other params)" do
+        action = BulkActions::IndexAction.new repository, document, id: "5", type: 'faketype', index: 'fakeindex'
+        expect(action.build).to eq [
+          { index: { _index: "fakeindex", _type: "faketype", _id: "5" } },
+          { some_field: "some_value" }
+        ]
+      end
+
+      it "should build an create action (with only a doc)" do
+        action = BulkActions::CreateAction.new repository, document
+        expect(action.build).to eq [
+          { create: { _index: "insights", _type: "post", _id: "2" } },
+          { some_field: "some_value" }
+        ]
+      end
+
+      it "should build an create action (with a doc and other params)" do
+        action = BulkActions::CreateAction.new repository, document, id: "5", type: 'faketype', index: 'fakeindex'
+        expect(action.build).to eq [
+          { create: { _index: "fakeindex", _type: "faketype", _id: "5" } },
+          { some_field: "some_value" }
+        ]
+      end
+
+    end
+
+    context "in the update-like actions" do
+
+      let(:source){ { some_field: 'some_value' } }
+
+      it "should build an update action" do
+        action = BulkActions::UpdateAction.new repository, "2", type: "post", index: "insights", source: source
+        expect(action.build).to eq [
+          { update: { _index: "insights", _type: "post", _id: "2" } },
+          { doc: { some_field: "some_value" } }
+        ]
+      end
+
+      it "should build an upsert action" do
+        action = BulkActions::UpsertAction.new repository, "2", type: "post", index: "insights", source: source
+        expect(action.build).to eq [
+          { update: { _index: "insights", _type: "post", _id: "2" } },
+          { doc: { some_field: "some_value" }, doc_as_upsert: true }
+        ]
+      end
+
+    end
+
+    context "in the delete-like actions" do
+
+      it "should build a delete action" do
+        action = BulkActions::DeleteAction.new repository, '2', index: 'insights', type: 'post'
+        expect(action.build).to eq [
+          { delete: { _index: "insights", _type: "post", _id: "2" } }
+        ]
+      end
+
+    end
+
+    context "the bulk request" do
+
+      let :source do
+        { some_field: 'some_value' }
+      end
+
+      let :document do
+        Document.create do |doc|
+          doc.id = '2'
+          doc.type = 'post'
+          doc.index = 'insights'
+          doc.source = source
+        end
+      end
+
+      it "should build a serie of actions correctly" do
+        bulk_request = BulkRequest.build_over repository do |r|
+          r.delete_document "2", type: 'post', index: 'insights'
+          r.create_document document
+          r.update_document "2", type: 'post', index: 'insights', source: source
+          r.index_document document
+        end
+        expect(bulk_request.to_text).to eq \
+          '{"delete":{"_index":"insights","_type":"post","_id":"2"}}' + "\n" +
+          '{"create":{"_index":"insights","_type":"post","_id":"2"}}' + "\n" +
+          '{"some_field":"some_value"}' + "\n" +
+          '{"update":{"_index":"insights","_type":"post","_id":"2"}}' + "\n" +
+          '{"doc":{"some_field":"some_value"}}' + "\n" +
+          '{"index":{"_index":"insights","_type":"post","_id":"2"}}' + "\n" +
+          '{"some_field":"some_value"}'
+      end
+
+      it "should parse a simple response" do
+        elasticsearch_response = open_response 'elasticsearch_bulk_response_1'
+        bulk_request = BulkRequest.build_over repository # actions actually doesnt matter
+        response = bulk_request.parse elasticsearch_response
+        expect(response.errors?).to be false
+        expect(response.items.count).to eql 4
+        item1 = response.items[0]
+        expect(item1.action).to eq 'delete'
+        expect(item1.index).to eq 'website'
+        expect(item1.type).to eq 'blog'
+        expect(item1.id).to eq '123'
+        expect(item1.version).to eql 2
+        expect(item1.status).to eq 200
+        expect(item1.found).to be true
+        expect(item1.error?).to be false
+        expect(item1.completed?).to be true
+      end
+
+      it "should parse a simple response with errors" do
+        elasticsearch_response = open_response 'elasticsearch_bulk_response_2'
+        bulk_request = BulkRequest.build_over repository # actions actually doesnt matter
+        response = bulk_request.parse elasticsearch_response
+        expect(response.errors?).to be true
+        expect(response.items.count).to eql 2
+        item1 = response.items[0]
+        expect(item1.action).to eq 'create'
+        expect(item1.index).to eq 'website'
+        expect(item1.type).to eq 'blog'
+        expect(item1.id).to eq '123'
+        expect(item1.status).to eq 409
+        expect(item1.error).to eq 'DocumentAlreadyExistsException[[website][4] [blog][123]: document already exists]'
+        expect(item1.error?).to be true
+        expect(item1.completed?).to be false
+      end
+
+    end
+
+  end
+end