es-elasticity 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 720cd13dc5ba2ce7e66491cf089672f11d8a755d
-  data.tar.gz: 203824852dd40864503b465430fa401fec2d758e
+  metadata.gz: 273883b20e8b274c5029f67bc93985b5f2405348
+  data.tar.gz: 5960c883e06d2bcfa72e501f3fc4dbd0cd941c46
 SHA512:
-  metadata.gz: 1bfd03d5807b389ba5d9cd38105ce26d997c1ae2d028462b1f28a6f271d1220e375c32c48af8c700cd232a881648599524a56e0ccc912c0489739d55509820af
-  data.tar.gz: 9246d84d925302de828eab261f940039b64bf28c6b72df57ae512c9854aeda14dd095a57cf62aa5da2f88ddd3769ba5ae226ec8ac60b70c8a0b05cb02da8e01b
+  metadata.gz: 1d9ecb7b3ede09bc2ea49b64cd3752fd8ca502b2381aee55e9dd3ce421f1a1c72f7d0dcb1a02fa8d8af5bdc66d9110f7cfc1025396d85fbb8a4126ab3e0b140b
+  data.tar.gz: d10e1f5da32facb6a8ecaead082f71add59a876de3e6220db95fb0984a50a09d2540d9cfd83e3fc3fa3186a70b745ba1c4b50dfedc17b2ba1c32f0f2ebd0ea85

data/.travis.yml

@@ -1,13 +1,17 @@
 language: ruby
 before_install:
   - gem update bundler
+  - curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.1.deb && sudo dpkg -i --force-confnew elasticsearch-5.1.1.deb && sudo service elasticsearch restart
 rvm:
   - 2.2.2
   - 2.2.3
-services:
-  - elasticsearch
+addons:
+  apt:
+    packages:
+      - oracle-java8-set-default
 before_script:
   - sleep 5
 env:
   global:
     - secure: "HZa3D2GGwC6Jl062LJulbWZLTCieeBFC3FOJrArC5ul7ACGR5CEtANe0/UTnIf/Ad40p7I5VhiTdNFTcHunTbNc7Ae7dE5fOkiBHtxo/zwgpvHZK0iPvIoxsSfdcHobHeaF7NvfcXUkYUKcdRUyplHdB56eHQqYPVsah66K/4XA="
+sudo: true

data/CHANGELOG

@@ -1,3 +1,6 @@
+v0.8.1
+	- loosen support for elasticsearch-ruby versions to support more versions of
+	elasticsearch
 v0.8.0
  - Make Elasticity::Strategies::AliasIndex the default
  - Use mapping instead of mappings, we wanna be consistent to ES not to elasticsearch-ruby

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in elasticity.gemspec
 gemspec
+
+gem "elasticsearch", "5.0.3"

data/README.md

@@ -28,6 +28,11 @@ Or install it yourself as:
 
 ## Usage
 
+### Version Support
+This gem has [elasticsearch-ruby](https://github.com/elastic/elasticsearch-ruby) as a dependency.  In order to use different versions of elasticsearch you will need to match your version of elasticsearch-ruby to the version of elasticsearch you want to use ([see here](https://github.com/elastic/elasticsearch-ruby#compatibility).  Elasticity should work across all versions of elastisearch-ruby, although they have not all been tested so there are likely edge cases.
+
+Currently tests are run on travis ci against elasticsearch 5.1.1 with elasticsearch-ruby 5.0.3.
+
 ### Configuration
 
 It is recommended you use Typhoeus for HTTP connections to Elasticsearch.

data/elasticity.gemspec

@@ -25,10 +25,11 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "simplecov", "~> 0.7.1"
   spec.add_development_dependency "oj"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "byebug"
   spec.add_development_dependency "codeclimate-test-reporter"
   spec.add_development_dependency "redis"
 
   spec.add_dependency "activesupport", ">= 4.0.0", "< 6"
   spec.add_dependency "activemodel",   ">= 4.0.0", "< 6"
-  spec.add_dependency "elasticsearch", "~> 1.0.12"
+  spec.add_dependency "elasticsearch", ">= 1.0"
 end

data/lib/elasticity/scrollable_search.rb

@@ -0,0 +1,15 @@
+module Elasticity
+  class ScrollableSearch
+    def self.search_type
+      if elasticsearch_gem_version < Gem::Version.create("2.0")
+        :scan
+      else
+        :query_then_fetch
+      end
+    end
+
+    def self.elasticsearch_gem_version
+      Gem.loaded_specs["elasticsearch"].version
+    end
+  end
+end

data/lib/elasticity/search.rb

@@ -177,7 +177,7 @@ module Elasticity
       def search
         return @search if defined?(@search)
         args    = @search_definition.to_search_args
-        args    = args.merge(search_type: 'scan', size: @size, scroll: @scroll)
+        args    = args.merge(search_type: ScrollableSearch.search_type, size: @size, scroll: @scroll)
         @search = @client.search(args)
       end
     end

data/lib/elasticity/strategies/alias_index.rb

@@ -52,9 +52,8 @@ module Elasticity
           })
 
           @client.index_flush(index: original_index)
-          cursor = @client.search index: original_index, search_type: 'scan', scroll: '10m', _source: false, size: 100
+          cursor = @client.search index: original_index, search_type: ScrollableSearch.search_type, scroll: '10m', size: 100
           loop do
-            cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
             hits   = cursor['hits']['hits']
             break if hits.empty?
 
@@ -84,6 +83,7 @@ module Elasticity
             end
 
             @client.bulk(body: ops) unless ops.empty?
+            cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
           end
 
           # Update aliases to only point to the new index.
@@ -103,9 +103,8 @@ module Elasticity
           })
 
           @client.index_flush(index: new_index)
-          cursor = @client.search index: new_index, search_type: 'scan', scroll: '1m', size: 100
+          cursor = @client.search index: new_index, search_type: ScrollableSearch.search_type, scroll: '1m', size: 100
           loop do
-            cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
             hits   = cursor['hits']['hits']
             break if hits.empty?
 
@@ -116,6 +115,7 @@ module Elasticity
             end
 
             @client.bulk(body: ops)
+            cursor = @client.scroll(scroll_id: cursor['_scroll_id'], scroll: '1m')
           end
 
           @client.index_flush(index: original_index)
@@ -149,13 +149,13 @@ module Elasticity
       end
 
       def main_indexes
-        @client.index_get_aliases(index: "#{@main_alias}-*", name: @main_alias).keys
+        @client.index_get_alias(index: "#{@main_alias}-*", name: @main_alias).keys
       rescue Elasticsearch::Transport::Transport::Errors::NotFound
         []
       end
 
       def update_indexes
-        @client.index_get_aliases(index: "#{@main_alias}-*", name: @update_alias).keys
+        @client.index_get_alias(index: "#{@main_alias}-*", name: @update_alias).keys
       rescue Elasticsearch::Transport::Transport::Errors::NotFound
         []
       end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "0.8.0"
+  VERSION = "0.8.1"
 end

data/lib/elasticity.rb

@@ -34,6 +34,7 @@ module Elasticity
   autoload :MultiSearch,        "elasticity/multi_search"
   autoload :Search,             "elasticity/search"
   autoload :Strategies,         "elasticity/strategies"
+  autoload :ScrollableSearch,   "elasticity/scrollable_search"
 
   def self.configure
     @config = Config.new

data/spec/functional/persistence_spec.rb

@@ -1,4 +1,8 @@
 RSpec.describe "Persistence", elasticsearch: true do
+  def random_birthdate
+    Time.at(0.0 + rand * (Time.now.to_f - 0.0.to_f))
+  end
+
   describe "single index strategy" do
     subject do
       Class.new(Elasticity::Document) do
@@ -13,7 +17,7 @@ RSpec.describe "Persistence", elasticsearch: true do
 
           c.mapping = {
             properties: {
-              name: { type: "string" },
+              name: { type: "string", index: "not_analyzed" },
               birthdate: { type: "date" },
             },
           }
@@ -50,23 +54,20 @@ RSpec.describe "Persistence", elasticsearch: true do
 
       subject.flush_index
 
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq 2
 
-      expect(results[0]).to eq(john)
-      expect(results[1]).to eq(mari)
-
-      expect(subject.search({query: {filtered: { query: { match_all: {} } } } }).count).to eq(2)
+      expect(subject.search({ query: { match_all: {} } }).count).to eq(2)
 
       john.update
       mari.delete
 
       subject.flush_index
 
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq 1
 
-      expect(results[0]).to eq(john)
+      expect(results[0].name).to eq(john.name)
     end
   end
 
@@ -85,7 +86,7 @@ RSpec.describe "Persistence", elasticsearch: true do
         c.strategy = Elasticity::Strategies::SingleIndex
         c.document_type  = "cat"
         c.mapping = { properties: {
-          name: { type: "string" },
+          name: { type: "string", index: "not_analyzed" },
           age: { type: "integer" }
         } }
       end
@@ -103,7 +104,7 @@ RSpec.describe "Persistence", elasticsearch: true do
         c.strategy = Elasticity::Strategies::SingleIndex
         c.document_type = "dog"
         c.mapping = { properties: {
-          name: { type: "string" },
+          name: { type: "string", index: "not_analyzed" },
           age: { type: "integer" },
           hungry: { type: "boolean" }
         } }
@@ -163,10 +164,9 @@ RSpec.describe "Persistence", elasticsearch: true do
           c.strategy        =  Elasticity::Strategies::AliasIndex
 
           c.mapping = {
-            _id: { path: "id" },
             properties: {
               id: { type: "integer" },
-              name: { type: "string" },
+              name: { type: "string", index: "not_analyzed" },
               birthdate: { type: "date" },
             },
           }
@@ -194,15 +194,14 @@ RSpec.describe "Persistence", elasticsearch: true do
     end
 
     it "remaps to a different index transparently" do
-      john = subject.new(id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
-      mari = subject.new(id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
+      john = subject.new(_id: 1, id: 1, name: "John", birthdate: "1985-10-31", sort: ['john'])
+      mari = subject.new(_id: 2, id: 2, name: "Mari", birthdate: "1986-09-24", sort: ['mari'])
 
       john.update
       mari.update
 
       subject.flush_index
-
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq 2
 
       subject.remap!
@@ -212,16 +211,16 @@ RSpec.describe "Persistence", elasticsearch: true do
 
       subject.flush_index
 
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq 1
 
-      expect(results[0]).to eq(john)
+      expect(results[0].name).to eq(john.name)
     end
 
     it "handles in between state while remapping" do
       number_of_docs = 2000
       docs = number_of_docs.times.map do |i|
-        subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+        subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
       end
 
       t = Thread.new { subject.remap! }
@@ -233,20 +232,20 @@ RSpec.describe "Persistence", elasticsearch: true do
       to_delete.each(&:delete)
 
       20.times.map do |i|
-        subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+        subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
       end
 
       t.join
 
       subject.flush_index
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq(2010)
     end
 
     it "recover from remap interrupts" do
       number_of_docs = 2000
       docs = number_of_docs.times.map do |i|
-        subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+        subject.new(id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
       end
 
       t = Thread.new { subject.remap! }
@@ -258,20 +257,20 @@ RSpec.describe "Persistence", elasticsearch: true do
       to_delete.each(&:delete)
 
       20.times.map do |i|
-        subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+        subject.new(id: i + number_of_docs, name: "User #{i + docs.length}", birthdate: random_birthdate).tap(&:update)
       end
 
       t.raise("Test Interrupt")
       expect { t.join }.to raise_error("Test Interrupt")
 
       subject.flush_index
-      results = subject.search(sort: :name)
+      results = subject.search({})
       expect(results.total).to eq(2010)
     end
 
     it "bulk indexes, updates and delete" do
       docs = 2000.times.map do |i|
-        subject.new(id: i, name: "User #{i}", birthdate: "#{rand(20)+1980}-#{rand(11)+1}-#{rand(28)+1}").tap(&:update)
+        subject.new(_id: i, id: i, name: "User #{i}", birthdate: random_birthdate).tap(&:update)
       end
 
       subject.bulk_index(docs)
@@ -289,7 +288,6 @@ RSpec.describe "Persistence", elasticsearch: true do
 
       results = subject.search(from: 0, size: 3000)
       expect(results.total).to eq 2000
-
       expect(subject.search({ query: { match: { name: "Updated" } } } ).count).to eq(2000)
 
       subject.bulk_delete(results.documents.map(&:_id))

data/spec/rspec_config.rb

@@ -4,6 +4,8 @@ require "oj"
 SimpleCov.start
 
 require "elasticity"
+require "pry"
+require "byebug"
 
 def elastic_search_client
   return @elastic_search_client if defined?(@elastic_search_client)

data/spec/units/search_spec.rb

@@ -108,7 +108,7 @@ RSpec.describe "Search" do
     end
 
     it "searches using scan&scroll" do
-      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: "scan", size: 100, scroll: "1m").and_return(scan_response)
+      expect(client).to receive(:search).with(index: index_name, type: document_type, body: body, search_type: :query_then_fetch, size: 100, scroll: "1m").and_return(scan_response)
       expect(client).to receive(:scroll).with(scroll_id: "abc123", scroll: "1m").and_return(scroll_response)
       expect(client).to receive(:scroll).with(scroll_id: "abc456", scroll: "1m").and_return(empty_response)
 

data/spec/units/strategies/single_index_spec.rb

@@ -8,7 +8,7 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
       "mappings" => {
         "document" => {
           "properties" => {
-            "name" => { "type" => "string" }
+            "name" => { "type" => "text" }
           }
         }
       }
@@ -54,14 +54,14 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
       results_a = subject.bulk do |b|
         b.index "document", 1, name: "foo"
       end
-      expect(results_a).to include("errors"=>false, "items"=>[{"index"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"1", "_version"=>1, "status"=>201}}])
+      expect(results_a["errors"]).to be_falsey
 
       results_b = subject.bulk do |b|
         b.index  "document", 2, name: "bar"
         b.delete "document", 1
       end
 
-      expect(results_b).to include("errors"=>false, "items"=>[{"index"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"2", "_version"=>1, "status"=>201}}, {"delete"=>{"_index"=>"test_index_name", "_type"=>"document", "_id"=>"1", "_version"=>2, "status"=>200, "found"=>true}}])
+      expect(results_b["errors"]).to be_falsey
 
       subject.flush
 
@@ -69,11 +69,12 @@ RSpec.describe Elasticity::Strategies::SingleIndex, elasticsearch: true do
       expect(subject.get_document("document", 2)).to eq({"_index"=>"test_index_name", "_type"=>"document", "_id"=>"2", "_version"=>1, "found"=>true, "_source"=>{"name"=>"bar"}})
     end
 
-    it "allows deleting by queryu" do
+    it "allows deleting by query" do
       subject.index_document("document", 1, name: "foo")
       subject.index_document("document", 2, name: "bar")
 
-      subject.delete_by_query("document", query: { term: { name: "foo" }})
+      subject.flush
+      subject.delete_by_query("document", query: { term: { name: "foo" } })
 
       expect { subject.get_document("document", 1) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       expect { subject.get_document("document", 2) }.to_not raise_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: es-elasticity
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Rodrigo Kochenburger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-22 00:00:00.000000000 Z
+date: 2017-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
@@ -166,16 +180,16 @@ dependencies:
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.12
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.12
+        version: '1.0'
 description: Elasticity provides a higher level abstraction on top of [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby)
   gem
 email:
@@ -208,6 +222,7 @@ files:
 - lib/elasticity/log_subscriber.rb
 - lib/elasticity/multi_search.rb
 - lib/elasticity/railtie.rb
+- lib/elasticity/scrollable_search.rb
 - lib/elasticity/search.rb
 - lib/elasticity/segmented_document.rb
 - lib/elasticity/strategies.rb
@@ -243,7 +258,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.3
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: ActiveModel-based library for working with Elasticsearch
@@ -257,4 +272,3 @@ test_files:
 - spec/units/multi_search_spec.rb
 - spec/units/search_spec.rb
 - spec/units/strategies/single_index_spec.rb
-has_rdoc: