elastic_record 3.1.4 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c59b1b2830ef2314166e53fed56b060628e31962
-  data.tar.gz: c88b89a8a5335502aa4cf0bb8b46fb477dcb8f4e
+  metadata.gz: 95195e9bd6f65e35a24b3ed7d8a3d8ee4170a214
+  data.tar.gz: 65331f5fec26898efe80717794ef0a5ca897895e
 SHA512:
-  metadata.gz: '09c62833cc9bb4bc486b5b53d65ac1c0a2395129edd6fbe1f887a5e208e75a99e585d898b2c8e1661084268d48f1d6a836e92654dba5bb6de0cf812849f9d502'
-  data.tar.gz: 8948ebb21a69e0eaa17e51f2456a73f2d7cde27312cdb4d7c52c661742c4e8fbbfef7f8996d0d4e9e652e98f95ecfba0782e90bd2966ff2606a239b49426615c
+  metadata.gz: d151b8500473e3634658de2f8b1869bc25a480c081faf6a48a952312ff6f5db7409c3b26a04424750ddf36611f45fbb659440c32a0e52deeb3147670668ac86c
+  data.tar.gz: 27171232b4f59a2f290e323c5742b24034926f1231275f6258cb6463e1f8bea9c217e07a3f16530d96791d8d4fbe11812967aa27b2e9fc99bbd137de33af3319

data/Gemfile

@@ -3,9 +3,10 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'dotenv-rails'
-gem 'mysql2'
+# gem 'mysql2'
 gem 'oj'
 gem 'pg'
 gem 'rails'
 gem 'rake', '~> 10.5.0'
 gem 'webmock', require: false
+gem 'arelastic'

data/README.md

@@ -48,7 +48,7 @@ If a simple hash is passed into filter, a term or terms query is created:
 ```ruby
 search.filter(color: 'red')         # Creates a 'term' filter
 search.filter(color: %w(red blue))  # Creates a 'terms' filter
-search.filter(color: nil)           # Creates a 'missing' filter
+search.filter(color: nil)           # Creates a 'must not exist' filter
 ```
 
 If a hash containing hashes is passed into filter, it is used directly as a filter DSL expression:

data/elastic_record.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = 'elastic_record'
-  s.version = '3.1.4'
+  s.version = '4.0.0'
   s.summary = 'An Elasticsearch querying ORM'
   s.description = 'Find your records with Elasticsearch'
 
@@ -18,6 +18,6 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files`.split("\n")
   s.test_files  = `git ls-files -- {test}/*`.split("\n")
 
-  s.add_dependency 'arelastic', '~> 1.1', '>= 1.1.2'
+  s.add_dependency 'arelastic', '>= 2.0.0'
   s.add_dependency 'activemodel'
 end

data/lib/elastic_record/config.rb

@@ -31,8 +31,8 @@ module ElasticRecord
         self.servers = settings['servers']
         self.connection_options = settings
 
-        if scroll_keep_alive = settings['scroll_keep_alive'].presence
-          self.scroll_keep_alive = scroll_keep_alive
+        if settings['scroll_keep_alive'].present?
+          self.scroll_keep_alive = settings['scroll_keep_alive']
         end
       end
     end

data/lib/elastic_record/index/analyze.rb

@@ -0,0 +1,10 @@
+module ElasticRecord
+  class Index
+    module Analyze
+      def analyze(params)
+        json = connection.json_get "/#{alias_name}/_analyze", params
+        json['tokens'].map { |token_hash| token_hash['token'] }
+      end
+    end
+  end
+end

data/lib/elastic_record/index/documents.rb

@@ -2,40 +2,51 @@ require 'active_support/core_ext/object/to_query'
 
 module ElasticRecord
   class Index
-    class ScanSearch
-      attr_reader :scroll_id
-      attr_accessor :total_hits
-
-      def initialize(elastic_index, scroll_id, options = {})
+    class ScrollEnumerator
+      attr_reader :keep_alive, :batch_size, :scroll_id
+      def initialize(elastic_index, search: nil, scroll_id: nil, keep_alive:, batch_size:)
         @elastic_index  = elastic_index
+        @search         = search
         @scroll_id      = scroll_id
-        @options        = options
+        @keep_alive     = keep_alive
+        @batch_size     = batch_size
       end
 
       def each_slice(&block)
         while (hit_ids = request_more_ids).any?
-          hit_ids.each_slice(requested_batch_size, &block)
+          hit_ids.each_slice(batch_size, &block)
         end
       end
 
-      def request_more_hits
-        request_next_scroll['hits']['hits']
-      end
-
       def request_more_ids
         request_more_hits.map { |hit| hit['_id'] }
       end
 
+      def request_more_hits
+        request_next_scroll['hits']['hits']
+      end
+
       def request_next_scroll
-        @elastic_index.scroll(@scroll_id, keep_alive)
+        if scroll_id.nil?
+          response = initial_search_response
+        else
+          response = @elastic_index.scroll(scroll_id, keep_alive)
+        end
+
+        @scroll_id = response['_scroll_id']
+        response
       end
 
-      def keep_alive
-        @options[:keep_alive] || (raise "Must provide a :keep_alive option")
+      def total_hits
+        initial_search_response['hits']['total']
       end
 
-      def requested_batch_size
-        @options[:batch_size]
+      def initial_search_response
+        @initial_search_response ||= begin
+          search_options = {size: batch_size, scroll: keep_alive}
+          elastic_query = @search.merge('sort' => '_doc')
+          @elastic_index.search(elastic_query, search_options)
+        end
       end
     end
 
@@ -101,9 +112,9 @@ module ElasticRecord
       end
 
       def delete_by_query(query)
-        scan_search = create_scan_search query
+        scroll_enumerator = build_scroll_enumerator search: query
 
-        scan_search.each_slice do |ids|
+        scroll_enumerator.each_slice do |ids|
           bulk do
             ids.each { |id| delete_document(id) }
           end
@@ -127,16 +138,8 @@ module ElasticRecord
         get "_explain", elastic_query
       end
 
-      def create_scan_search(elastic_query, options = {})
-        options[:batch_size] ||= 100
-        options[:keep_alive] ||= ElasticRecord::Config.scroll_keep_alive
-
-        search_options = {search_type: 'scan', size: options[:batch_size], scroll: options[:keep_alive]}
-        json = search(elastic_query, search_options)
-
-        ScanSearch.new(self, json['_scroll_id'], options).tap do |scan_search|
-          scan_search.total_hits = json['hits']['total']
-        end
+      def build_scroll_enumerator(search: nil, scroll_id: nil, batch_size: 100, keep_alive: ElasticRecord::Config.scroll_keep_alive)
+        ScrollEnumerator.new(self, search: search, scroll_id: scroll_id, batch_size: batch_size, keep_alive: keep_alive)
       end
 
       def scroll(scroll_id, scroll_keep_alive)

data/lib/elastic_record/index.rb

@@ -1,3 +1,4 @@
+require 'elastic_record/index/analyze'
 require 'elastic_record/index/configurator'
 require 'elastic_record/index/deferred'
 require 'elastic_record/index/documents'
@@ -7,12 +8,7 @@ require 'elastic_record/index/percolator'
 require 'elastic_record/index/settings'
 require 'elastic_record/index/warmer'
 
-require 'active_support/version'
-if ActiveSupport::VERSION::STRING < '4'
-  require 'active_support/core_ext/hash/deep_dup'
-else
-  require 'active_support/core_ext/object/deep_dup'
-end
+require 'active_support/core_ext/object/deep_dup'
 
 module ElasticRecord
   # ElasticRecord::Index provides access to elastic search's API. It is accessed with
@@ -37,6 +33,7 @@ module ElasticRecord
     include Manage
     include Mapping, Settings
     include Percolator, Warmer
+    include Analyze
     include Deferred
 
     attr_accessor :model

data/lib/elastic_record/relation/batches.rb

@@ -14,11 +14,11 @@ module ElasticRecord
       end
 
       def find_ids_in_batches(options = {}, &block)
-        create_scan_search(options).each_slice(&block)
+        build_scroll_enumerator(options).each_slice(&block)
       end
 
-      def create_scan_search(options)
-        elastic_index.create_scan_search(as_elastic, options)
+      def build_scroll_enumerator(options)
+        elastic_index.build_scroll_enumerator(search: as_elastic, **options)
       end
 
       def reindex

data/lib/elastic_record/relation/none.rb

@@ -14,7 +14,7 @@ module ElasticRecord
       end
 
       def as_elastic
-        Arelastic::Filters::Not.new(Arelastic::Queries::MatchAll.new).as_elastic
+        Arelastic::Queries::MatchAll.new.negate.as_elastic
       end
     end
   end

data/lib/elastic_record/relation/search_methods.rb

@@ -8,13 +8,13 @@ module ElasticRecord
 
         def not(*filters)
           add_filter_nodes_to_scope(filters) do |filter_node|
-            Arelastic::Filters::Not.new(filter_node)
+            Arelastic::Queries::Bool.new must_not: filter_node
           end
         end
 
         def nested(path, *filters)
           add_filter_nodes_to_scope(filters) do |filter_node|
-            Arelastic::Filters::Nested.new(path, filter_node)
+            Arelastic::Queries::Nested.new path, filter_node
           end
         end
 
@@ -120,8 +120,12 @@ module ElasticRecord
       end
 
       def search_type!(type)
-        self.search_type_value = type
-        self
+        if type == :count # TODO: Deprecate support
+          limit! 0
+        else
+          self.search_type_value = type
+          self
+        end
       end
 
       def search_type(type)
@@ -196,8 +200,9 @@ module ElasticRecord
         def build_query_and_filter(query, filters)
           query = build_query(query)
           filter = build_filter(filters)
+
           if filter
-            arelastic.query.filtered(filter: filter, query: query)
+            arelastic.query.bool(filter: filter, must: query)
           elsif query
             Arelastic::Searches::Query.new(query)
           else
@@ -219,7 +224,7 @@ module ElasticRecord
           if nodes.size == 1
             nodes.first
           elsif nodes.size > 1
-            Arelastic::Filters::And.new(nodes)
+            Arelastic::Queries::Bool.new(must: nodes)
           end
         end
 
@@ -228,7 +233,7 @@ module ElasticRecord
             if filter.is_a?(Arelastic::Nodes::Node)
               nodes << filter
             elsif filter.is_a?(ElasticRecord::Relation)
-              nodes << Arelastic::Filters::HasChild.new(filter.elastic_index.type, filter.as_elastic['query'])
+              nodes << Arelastic::Queries::HasChild.new(filter.elastic_index.type, filter.as_elastic['query'])
             else
               filter.each do |field, terms|
                 case terms

data/lib/elastic_record/relation.rb

@@ -75,20 +75,13 @@ module ElasticRecord
       def search_results
         @search_results ||= begin
           options = search_type_value ? {search_type: search_type_value} : {}
-          klass.elastic_index.search(as_elastic, options.update(fields: ''))
+
+          klass.elastic_index.search(as_elastic.update('_source' => false), options)
         end
       end
 
       def load_hits(ids)
         scope = select_values.any? ? klass.select(select_values) : klass
-        if defined?(ActiveRecord::Base) && klass < ActiveRecord::Base
-          case klass.connection.adapter_name
-          when /Mysql/
-            scope = scope.order("FIELD(#{connection.quote_column_name(primary_key)}, #{ids.join(',')})")
-          when /Pos/
-            scope = scope.order(ids.map { |id| "ID=#{connection.quote(id)} DESC" }.join(','))
-          end
-        end
         scope.find(ids)
       end
   end

data/lib/elastic_record.rb

@@ -1,5 +1,4 @@
 require 'arelastic'
-require 'active_support/core_ext/object/blank' # required because ActiveModel depends on this but does not require it
 require 'active_support/concern'
 require 'active_model'
 

data/test/dummy/.env.test

@@ -1 +1 @@
-DATABASE_URL=postgres://postgres:@localhost/elastic_record_test
+DATABASE_URL=postgres://localhost/elastic_record_test

data/test/dummy/app/models/widget.rb

@@ -9,7 +9,7 @@ class Widget
 
   self.elastic_index.mapping[:properties].update(
     'name' => {
-      type: 'multi_field',
+      type: 'string',
       fields: {
         name: {type: 'string', index: 'not_analyzed'},
         analyzed: {type: 'string', index: 'analyzed'}

data/test/elastic_record/index/analyze_test.rb

@@ -0,0 +1,12 @@
+require 'helper'
+
+class ElasticRecord::Index::PercolatorTest < MiniTest::Test
+  def test_analyze
+    tokens = Widget.elastic_index.analyze(
+      "analyzer" => "standard",
+      "text"     => "this is a test"
+    )
+
+    assert_equal ["this", "is", "a", "test"], tokens
+  end
+end

data/test/elastic_record/index/documents_test.rb

@@ -47,21 +47,20 @@ class ElasticRecord::Index::DocumentsTest < MiniTest::Test
     index.index_document('bob', name: 'bob')
     index.index_document('joe', name: 'joe')
 
-    index.delete_by_query('query' => {query_string: {query: 'name.analyzed:bob'}})
+    index.delete_by_query('query' => {query_string: {query: 'name:bob'}})
 
     refute index.record_exists?('bob')
     assert index.record_exists?('joe')
   end
 
-  def test_create_scan_search
+  def test_build_scroll_enumerator
     index.index_document('bob', name: 'bob')
     index.index_document('joe', name: 'joe')
 
-    scan_search = index.create_scan_search('query' => {query_string: {query: 'name.analyzed:bob'}})
+    scroll_enumerator = index.build_scroll_enumerator(search: {'query' => {query_string: {query: 'name:bob'}}})
 
-    assert_equal 1, scan_search.total_hits
-    refute_nil scan_search.scroll_id
-    assert_equal 1, scan_search.request_more_ids.size
+    assert_equal 1, scroll_enumerator.total_hits
+    assert_equal 1, scroll_enumerator.request_more_ids.size
   end
 
   def test_bulk_add

data/test/elastic_record/integration/active_record_test.rb

@@ -1,72 +1,21 @@
 require 'helper'
 
-require 'mysql2'
-require 'pg'
-
-module ElasticRecord
-  module ActiveRecordIntegration
-    def setup_project_database(config)
-      ActiveRecord::Tasks::DatabaseTasks.drop config
-      ActiveRecord::Tasks::DatabaseTasks.create config
-      ActiveRecord::Base.establish_connection config
-
-      ActiveRecord::Migration.suppress_messages do
-        ActiveRecord::Migration.create_table :projects do |t|
-          t.string :name, null: false
-        end
-      end
-
-      Project.elastic_index.create_and_deploy if Project.elastic_index.all_names.empty?
-    end
-
-    def test_ordering
-      poo_product = Project.create! name: "Poo"
-      bear_product = Project.create! name: "Bear"
-      Project.elastic_index.refresh
-
-      assert_equal [bear_product, poo_product], Project.elastic_relation.order(name: 'asc')
-      assert_equal [poo_product, bear_product], Project.elastic_relation.order(name: 'desc')
-    end
-
-    def test_update_callback
-      project = Project.create! name: "Ideas"
-      Project.elastic_index.refresh
-      project.update! name: 'Terrible Stuff'
-      Project.elastic_index.refresh
-
-      assert_equal [project], Project.elastic_relation.filter(name: 'Terrible Stuff')
-    end
+class ElasticRecord::ActiveRecordTest < MiniTest::Test
+  def test_ordering
+    poo_product = Project.create! name: "Poo"
+    bear_product = Project.create! name: "Bear"
+    Project.elastic_index.refresh
+
+    assert_equal [bear_product, poo_product], Project.elastic_relation.order(name: 'asc')
+    assert_equal [poo_product, bear_product], Project.elastic_relation.order(name: 'desc')
   end
-end
-
-class ElasticRecord::Mysql2Test < MiniTest::Test
-  include ElasticRecord::ActiveRecordIntegration
-
-  def setup
-    super
-
-    setup_project_database(
-      'adapter'   => 'mysql2',
-      'host'      => "localhost",
-      'database'  => 'elastic_record_subtest',
-      'username'  => 'root'
-    )
-  end
-end
-
-class ElasticRecord::PostgresqlTest < MiniTest::Test
-  include ElasticRecord::ActiveRecordIntegration
 
-  def setup
-    super
+  def test_update_callback
+    project = Project.create! name: "Ideas"
+    Project.elastic_index.refresh
+    project.update! name: 'Terrible Stuff'
+    Project.elastic_index.refresh
 
-    setup_project_database(
-      'adapter'   => 'postgresql',
-      'encoding'  => 'unicode',
-      'database'  => 'elastic_record_subtest',
-      'pool'      => 5,
-      'host'      => 'localhost',
-      'password'  => ''
-    )
+    assert_equal [project], Project.elastic_relation.filter(name: 'Terrible Stuff')
   end
 end

data/test/elastic_record/relation/batches_test.rb

@@ -67,7 +67,7 @@ class ElasticRecord::Relation::BatchesTest < MiniTest::Test
       Widget.elastic_index.bulk_add [
         Widget.new(id: 5, color: 'red'),
         Widget.new(id: 10, color: 'blue'),
-        Widget.new(id: 15, color: 'green'),
+        Widget.new(id: 15, color: 'green')
       ]
     end
 end

data/test/elastic_record/relation/delegation_test.rb

@@ -22,7 +22,7 @@ class ElasticRecord::Relation::DelegationTest < MiniTest::Test
 
     result = model.elastic_relation.filter('foo' => 'bar').do_it
 
-    expected = {"query" => {"filtered" => {"filter" => {"term" => {"foo" => "bar"}}}}}
+    expected = {"query" => {"bool" => {"filter" => {"term" => {"foo" => "bar"}}}}}
     assert_equal expected, result
   end
 end

data/test/elastic_record/relation/none_test.rb

@@ -9,7 +9,7 @@ class ElasticRecord::Relation::NoneTest < MiniTest::Test
     assert_equal [],    none.to_a
     assert_equal({},    none.aggregations)
 
-    expected_elastic = {"not" => {"match_all" => {}}}
+    expected_elastic = {"bool" => {"must_not" => {"match_all" => {}}}}
     assert_equal expected_elastic, none.as_elastic
   end
 end

data/test/elastic_record/relation/search_methods_test.rb

@@ -12,12 +12,14 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     relation.filter!(Widget.arelastic['faz'].in ['baz', 'fum'])
 
     expected = {
-      "filtered" => {
+      "bool" => {
         "filter" => {
-          "and" => [
-            {"term"   => {"foo" => "bar"}},
-            {"terms"  => {"faz" => ["baz", "fum"]}}
-          ]
+          "bool" => {
+            "must" => [
+              {"term"   => {"foo" => "bar"}},
+              {"terms"  => {"faz" => ["baz", "fum"]}}
+            ]
+          }
         }
       }
     }
@@ -29,7 +31,7 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     relation.filter!(Widget.arelastic['faz'].in 3..5)
 
     expected = {
-      "filtered" => {
+      "bool" => {
         "filter" => {
           "range" => {
             "faz" => {"gte"=>3, "lte"=>5}
@@ -45,7 +47,7 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     relation.filter!("prefix" => {"name" => "Jo"})
 
     expected = {
-      "filtered" => {
+      "bool" => {
         "filter" => {
           "prefix" => {
             "name" => "Jo"
@@ -61,11 +63,13 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     scope = relation.filter.not("prefix" => {"name" => "Jo"})
 
     expected = {
-      "filtered" => {
+      "bool" => {
         "filter" => {
-          "not" => {
-            "prefix" => {
-              "name" => "Jo"
+          "bool" => {
+            "must_not" => {
+              "prefix" => {
+                "name" => "Jo"
+              }
             }
           }
         }
@@ -79,11 +83,11 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     scope = relation.filter.nested("contacts", "prefix" => {"contacts.name" => "Jo"})
 
     expected = {
-      "filtered" => {
+      "bool" => {
         "filter" => {
           "nested" => {
             "path" => "contacts",
-            "filter" => {
+            "query" => {
               "prefix" => {
                 "contacts.name" => "Jo"
               }
@@ -109,8 +113,8 @@ class ElasticRecord::Relation::SearchMethodsTest < MiniTest::Test
     relation.filter!(Widget.arelastic['name'].prefix "mat")
 
     expected = {
-      "filtered" => {
-        "query" => {
+      "bool" => {
+        "must" => {
           "field" => {
             "name"=>"joe"
           },

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastic_record
 version: !ruby/object:Gem::Version
-  version: 3.1.4
+  version: 4.0.0
 platform: ruby
 authors:
 - Infogroup
@@ -9,28 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-07 00:00:00.000000000 Z
+date: 2017-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: arelastic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.1.2
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +59,7 @@ files:
 - lib/elastic_record/connection.rb
 - lib/elastic_record/errors.rb
 - lib/elastic_record/index.rb
+- lib/elastic_record/index/analyze.rb
 - lib/elastic_record/index/configurator.rb
 - lib/elastic_record/index/deferred.rb
 - lib/elastic_record/index/documents.rb
@@ -143,6 +138,7 @@ files:
 - test/elastic_record/callbacks_test.rb
 - test/elastic_record/config_test.rb
 - test/elastic_record/connection_test.rb
+- test/elastic_record/index/analyze_test.rb
 - test/elastic_record/index/configurator_test.rb
 - test/elastic_record/index/documents_test.rb
 - test/elastic_record/index/manage_test.rb