sequel-elasticsearch 0.4.7 → 0.4.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f6e85a7a9a92736d74000717cf12de5fb7b42f20e839a0d57611e79ca014f52
-  data.tar.gz: 1e049bd725b6a164747e1982d119a9adc214788dc5a1624fafcdfcd612e7d767
+  metadata.gz: 37a0e42fb1cc78e626eeb7b83b9054bd932ffe60de8bd02cbbb99f8d2997c190
+  data.tar.gz: a1cd9bc96071fa402aeb274269f8f9ac5aacc122153e5f6660d1317ffe0d88ec
 SHA512:
-  metadata.gz: a72f7253761846209b0b86f704cb44e57be9d0d2561ab78b92bdd6912799ecc8e424c4779c252bf67da61aa62003c1f5bc9dcf0043ca1e3fee618ab3de9b89b3
-  data.tar.gz: 14de18c7db260ad3615a732de6338f7402aaedd34721e3826ba4cb3f7faa052e615080fcafef2e5155c2977cfd44a3bfaac5628268a07b9784446e624c40185a
+  metadata.gz: bed637128c79e6db4afd8feaba9d24bcd5c09386e4bed7d0a94d31e61a7a6319d0b0c51e793597b0a00857dd4663ef689815e083f4026bbfa723aa32c834bc7a
+  data.tar.gz: f6edecdb9f05ad6cfa920e388c3c58551370259c3c4700a31f4e9fe4ac7d4eef41fc44394ed9157cccc3fbc14189bdc87df71be9be9ec157f12be2c3f70a0d6f

data/.travis.yml

@@ -1,22 +1,26 @@
-sudo: false
+os:
+  - linux
+dist: xenial
+services:
+  - elasticsearch
 language: ruby
 rvm:
-- 2.3
 - 2.4
 - 2.5
 - 2.6
+- 2.7
 gemfile: Gemfile.ci
 env:
   global:
   - CC_TEST_REPORTER_ID=f35fd15664b071c621a239733eb3b063caf333ac7e795d3a20690942f64caf62
 before_install:
-- gem install bundler -v 1.13.6
+- gem install bundler
 before_script:
-- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64
-  > ./cc-test-reporter
+- curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
 - chmod +x ./cc-test-reporter
 - "./cc-test-reporter before-build"
 - export TZ=Africa/Johannesburg
+- sleep 10
 after_script:
 - "./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT"
 deploy:

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in sequel-elasticsearch.gemspec

data/Gemfile.ci

@@ -4,3 +4,4 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'sqlite3'
+gem 'codeclimate'

data/README.md

@@ -62,6 +62,12 @@ Sequel::Model.plugin :elasticsearch,
 
 And that's it! Just transact as you normally would, and your records will be created and updated in the Elasticsearch cluster.
 
+### Indexing
+
+Ensure that you create the [index mappings](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html) for your data before using this plugin, otherwise you might get some weird results.
+
+The records will by default be indexed using the `values` call of the model. Should you need to customize what's indexed, you can define a `indexed_values` method (or `as_indexed_json` method if you prefer the Rails way).
+
 ### Searching
 
 Your model is now searchable through Elasticsearch. Just pass down a string that's parsable as a [query string query](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html).
@@ -105,6 +111,22 @@ while (scroll = Document.es(scroll, scroll: '1m')) && scroll.empty? == false do
 end
 ```
 
+### Import
+
+You can import the whole dataset, or specify a dataset to be imported. This will create a new, timestamped index for your dataset, and import all the records from that dataset into the index. An alias will be created (or updated) to point to the newly created index.
+
+```ruby
+Document.import! # Import all the Document records. Use the default settings.
+
+Document.import!(dataset: Document.where(active: true)) # Import all the active Document records
+
+Document.import!(
+    index: 'active-documents', # Use the active-documents index
+    dataset: Document.where(active: true), # Only index active documents
+    batch_size: 20 # Send documents to Elasticsearch in batches of 20 records
+)
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 libdir = File.expand_path(File.dirname(__FILE__) + '/lib')
 $LOAD_PATH.unshift(libdir) unless $LOAD_PATH.include?(libdir)
 

data/lib/sequel/plugins/elasticsearch.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'elasticsearch'
 require 'sequel/plugins/elasticsearch/result'
 
@@ -16,17 +18,14 @@ module Sequel
     module Elasticsearch
       # Apply the plugin to the specified model
       def self.apply(model, _opts = OPTS)
-        model.instance_variable_set(:@elasticsearch_opts, {})
-        model.instance_variable_set(:@elasticsearch_index, nil)
-        model.instance_variable_set(:@elasticsearch_type, '_doc')
         model
       end
 
       # Configure the plugin
       def self.configure(model, opts = OPTS)
         model.elasticsearch_opts = opts[:elasticsearch] || {}
-        model.elasticsearch_index = (opts[:index] || model.table_name).to_sym
-        model.elasticsearch_type = (opts[:type] || :_doc).to_sym
+        model.elasticsearch_index = (opts[:index] || model.table_name.to_s.downcase).to_sym
+        model.elasticsearch_type = opts[:type]&.to_sym
         model
       end
 
@@ -59,7 +58,7 @@ module Sequel
           scroll_id = scroll_id.scroll_id if scroll_id.is_a? Result
           return nil unless scroll_id
 
-          Result.new es_client.scroll(scroll_id: scroll_id, scroll: duration), self
+          Result.new es_client.scroll(body: scroll_id, scroll: duration), self
         end
 
         # Execute a search or a scroll on the Model's Elasticsearch index.
@@ -88,27 +87,30 @@ module Sequel
         # Use the +reindex!+ method to create a completely new index and alias.
         def import!(index: nil, dataset: nil, batch_size: 100)
           dataset ||= self.dataset
-          index_name = index || last_index
+          index_name = index || last_index || elasticsearch_index
 
           # Index all the documents
           body = []
-          dataset.all.each do |row|
-            body << {
-              update: {
-                _index: index_name,
-                _type: elasticsearch_type,
-                _id: row.document_id,
-                data: { doc: row.indexed_values, doc_as_upsert: true }
-              }
-            }
-            print '.'
-            next unless body.count >= batch_size
+          dataset.each_page(batch_size) do |ds|
+            body = []
+            ds.all.each do |row|
+              print '.'
+              body << { update: import_object(index_name, row) }
+            end
             puts '/'
-
             es_client.bulk body: body
-            body = []
+            body = nil
           end
-          es_client.bulk body: body if body.count.positive?
+        end
+
+        def import_object(idx, row)
+          val = {
+            _index: idx,
+            _id: row.document_id,
+            data: { doc: row.as_indexed_json, doc_as_upsert: true }
+          }
+          val[:_type] = elasticsearch_type if elasticsearch_type
+          val
         end
 
         # Creates a new index in Elasticsearch from the specified dataset, as
@@ -123,6 +125,7 @@ module Sequel
           alias_index(index_name)
         end
 
+        # Remove previous aliases and point the `elasticsearch_index` to the new index.
         def alias_index(new_index)
           es_client.indices.update_aliases body: {
             actions: [
@@ -135,24 +138,30 @@ module Sequel
         # Find the last created index that matches the specified index name.
         def last_index
           es_client.indices.get_alias(name: elasticsearch_index)&.keys&.sort&.first
+        rescue ::Elasticsearch::Transport::Transport::Errors::NotFound
+          nil
         end
 
-        # Generate a timestamped index name according to the environment.
-        # This will use the +APP_ENV+ ENV variable and a timestamp to construct
-        # index names like this:
-        #
-        #    base-name-staging-20191004.123456 # This is a staging index
-        #    base-name-20191005.171213 # This is a production index
+        # Generate a timestamped index name.
+        # This will use the current timestamp to construct index names like this:
         #
+        #    base-name-20191004.123456
         def timestamped_index
-          time_str = Time.now.strftime('%Y%m%d.%H%M%S')
-          env_str = ENV['APP_ENV'] == 'production' ? nil : ENV['APP_ENV']
-          [elasticsearch_index, env_str, time_str].compact.join('-')
+          time_str = Time.now.strftime('%Y%m%d.%H%M%S') # TODO: Make the format configurable
+          "#{elasticsearch_index}-#{time_str}".to_sym
         end
       end
 
       # The instance methods that will be added to the Sequel::Model
       module InstanceMethods
+        def elasticsearch_index
+          self.class.elasticsearch_index
+        end
+
+        def elasticsearch_type
+          self.class.elasticsearch_type
+        end
+
         # Sequel::Model after_create hook to add the new record to the Elasticsearch index.
         # It's "safe" in that it won't raise an error if it fails.
         def after_create
@@ -179,6 +188,8 @@ module Sequel
           self.class.es_client
         end
 
+        # Mirror the Elasticsearch Rails plugin. Use this to override what data
+        # is sent to Elasticsearch
         def as_indexed_json
           indexed_values
         end
@@ -192,7 +203,7 @@ module Sequel
         # Create or update the document on the Elasticsearch cluster.
         def index_document(opts = {})
           params = document_path(opts)
-          params[:body] = indexed_values
+          params[:body] = as_indexed_json
           es_client.index params
         end
 
@@ -210,8 +221,8 @@ module Sequel
         # Determine the complete path to a document (/index/type/id) in the Elasticsearch cluster.
         def document_path(opts = {})
           {
-            index: opts.delete(:index) || self.class.elasticsearch_index,
-            type: opts.delete(:type) || self.class.elasticsearch_type,
+            index: opts.delete(:index) || elasticsearch_index,
+            type: opts.delete(:type) || elasticsearch_type,
             id: opts.delete(:id) || document_id
           }
         end

data/lib/sequel/plugins/elasticsearch/result.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Sequel
   module Plugins
     module Elasticsearch
@@ -42,6 +44,11 @@ module Sequel
           result['hits']['hits'].each { |h| yield h }
         end
 
+        # Send back the complete result set
+        def all
+          result['hits']['hits']
+        end
+
         # Send all undefined methods to the +result['hits']['hits']+ array.
         def method_missing(meth, *args, &block)
           respond_to_missing?(meth) ? result['hits']['hits'].send(meth, *args, &block) : super

data/lib/sequel/plugins/elasticsearch/version.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module Sequel
   # The Sequel::Elasticsearch model plugin
   #
   # See https://jrgns.github.io/sequel-elasticsearch
   module Elasticsearch
     # The Gem's version.
-    VERSION = '0.4.7'.freeze
+    VERSION = '0.4.13'
   end
 end

data/sequel-elasticsearch.gemspec

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'sequel/plugins/elasticsearch/version'
 
@@ -25,9 +26,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sequel', '>= 4.0'
 
   spec.add_development_dependency 'bundler', '>= 1.13'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '~> 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.52'
-  spec.add_development_dependency 'simplecov', '~> 0.15'
+  spec.add_development_dependency 'rubocop', '>= 0.52'
+  spec.add_development_dependency 'rubocop-rspec', '>= 1.37'
+  spec.add_development_dependency 'simplecov', '~> 0.15', '< 0.18'
+  spec.add_development_dependency 'sqlite3', '~> 1.4'
+  spec.add_development_dependency 'timecop', '>= 0.9'
   spec.add_development_dependency 'webmock', '~> 3.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sequel-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.4.13
 platform: ruby
 authors:
 - Jurgens du Toit
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-10-30 00:00:00.000000000 Z
+date: 2020-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +84,30 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.52'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.52'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.37'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.37'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -101,6 +115,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -108,6 +125,37 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.18'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -163,8 +211,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: A plugin for the Sequel gem to sync data to Elasticsearch.