zelastic 0.3.4 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7b20a3c4e66ac288594edfc999f4499ce50c5f0c
-  data.tar.gz: caacc8874bc47d4fa812fdadc06c0795ad732c73
+SHA256:
+  metadata.gz: 2f54a69201807f3ba79aa72f7ea5b51c1adfebcdc06d4b51f4662f49c96bc0c7
+  data.tar.gz: '0480ef88c0c3f3f170c18712d2e9208b4fbcd0f7614abbb9cb9b89d2636443be'
 SHA512:
-  metadata.gz: 9a09f4b7bc7298a98d99a78345207b0a3cf9ff8855e8ca5b615d4f5f04e21c3495988e4e7a4100e8e1a125c85f5d64cc4b606c07b82ea222eff736d757c0c235
-  data.tar.gz: 8f2fef61bcb084ac942b23fecc6c9dff60fe198dca81387be8b6a0a9e880f2687fd1ae44224c389f4353c652a727d9ae31f6f551a06b0a34cc0888474937cae0
+  metadata.gz: 4ba1d3c04f92728c3a414a3b09551117c664eeb3e1ec9e4f395bb393b33cc8de55bb4239da17fd98be0a8171f3ae038a769f3a2b93147abf42fcac8a13ab0254
+  data.tar.gz: 954fe26c4ed02e57e00ae29c82f26b1eba33f6c637e520db7610a220a851b9d158f4052a9323c506804247b8f527115ce808401cb42c1a893ab93d64f9025a34

data/.circleci/config.yml

@@ -1,8 +1,14 @@
-ruby: &ruby
-  image: carwow/ruby-ci:2.4.3
+references:
+  ruby: &ruby
+    image: carwow/ruby-ci:2.6
+    environment:
+      ELASTICSEARCH_URL: http://localhost:9200
 
-elasticsearch_container: &elasticsearch_container
-  image: carwow/elasticsearch-ci:5.5.1
+  elasticsearch_container: &elasticsearch_container
+    image: carwow/elasticsearch-ci:5.5.1
+
+  elasticsearch_7_container: &elasticsearch_7_container
+    image: carwow/elasticsearch-ci:7.6.1
 
 version: 2
 jobs:
@@ -30,7 +36,7 @@ jobs:
       - *ruby
     steps:
       - checkout
-      - restore_cache: { keys: ['bundle-{{ checkout "Gemfile.lock" }}'] }
+      - restore_cache: { keys: ['bundle-{{ checksum "Gemfile.lock" }}'] }
       - run: bundle exec rubocop
 
   tests:
@@ -40,9 +46,35 @@ jobs:
       - *elasticsearch_container
     steps:
       - checkout
-      - restore_cache: { keys: ['bundle-{{ checkout "Gemfile.lock" }}'] }
+      - restore_cache: { keys: ['bundle-{{ checksum "Gemfile.lock" }}'] }
+      - run:
+          name: Wait for ES to be ready
+          command: |
+            until curl $ELASTICSEARCH_URL/_cat/health | egrep '(green|yellow)' 2>&1 > /dev/null; do
+              echo -n .
+              sleep 1
+            done
       - run: |
           bundle exec rspec --pattern "**/*_spec.rb" --format "progress"
+
+  tests_7:
+    working_directory: ~/zelastic
+    docker:
+      - *ruby
+      - *elasticsearch_7_container
+    steps:
+      - checkout
+      - restore_cache: { keys: ['bundle-{{ checksum "Gemfile.lock" }}'] }
+      - run:
+          name: Wait for ES to be ready
+          command: |
+            until curl $ELASTICSEARCH_URL/_cat/health | egrep '(green|yellow)' 2>&1 > /dev/null; do
+              echo -n .
+              sleep 1
+            done
+      - run: |
+          bundle exec rspec --pattern "**/*_spec.rb" --format "progress"
+
 workflows:
   version: 2
   build:
@@ -52,3 +84,5 @@ workflows:
           requires: [bundle]
       - tests:
           requires: [bundle]
+      - tests_7:
+          requires: [bundle]

data/.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: daily
+      time: '09:00'
+      timezone: 'Europe/London'
+    open-pull-requests-limit: 10

data/.rubocop.yml

@@ -1,3 +1,5 @@
+inherit_from: .rubocop_todo.yml
+
 AllCops:
   TargetRubyVersion: 2.4
 
@@ -5,6 +7,7 @@ AllCops:
 Metrics/BlockLength:
   Exclude:
     - '*.gemspec'
+    - 'spec/**/*'
 
 # Configuration parameters: CountComments.
 Metrics/MethodLength:
@@ -18,4 +21,4 @@ Style/Documentation:
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
 # URISchemes: http, https
 Metrics/LineLength:
-  Max: 102
+  Max: 100

data/.rubocop_todo.yml

@@ -0,0 +1,32 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-05-27 17:11:31 +0100 using RuboCop version 0.52.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 1
+Lint/ShadowingOuterLocalVariable:
+  Exclude:
+    - 'lib/zelastic/indexer.rb'
+
+# Offense count: 1
+Metrics/AbcSize:
+  Max: 17
+
+# Offense count: 1
+# Configuration parameters: CountComments, ExcludedMethods.
+Metrics/BlockLength:
+  Max: 83
+
+# Offense count: 4
+# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
+# URISchemes: http, https
+Metrics/LineLength:
+  Max: 146
+
+# Offense count: 1
+Naming/AccessorMethodName:
+  Exclude:
+    - 'spec/zelastic/indexer_spec.rb'

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+v0.5.1 - Aug 8th, 2018
+---
+
+- Fix re-indexing log output upon initial index creation (#7)
+
+v0.5.0 - Aug 3rd, 2018
+---
+
+- An _ESTIMATED_ % completion of ES re-indexing is logged, as the new index gets populated (#5)
+
+
+v0.4.0 - May 29th, 2018
+---
+
+- Warning rather than exception on ignorable version conflicts (#4)

data/Gemfile.lock

@@ -1,66 +1,94 @@
 PATH
   remote: .
   specs:
-    zelastic (0.3.4)
+    zelastic (0.6.2)
       activesupport
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (5.1.4)
-      activesupport (= 5.1.4)
-    activerecord (5.1.4)
-      activemodel (= 5.1.4)
-      activesupport (= 5.1.4)
-      arel (~> 8.0)
-    activesupport (5.1.4)
+    activemodel (6.1.3)
+      activesupport (= 6.1.3)
+    activerecord (6.1.3)
+      activemodel (= 6.1.3)
+      activesupport (= 6.1.3)
+    activesupport (6.1.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-    arel (8.0.0)
-    ast (2.3.0)
-    coderay (1.1.2)
-    concurrent-ruby (1.0.5)
-    diff-lcs (1.3)
-    i18n (0.9.1)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.1)
+    carwow_rubocop (3.3.0)
+      rubocop (>= 0.78)
+      rubocop-performance
+      rubocop-rspec
+    coderay (1.1.3)
+    concurrent-ruby (1.1.8)
+    diff-lcs (1.4.4)
+    elasticsearch (7.11.2)
+      elasticsearch-api (= 7.11.2)
+      elasticsearch-transport (= 7.11.2)
+    elasticsearch-api (7.11.2)
+      multi_json
+    elasticsearch-transport (7.11.2)
+      faraday (~> 1)
+      multi_json
+    faraday (1.3.0)
+      faraday-net_http (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    faraday-net_http (1.0.1)
+    i18n (1.8.9)
       concurrent-ruby (~> 1.0)
-    method_source (0.9.0)
-    minitest (5.11.1)
-    parallel (1.12.1)
-    parser (2.4.0.2)
-      ast (~> 2.3)
-    powerpack (0.1.1)
-    pry (0.11.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.9.0)
+    method_source (1.0.0)
+    minitest (5.14.3)
+    multi_json (1.15.0)
+    multipart-post (2.1.1)
+    parallel (1.19.1)
+    parser (2.7.1.3)
+      ast (~> 2.4.0)
+    pry (0.14.0)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     rainbow (3.0.0)
-    rake (12.3.0)
-    rspec (3.7.0)
-      rspec-core (~> 3.7.0)
-      rspec-expectations (~> 3.7.0)
-      rspec-mocks (~> 3.7.0)
-    rspec-core (3.7.1)
-      rspec-support (~> 3.7.0)
-    rspec-expectations (3.7.0)
+    rake (13.0.3)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-mocks (3.7.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.7.0)
-    rspec-support (3.7.0)
-    rubocop (0.52.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    rubocop (0.85.1)
       parallel (~> 1.10)
-      parser (>= 2.4.0.2, < 3.0)
-      powerpack (~> 0.1)
+      parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.0.3)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.9.0)
-    thread_safe (0.3.6)
-    tzinfo (1.2.4)
-      thread_safe (~> 0.1)
-    unicode-display_width (1.3.0)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
+    rubocop-performance (1.6.1)
+      rubocop (>= 0.71.0)
+    rubocop-rspec (1.40.0)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    ruby2_keywords (0.0.4)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -68,11 +96,12 @@ PLATFORMS
 DEPENDENCIES
   activerecord
   bundler
+  carwow_rubocop
+  elasticsearch
   pry
   rake
   rspec
-  rubocop
   zelastic!
 
 BUNDLED WITH
-   1.16.1
+   2.1.4

data/lib/zelastic/config.rb

@@ -15,9 +15,14 @@ module Zelastic
       @data_source = data_source
       @mapping = mapping
       @index_data = index_data
+      @_type = overrides.fetch(:type, true)
       @overrides = overrides
     end
 
+    def type?
+      @_type
+    end
+
     def index_data(model)
       @index_data.call(model)
     end
@@ -36,13 +41,14 @@ module Zelastic
 
     def logger
       return Rails.logger if defined?(Rails)
+
       @logger ||= Logger.new(STDOUT)
     end
 
     def index_definition
       {
         settings: overrides.fetch(:index_settings, {}),
-        mappings: { type => mapping }
+        mappings: type ? { type => mapping } : mapping
       }
     end
 

data/lib/zelastic/index_manager.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 module Zelastic
-  class IndexManager
+  # rubocop:disable Metrics/AbcSize
+  class IndexManager # rubocop:disable Metrics/ClassLength
     extend Forwardable
 
     def initialize(config, client: nil)
@@ -10,29 +11,21 @@ module Zelastic
     end
 
     def create_index(unique_name)
-      full_name = [config.read_alias, unique_name].join('_')
+      index_name = index_name_from_unique(unique_name)
 
-      client.indices.create(
-        index: full_name,
-        body: config.index_definition
-      )
-
-      client.indices.put_alias(index: full_name, name: config.write_alias)
+      client.indices.create(index: index_name, body: config.index_definition)
+      client.indices.put_alias(index: index_name, name: config.write_alias)
     end
 
     def populate_index(unique_name = nil, batch_size: 3000)
-      index_name = if unique_name
-                     [config.read_alias, unique_name].join('_')
-                   else
-                     config.write_alias
-                   end
+      index_name = index_name_from_unique(unique_name)
 
-      config.data_source.find_in_batches(batch_size: batch_size) do |batch|
+      config.data_source.find_in_batches(batch_size: batch_size).with_index do |batch, i|
+        logger.info(populate_index_log(batch_size: batch_size, batch_number: i + 1))
         indexer.index_batch(batch, client: client, index_name: index_name)
       end
     end
 
-    # rubocop:disable Metrics/AbcSize
     def switch_read_index(new_name)
       new_index = [config.read_alias, new_name].join('_')
 
@@ -91,10 +84,11 @@ module Zelastic
         logger.info('Nothing to do: no old indices')
         return
       end
-      logger.info("Deleting #{indices_to_delete.count} old indices: #{indices_to_delete.join(', ')}")
+      logger.info(
+        "Deleting #{indices_to_delete.count} old indices: #{indices_to_delete.join(', ')}"
+      )
       client.indices.delete(index: indices_to_delete)
     end
-    # rubocop:enable Metrics/AbcSize
 
     private
 
@@ -104,5 +98,42 @@ module Zelastic
     def indexer
       @indexer ||= Indexer.new(config)
     end
+
+    def index_name_from_unique(unique_name)
+      if unique_name
+        [config.read_alias, unique_name].join('_')
+      else
+        config.write_alias
+      end
+    end
+
+    def populate_index_log(batch_size:, batch_number:)
+      progress = if current_index_exists?
+                   "ESTIMATED: #{indexed_percent(batch_size, batch_number)}%"
+                 else
+                   'First index'
+                 end
+      "ES: (#{progress}) Indexing #{config.type} records"
+    end
+
+    def current_index_size
+      @current_index_size ||= client.count(**count_params)['count']
+    end
+
+    def count_params
+      {
+        index: config.read_alias,
+        type: config.type? ? config.type : nil
+      }.compact
+    end
+
+    def indexed_percent(batch_size, batch_number)
+      (batch_size * batch_number.to_f / current_index_size * 100).round(2)
+    end
+
+    def current_index_exists?
+      client.indices.exists?(index: config.read_alias)
+    end
   end
+  # rubocop:enable Metrics/AbcSize
 end

data/lib/zelastic/indexer.rb

@@ -5,10 +5,8 @@ module Zelastic
     class IndexingError < StandardError
       attr_reader :errors
 
-      def initialize(result)
-        @errors = result['items'].map do |item|
-          item['error'] || item.fetch('index', {})['error']
-        end.compact
+      def initialize(errors)
+        @errors = errors
         super("Errors indexing: #{errors.join(', ')}")
       end
     end
@@ -19,21 +17,19 @@ module Zelastic
       @config = config
     end
 
-    def index_batch(batch, client: nil, index_name: nil)
-      logger.info("ES: Indexing #{config.type} record")
-
+    def index_batch(batch, client: nil, index_name: nil, refresh: false)
       version = current_version
-      execute_bulk(client: client, index_name: index_name) do |index_name|
+      execute_bulk(client: client, index_name: index_name, refresh: refresh) do |index|
         batch.map do |record|
-          index_command(index: index_name, version: version, record: record)
+          index_command(index: index, version: version, record: record)
         end
       end
     end
 
-    def index_record(record)
+    def index_record(record, refresh: false)
       version = current_version
 
-      execute_bulk do |index_name|
+      execute_bulk(refresh: refresh) do |index_name|
         [index_command(index: index_name, version: version, record: record)]
       end
     end
@@ -47,13 +43,10 @@ module Zelastic
 
       execute_bulk do |index_name|
         ids.map do |id|
-          {
-            delete: {
-              _index: index_name,
-              _type: config.type,
-              _id: id
-            }
-          }
+          delete_params = { _index: index_name, _id: id }
+          delete_params[:_type] = config.type if config.type?
+
+          { delete: delete_params }
         end
       end
     end
@@ -82,19 +75,23 @@ module Zelastic
     end
 
     def index_command(index:, version:, record:)
+      version_params =
+        if config.type?
+          { _version: version, _version_type: :external, _type: config.type }
+        else
+          { version: version, version_type: :external }
+        end
+
       {
         index: {
           _index: index,
-          _type: config.type,
           _id: record.id,
-          _version: version,
-          _version_type: :external,
           data: config.index_data(record)
-        }
+        }.merge(version_params)
       }
     end
 
-    def execute_bulk(client: nil, index_name: nil)
+    def execute_bulk(client: nil, index_name: nil, refresh: false)
       clients = Array(client || config.clients)
 
       clients.map do |current_client|
@@ -102,10 +99,40 @@ module Zelastic
 
         commands = indices.flat_map { |index| yield(index) }
 
-        current_client.bulk(body: commands).tap do |result|
-          raise IndexingError, result if result['errors']
+        current_client.bulk(body: commands, refresh: refresh).tap do |result|
+          check_errors!(result)
         end
       end
     end
+
+    def check_errors!(result)
+      return false unless result['errors']
+
+      errors = result['items']
+               .map { |item| item['error'] || item.fetch('index', {})['error'] }
+               .compact
+
+      ignorable_errors, important_errors = errors
+                                           .partition { |error| ignorable_error?(error) }
+
+      logger.warn("Ignoring #{ignorable_errors.count} version conflicts") if ignorable_errors.any?
+
+      return unless important_errors.any?
+
+      raise IndexingError, important_errors
+    end
+
+    def ignorable_error?(error)
+      # rubocop:disable Layout/LineLength
+      regexp =
+        if config.type?
+          /^\[#{config.type}\]\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
+        else
+          /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
+        end
+      # rubocop:enable Layout/LineLength
+      error['type'] == 'version_conflict_engine_exception' &&
+        error['reason'] =~ regexp
+    end
   end
 end

data/lib/zelastic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zelastic
-  VERSION = '0.3.4'
+  VERSION = '0.6.2'
 end

data/zelastic.gemspec

@@ -1,7 +1,6 @@
-
 # frozen_string_literal: true
 
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'zelastic/version'
 
@@ -27,8 +26,9 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'activerecord'
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'carwow_rubocop'
+  spec.add_development_dependency 'elasticsearch'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
-  spec.add_development_dependency 'rubocop'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zelastic
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.6.2
 platform: ruby
 authors:
 - carwow Developers
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-05-22 00:00:00.000000000 Z
+date: 2021-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -53,7 +53,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: carwow_rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -116,9 +130,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
+- ".github/dependabot.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
+- ".rubocop_todo.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -136,7 +153,7 @@ homepage: https://github.com/carwow/zelastic
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -151,9 +168,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.
 test_files: []