zelastic 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82591810835538bb7b3ede975d3731fc2b3300dfa9934f72f81ed323458c8145
-  data.tar.gz: c7631b4cb764dbfffdf4efd35296b9b9c5d5ce790c03464c7ba584ad060cae47
+  metadata.gz: 7b45ad7dd15f213cc571c3da4f021ff80129dcccb375ae98aedde7b22b759b05
+  data.tar.gz: '08328c62f6485a0b11300dc9c651ca3e8c99179c84ea2c3f5235a921ca676d9a'
 SHA512:
-  metadata.gz: 43647ec72340be7b8a4121eb7190d74844ad3284f38457dbe6948ddd81d4ed46fe9c2c7b9b8ae057db07b506efb5c2e7393f5eba43e209ecbf49dd58ec22b70c
-  data.tar.gz: 1b834cba1e6bffd46e108528b3d0eac68ca3766b86c5a28741db3c838b71d9c509cd1e42ac6dd82ff5f9ed7fc99a7cf1534507fc48b3fd1b388980ebbf0b33cd
+  metadata.gz: a3e38047bf9f47bc8e7353f8e52387a7bd6722e291f96291d09a50257cd28a43cf7fba2abcd9681abb3797aa5acce67f38b09cabb6ecbaf8e4638b2a7e6e21b6
+  data.tar.gz: 94572581d7fa5f6c6de6485cacd9232a310418379f981d12d0523183f6922f5bcbbeb290605e4452ccb51603f702aa5bf52a9540ed681002a1b2e389a2dff670

data/.circleci/config.yml

@@ -14,6 +14,7 @@ jobs:
       - image: "elasticsearch:<< parameters.elasticsearch_version >>"
         environment:
           "discovery.type": single-node
+          "xpack.security.enabled": false
     environment:
       ELASTICSEARCH_GEM_VERSION: "<< parameters.elasticsearch_gem_version >>"
     steps:
@@ -28,20 +29,20 @@ workflows:
   test:
     jobs:
       - test:
-          name: "test with elasticsearch v7
+          name: "test with elasticsearch v8
             and gem << matrix.elasticsearch_gem_version >>
             and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "7.17.5"
+          elasticsearch_version: "8.3.3"
           matrix:
             parameters:
               ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 7", "~> 5"]
+              elasticsearch_gem_version: ["~> 8", "~> 7"]
       - test:
-          name: "test with elasticsearch v5
-            and gem ~> 5
+          name: "test with elasticsearch v7
+            and gem << matrix.elasticsearch_gem_version >>
             and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "5.6-alpine"
-          elasticsearch_gem_version: "~> 5"
+          elasticsearch_version: "7.17.5"
           matrix:
             parameters:
               ruby_version: ["3.1", "2.7"]
+              elasticsearch_gem_version: ["~> 7"]

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/log/*.log
 /Gemfile.lock
 
 # rspec failure tracking

data/.rubocop.yml

@@ -8,3 +8,6 @@ inherit_mode:
 
 Metrics:
   Enabled: false
+
+Gemspec/RequireMFA:
+  Enabled: false

data/CHANGELOG.md

@@ -1,15 +1,47 @@
-v0.5.1 - Aug 8th, 2018
----
+# Changelog
 
-- Fix re-indexing log output upon initial index creation (#7)
+All notable changes to this project will be documented in this file.
 
-v0.5.0 - Aug 3rd, 2018
----
+The format is based on [Keep a Changelog],
+and this project adheres to [Semantic Versioning].
 
-- An _ESTIMATED_ % completion of ES re-indexing is logged, as the new index gets populated (#5)
+[Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
+[Semantic Versioning]: https://semver.org/spec/v2.0.0.html
 
+## How do I make a good changelog?
 
-v0.4.0 - May 29th, 2018
----
+### Guiding Principles
 
-- Warning rather than exception on ignorable version conflicts (#4)
+- Changelogs are for humans, not machines.
+- There should be an entry for every single version.
+- The same types of changes should be grouped.
+- Versions and sections should be linkable.
+- The latest version comes first.
+- The release date of each version is displayed.
+- Keep an `Unreleased` section at the top to track upcoming changes.
+
+### Types of changes
+
+- `Added` for new features.
+- `Changed` for changes in existing functionality.
+- `Deprecated` for soon-to-be removed features.
+- `Removed` for now removed features.
+- `Fixed` for any bug fixes.
+- `Security` in case of vulnerabilities.
+
+## [Unreleased] - XXXX-XX-XX
+
+## [0.8.0] - 2022-08-19
+### Added
+- Support for Ruby 3 (and keep support for 2.7).
+- Support for Elasticsearch v8 (and keep support for v7).
+- Support setting a logger in `Config`.
+- Support refresh on `IndexManager#populate_index`.
+- Support Proc in `Config#data_source` so it can be lazily evaluated.
+
+### Removed
+- Drop support for Ruby 2.6.
+- Drop support for Elasticsearch v5 and v6.
+
+[Unreleased]: https://github.com/carwow/zelastic/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/carwow/zelastic/releases/tag/v0.8.0

data/README.md

@@ -1,4 +1,7 @@
-# Zero-downtime indexing from ActiveRecord->Elasticsearch
+# Zelastic
+
+Zero-downtime Elasticsearch tooling for managing indices and indexing from
+ActiveRecord with PostgreSQL to Elasticsearch.
 
 ## Installation
 
@@ -10,13 +13,14 @@ gem 'zelastic'
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
     $ gem install zelastic
 
 ## Usage
+
 ### Setup
 
 For each ActiveRecord scope you want to index, you'll need a configuration:
@@ -46,7 +50,6 @@ You can also override some defaults, if you wish:
   here
 - `read_alias`: by default this is the table name of the `data_source`
 - `write_alias`: by default this is the `read_alias`, with `_write` appended
-- `type`: by default this is `read_alias.singularize`
 
 If you pass an array to as the `client` argument, all writes will be applied to every client in the
 array.

data/lib/zelastic/config.rb

@@ -2,31 +2,36 @@
 
 module Zelastic
   class Config
-    attr_reader :clients, :data_source
+    attr_reader :clients
 
     def initialize(
       client:,
       data_source:,
       mapping:,
+      logger: nil,
       **overrides,
       &index_data
     )
       @clients = Array(client)
       @data_source = data_source
       @mapping = mapping
-      @index_data = index_data
-      @_type = overrides.fetch(:type, true)
+      @logger = logger
       @overrides = overrides
-    end
-
-    def type?
-      @_type
+      @index_data = index_data
     end
 
     def index_data(model)
       @index_data.call(model)
     end
 
+    def data_source
+      if @data_source.respond_to? :call
+        @data_source.call
+      else
+        @data_source
+      end
+    end
+
     def read_alias
       @read_alias ||= overrides.fetch(:read_alias) { data_source.table_name }
     end
@@ -35,10 +40,6 @@ module Zelastic
       @write_alias ||= overrides.fetch(:write_alias) { [read_alias, 'write'].join('_') }
     end
 
-    def type
-      @type ||= overrides.fetch(:type, read_alias.singularize)
-    end
-
     def logger
       return Rails.logger if defined?(Rails)
 
@@ -48,7 +49,7 @@ module Zelastic
     def index_definition
       {
         settings: overrides.fetch(:index_settings, {}),
-        mappings: type ? { type => mapping } : mapping
+        mappings: mapping
       }
     end
 

data/lib/zelastic/index_manager.rb

@@ -16,12 +16,12 @@ module Zelastic
       client.indices.put_alias(index: index_name, name: config.write_alias)
     end
 
-    def populate_index(unique_name = nil, batch_size: 3000)
+    def populate_index(unique_name = nil, batch_size: 3000, refresh: false)
       index_name = index_name_from_unique(unique_name)
 
       config.data_source.find_in_batches(batch_size: batch_size).with_index do |batch, i|
         logger.info(populate_index_log(batch_size: batch_size, batch_number: i + 1))
-        indexer.index_batch(batch, client: client, index_name: index_name)
+        indexer.index_batch(batch, client: client, index_name: index_name, refresh: refresh)
       end
     end
 
@@ -116,18 +116,11 @@ module Zelastic
                  else
                    'First index'
                  end
-      "ES: (#{progress}) Indexing #{config.type} records"
+      "ES: (#{progress}) Indexing records"
     end
 
     def current_index_size
-      @current_index_size ||= client.count(**count_params)['count']
-    end
-
-    def count_params
-      {
-        index: config.read_alias,
-        type: config.type? ? config.type : nil
-      }.compact
+      @current_index_size ||= client.count(index: config.read_alias)['count']
     end
 
     def indexed_percent(batch_size, batch_number)

data/lib/zelastic/indexer.rb

@@ -43,10 +43,7 @@ module Zelastic
 
       execute_bulk do |index_name|
         ids.map do |id|
-          delete_params = { _index: index_name, _id: id }
-          delete_params[:_type] = config.type if config.type?
-
-          { delete: delete_params }
+          { delete: { _index: index_name, _id: id } }
         end
       end
     end
@@ -76,19 +73,14 @@ module Zelastic
     end
 
     def index_command(index:, version:, record:)
-      version_params =
-        if config.type?
-          { _version: version, _version_type: :external, _type: config.type }
-        else
-          { version: version, version_type: :external }
-        end
-
       {
         index: {
           _index: index,
           _id: record.id,
-          data: config.index_data(record)
-        }.merge(version_params)
+          data: config.index_data(record),
+          version: version,
+          version_type: :external
+        }
       }
     end
 
@@ -123,16 +115,11 @@ module Zelastic
     end
 
     def ignorable_error?(error)
-      # rubocop:disable Layout/LineLength
-      regexp =
-        if config.type?
-          /^\[#{config.type}\]\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
-        else
-          /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
-        end
-      # rubocop:enable Layout/LineLength
       error['type'] == 'version_conflict_engine_exception' &&
-        error['reason'] =~ regexp
+        error['reason'] =~ VERSION_CONFLICT_ERROR_REGEXP
     end
+
+    VERSION_CONFLICT_ERROR_REGEXP =
+      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/.freeze
   end
 end

data/lib/zelastic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zelastic
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/zelastic.gemspec

@@ -22,12 +22,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'elasticsearch', '>= 7', '< 9'
+
+  spec.add_dependency 'activerecord'
   spec.add_dependency 'activesupport'
 
-  spec.add_development_dependency 'activerecord'
   spec.add_development_dependency 'bundler', '~> 2'
   spec.add_development_dependency 'carwow_rubocop', '~> 4'
-  spec.add_development_dependency 'elasticsearch', '>= 5', '< 8'
   spec.add_development_dependency 'pry', '~> 0.14'
   spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'

metadata

@@ -1,17 +1,37 @@
 --- !ruby/object:Gem::Specification
 name: zelastic
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - carwow Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-16 00:00:00.000000000 Z
+date: 2022-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
+- !ruby/object:Gem::Dependency
+  name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -25,13 +45,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -66,26 +86,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4'
-- !ruby/object:Gem::Dependency
-  name: elasticsearch
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '8'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,7 @@ files:
 - lib/zelastic/index_manager.rb
 - lib/zelastic/indexer.rb
 - lib/zelastic/version.rb
+- log/.keep
 - zelastic.gemspec
 homepage: https://github.com/carwow/zelastic
 licenses: