zelastic 0.7.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82591810835538bb7b3ede975d3731fc2b3300dfa9934f72f81ed323458c8145
-  data.tar.gz: c7631b4cb764dbfffdf4efd35296b9b9c5d5ce790c03464c7ba584ad060cae47
+  metadata.gz: be9663f3819fc9057a838604106173e90e8b48219542a67eeac5f6493e696921
+  data.tar.gz: 67c3775b7911527728fc417c1978e223155d73bb7ade42629d8c595f2fcb0193
 SHA512:
-  metadata.gz: 43647ec72340be7b8a4121eb7190d74844ad3284f38457dbe6948ddd81d4ed46fe9c2c7b9b8ae057db07b506efb5c2e7393f5eba43e209ecbf49dd58ec22b70c
-  data.tar.gz: 1b834cba1e6bffd46e108528b3d0eac68ca3766b86c5a28741db3c838b71d9c509cd1e42ac6dd82ff5f9ed7fc99a7cf1534507fc48b3fd1b388980ebbf0b33cd
+  metadata.gz: 668853db1e94c690e8a86c32b1de997d11563d531cdef95b8f9dcab49df74ac412e36d2b79832ff12bcd82a8dc7740ca0afaa76acf82af2a83efb12e2d55f15d
+  data.tar.gz: e2e6f75c50b0c763654136fdf753e0734c0d79f5112f467c5f4fa6142cb332272595eecf1061120e82110bfd18ae08e0215fe205090cf35c23ea03b461fe6b98

data/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    name: ES ${{ matrix.elasticsearch }} / Gem ${{ matrix.elasticsearch_gem }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        elasticsearch: ["8.3.3"]
+        elasticsearch_gem: ["~> 8", "~> 7"]
+        include:
+          - elasticsearch: "7.17.5"
+            elasticsearch_gem: "~> 7"
+
+    services:
+      elasticsearch:
+        image: elasticsearch:${{ matrix.elasticsearch }}
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: "false"
+        ports:
+          - 9200:9200
+        options: --health-cmd "curl -s http://localhost:9200" --health-interval 5s --health-timeout 3s --health-retries 20
+
+    env:
+      ELASTICSEARCH_GEM_VERSION: ${{ matrix.elasticsearch_gem }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.4"
+          bundler-cache: true
+
+      - run: bin/rubocop
+      - run: bin/rspec

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+/log/*.log
 /Gemfile.lock
 
 # rspec failure tracking

data/.rubocop.yml

@@ -8,3 +8,6 @@ inherit_mode:
 
 Metrics:
   Enabled: false
+
+Gemspec/RequireMFA:
+  Enabled: false

data/CHANGELOG.md

@@ -1,15 +1,47 @@
-v0.5.1 - Aug 8th, 2018
----
+# Changelog
 
-- Fix re-indexing log output upon initial index creation (#7)
+All notable changes to this project will be documented in this file.
 
-v0.5.0 - Aug 3rd, 2018
----
+The format is based on [Keep a Changelog],
+and this project adheres to [Semantic Versioning].
 
-- An _ESTIMATED_ % completion of ES re-indexing is logged, as the new index gets populated (#5)
+[Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
+[Semantic Versioning]: https://semver.org/spec/v2.0.0.html
 
+## How do I make a good changelog?
 
-v0.4.0 - May 29th, 2018
----
+### Guiding Principles
 
-- Warning rather than exception on ignorable version conflicts (#4)
+- Changelogs are for humans, not machines.
+- There should be an entry for every single version.
+- The same types of changes should be grouped.
+- Versions and sections should be linkable.
+- The latest version comes first.
+- The release date of each version is displayed.
+- Keep an `Unreleased` section at the top to track upcoming changes.
+
+### Types of changes
+
+- `Added` for new features.
+- `Changed` for changes in existing functionality.
+- `Deprecated` for soon-to-be removed features.
+- `Removed` for now removed features.
+- `Fixed` for any bug fixes.
+- `Security` in case of vulnerabilities.
+
+## [Unreleased] - XXXX-XX-XX
+
+## [0.8.0] - 2022-08-19
+### Added
+- Support for Ruby 3 (and keep support for 2.7).
+- Support for Elasticsearch v8 (and keep support for v7).
+- Support setting a logger in `Config`.
+- Support refresh on `IndexManager#populate_index`.
+- Support Proc in `Config#data_source` so it can be lazily evaluated.
+
+### Removed
+- Drop support for Ruby 2.6.
+- Drop support for Elasticsearch v5 and v6.
+
+[Unreleased]: https://github.com/carwow/zelastic/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/carwow/zelastic/releases/tag/v0.8.0

data/CODEOWNERS

@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+* @carwow/platform @carwow/tech-leads

data/README.md

@@ -1,4 +1,7 @@
-# Zero-downtime indexing from ActiveRecord->Elasticsearch
+# Zelastic
+
+Zero-downtime Elasticsearch tooling for managing indices and indexing from
+ActiveRecord with PostgreSQL to Elasticsearch.
 
 ## Installation
 
@@ -10,13 +13,14 @@ gem 'zelastic'
 
 And then execute:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
     $ gem install zelastic
 
 ## Usage
+
 ### Setup
 
 For each ActiveRecord scope you want to index, you'll need a configuration:
@@ -46,7 +50,6 @@ You can also override some defaults, if you wish:
   here
 - `read_alias`: by default this is the table name of the `data_source`
 - `write_alias`: by default this is the `read_alias`, with `_write` appended
-- `type`: by default this is `read_alias.singularize`
 
 If you pass an array to as the `client` argument, all writes will be applied to every client in the
 array.
@@ -99,6 +102,45 @@ The `client` keyword argument to `Zelastic::IndexManager.new` is optional. It de
 passed to `Zelastic::Config.new`, if one client is passed, or the first client in the array, if an
 array is passed to `Zelastic::Config.new`.
 
+### Reindexing using Elasticsearch's Reindex API
+
+For large indices, you can use Elasticsearch's native reindex API instead of `populate_index`:
+
+```ruby
+index_manager = Zelastic::IndexManager.new(MyModelIndex)
+source_index = index_manager.current_read_index
+dest_index = index_manager.current_write_index  # or any target index name
+index_manager.reindex(source_index: source_index, dest_index: dest_index)
+```
+
+### Reindexing from a remote Elasticsearch instance
+
+If you have multiple Elasticsearch instances (e.g., primary and secondary), you can copy data
+from one to another using the `reindex_from_remote` method:
+
+```ruby
+# On the destination ES instance
+secondary_client = Elasticsearch::Client.new(url: ENV['SECONDARY_ELASTICSEARCH_URL'])
+index_manager = Zelastic::IndexManager.new(MyModelIndex, client: secondary_client)
+
+# Get the source index name from the primary instance
+primary_client = Elasticsearch::Client.new(url: ENV['ELASTICSEARCH_URL'])
+source_index = primary_client.indices.get_alias(name: MyModelIndex.read_alias).keys.first
+
+dest_index = index_manager.current_write_index
+index_manager.reindex_from_remote(
+  source_host: ENV['ELASTICSEARCH_URL'],
+  source_index: source_index,
+  dest_index: dest_index,
+  username: ENV['ELASTICSEARCH_USERNAME'],  # optional
+  password: ENV['ELASTICSEARCH_PASSWORD'],  # optional
+  wait_for_completion: false                # runs asynchronously by default
+)
+```
+
+Note: For remote reindexing to work, the destination Elasticsearch cluster must have the source
+host configured in `reindex.remote.whitelist` in `elasticsearch.yml`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/zelastic/config.rb

@@ -2,31 +2,36 @@
 
 module Zelastic
   class Config
-    attr_reader :clients, :data_source
+    attr_reader :clients
 
     def initialize(
       client:,
       data_source:,
       mapping:,
+      logger: nil,
       **overrides,
       &index_data
     )
       @clients = Array(client)
       @data_source = data_source
       @mapping = mapping
-      @index_data = index_data
-      @_type = overrides.fetch(:type, true)
+      @logger = logger
       @overrides = overrides
-    end
-
-    def type?
-      @_type
+      @index_data = index_data
     end
 
     def index_data(model)
       @index_data.call(model)
     end
 
+    def data_source
+      if @data_source.respond_to? :call
+        @data_source.call
+      else
+        @data_source
+      end
+    end
+
     def read_alias
       @read_alias ||= overrides.fetch(:read_alias) { data_source.table_name }
     end
@@ -35,10 +40,6 @@ module Zelastic
       @write_alias ||= overrides.fetch(:write_alias) { [read_alias, 'write'].join('_') }
     end
 
-    def type
-      @type ||= overrides.fetch(:type, read_alias.singularize)
-    end
-
     def logger
       return Rails.logger if defined?(Rails)
 
@@ -48,7 +49,7 @@ module Zelastic
     def index_definition
       {
         settings: overrides.fetch(:index_settings, {}),
-        mappings: type ? { type => mapping } : mapping
+        mappings: mapping
       }
     end
 

data/lib/zelastic/index_manager.rb

@@ -16,12 +16,12 @@ module Zelastic
       client.indices.put_alias(index: index_name, name: config.write_alias)
     end
 
-    def populate_index(unique_name = nil, batch_size: 3000)
+    def populate_index(unique_name = nil, batch_size: 3000, refresh: false)
       index_name = index_name_from_unique(unique_name)
 
       config.data_source.find_in_batches(batch_size: batch_size).with_index do |batch, i|
         logger.info(populate_index_log(batch_size: batch_size, batch_number: i + 1))
-        indexer.index_batch(batch, client: client, index_name: index_name)
+        indexer.index_batch(batch, client: client, index_name: index_name, refresh: refresh)
       end
     end
 
@@ -92,12 +92,50 @@ module Zelastic
       client.indices.delete(index: indices_to_delete)
     end
 
+    def reindex_from_local(source_index:, dest_index:, wait_for_completion: false)
+      logger.info("Reindexing from #{source_index} to #{dest_index}")
+      reindex(source: { index: source_index }, dest_index: dest_index,
+        wait_for_completion: wait_for_completion
+      )
+    end
+
+    def reindex_from_remote(source_host:, source_index:, dest_index:,
+      username: nil, password: nil, wait_for_completion: false)
+      logger.info("Reindexing from remote #{source_host}/#{source_index} to #{dest_index}")
+
+      remote = { host: source_host }
+      remote[:username] = username if username
+      remote[:password] = password if password
+
+      reindex(source: { remote: remote, index: source_index }, dest_index: dest_index,
+        wait_for_completion: wait_for_completion
+      )
+    end
+
+    def current_read_index
+      client.indices.get_alias(name: config.read_alias).keys.first
+    end
+
+    def current_write_index
+      client.indices.get_alias(name: config.write_alias).keys.first
+    end
+
     private
 
     attr_reader :config, :client
 
     def_delegators :config, :logger
 
+    def reindex(source:, dest_index:, wait_for_completion:)
+      client.reindex(
+        body: {
+          source: source,
+          dest: { index: dest_index }
+        },
+        wait_for_completion: wait_for_completion
+      )
+    end
+
     def indexer
       @indexer ||= Indexer.new(config)
     end
@@ -116,18 +154,11 @@ module Zelastic
                  else
                    'First index'
                  end
-      "ES: (#{progress}) Indexing #{config.type} records"
+      "ES: (#{progress}) Indexing records"
     end
 
     def current_index_size
-      @current_index_size ||= client.count(**count_params)['count']
-    end
-
-    def count_params
-      {
-        index: config.read_alias,
-        type: config.type? ? config.type : nil
-      }.compact
+      @current_index_size ||= client.count(index: config.read_alias)['count']
     end
 
     def indexed_percent(batch_size, batch_number)

data/lib/zelastic/indexer.rb

@@ -13,6 +13,10 @@ module Zelastic
 
     extend Forwardable
 
+    VERSION_CONFLICT_ERROR_REGEXP =
+      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
+    private_constant :VERSION_CONFLICT_ERROR_REGEXP
+
     def initialize(config)
       @config = config
     end
@@ -43,10 +47,7 @@ module Zelastic
 
       execute_bulk do |index_name|
         ids.map do |id|
-          delete_params = { _index: index_name, _id: id }
-          delete_params[:_type] = config.type if config.type?
-
-          { delete: delete_params }
+          { delete: { _index: index_name, _id: id } }
         end
       end
     end
@@ -76,19 +77,14 @@ module Zelastic
     end
 
     def index_command(index:, version:, record:)
-      version_params =
-        if config.type?
-          { _version: version, _version_type: :external, _type: config.type }
-        else
-          { version: version, version_type: :external }
-        end
-
       {
         index: {
           _index: index,
           _id: record.id,
-          data: config.index_data(record)
-        }.merge(version_params)
+          data: config.index_data(record),
+          version: version,
+          version_type: :external
+        }
       }
     end
 
@@ -123,16 +119,8 @@ module Zelastic
     end
 
     def ignorable_error?(error)
-      # rubocop:disable Layout/LineLength
-      regexp =
-        if config.type?
-          /^\[#{config.type}\]\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
-        else
-          /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
-        end
-      # rubocop:enable Layout/LineLength
       error['type'] == 'version_conflict_engine_exception' &&
-        error['reason'] =~ regexp
+        error['reason'] =~ VERSION_CONFLICT_ERROR_REGEXP
     end
   end
 end

data/lib/zelastic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zelastic
-  VERSION = '0.7.0'
+  VERSION = '0.9.0'
 end

data/zelastic.gemspec

@@ -22,12 +22,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'elasticsearch', '>= 7', '< 9'
+
+  spec.add_dependency 'activerecord'
   spec.add_dependency 'activesupport'
 
-  spec.add_development_dependency 'activerecord'
   spec.add_development_dependency 'bundler', '~> 2'
-  spec.add_development_dependency 'carwow_rubocop', '~> 4'
-  spec.add_development_dependency 'elasticsearch', '>= 5', '< 8'
+  spec.add_development_dependency 'carwow_rubocop', '~> 6'
   spec.add_development_dependency 'pry', '~> 0.14'
   spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'

metadata

@@ -1,29 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: zelastic
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - carwow Developers
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '9'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -31,7 +36,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -39,53 +44,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: carwow_rubocop
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: elasticsearch
+  name: carwow_rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '8'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '5'
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '8'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -135,12 +134,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md
+- CODEOWNERS
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -154,12 +154,12 @@ files:
 - lib/zelastic/index_manager.rb
 - lib/zelastic/indexer.rb
 - lib/zelastic/version.rb
+- log/.keep
 - zelastic.gemspec
 homepage: https://github.com/carwow/zelastic
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -174,8 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.
 test_files: []

data/.circleci/config.yml

@@ -1,47 +0,0 @@
-version: 2.1
-
-jobs:
-  test:
-    parameters:
-      ruby_version:
-        type: string
-      elasticsearch_version:
-        type: string
-      elasticsearch_gem_version:
-        type: string
-    docker:
-      - image: "cimg/ruby:<< parameters.ruby_version >>"
-      - image: "elasticsearch:<< parameters.elasticsearch_version >>"
-        environment:
-          "discovery.type": single-node
-    environment:
-      ELASTICSEARCH_GEM_VERSION: "<< parameters.elasticsearch_gem_version >>"
-    steps:
-      - checkout
-      - run: bin/setup
-      - run: bin/rubocop
-      - run: dockerize -wait http://localhost:9200 -timeout 1m
-      - run: bin/rspec
-
-workflows:
-  version: 2
-  test:
-    jobs:
-      - test:
-          name: "test with elasticsearch v7
-            and gem << matrix.elasticsearch_gem_version >>
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "7.17.5"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 7", "~> 5"]
-      - test:
-          name: "test with elasticsearch v5
-            and gem ~> 5
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "5.6-alpine"
-          elasticsearch_gem_version: "~> 5"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]