zelastic 0.8.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b45ad7dd15f213cc571c3da4f021ff80129dcccb375ae98aedde7b22b759b05
-  data.tar.gz: '08328c62f6485a0b11300dc9c651ca3e8c99179c84ea2c3f5235a921ca676d9a'
+  metadata.gz: 57c75cdfd5b9131b9836f4fe62975e634b19e6fdaf203c43dfb04fa47cfab19e
+  data.tar.gz: a45bbd2a537e3363e3ab77690b5ad60a059f6e2526f7ce7410242e93c6b266bb
 SHA512:
-  metadata.gz: a3e38047bf9f47bc8e7353f8e52387a7bd6722e291f96291d09a50257cd28a43cf7fba2abcd9681abb3797aa5acce67f38b09cabb6ecbaf8e4638b2a7e6e21b6
-  data.tar.gz: 94572581d7fa5f6c6de6485cacd9232a310418379f981d12d0523183f6922f5bcbbeb290605e4452ccb51603f702aa5bf52a9540ed681002a1b2e389a2dff670
+  metadata.gz: b2090c1670134753ac7f3cab263c36d795f3b409b7e4a0d74cf9d27a57d1a80250ce7f2ec47dcacae564485a7645775010d57a55cd5ee61216af7584865b9b45
+  data.tar.gz: 05f156758b28a8c5198282567490ccfc93d3c499f785dd2eaf654b5243a3cada6e2f5362e7842bc9f32bda084f5d43a4fb775ecc91f90e3bbdbf8fab0680153b

data/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    name: ES ${{ matrix.elasticsearch }} / Gem ${{ matrix.elasticsearch_gem }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        elasticsearch: ["8.19.15"]
+        elasticsearch_gem: ["~> 8", "~> 7"]
+        include:
+          - elasticsearch: "7.17.28"
+            elasticsearch_gem: "~> 7"
+
+    services:
+      elasticsearch:
+        image: elasticsearch:${{ matrix.elasticsearch }}
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: "false"
+        ports:
+          - 9200:9200
+        options: --health-cmd "curl -s http://localhost:9200" --health-interval 5s --health-timeout 3s --health-retries 20
+
+    env:
+      ELASTICSEARCH_GEM_VERSION: ${{ matrix.elasticsearch_gem }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.4"
+          bundler-cache: true
+
+      - run: bin/rubocop
+      - run: bin/rspec

data/CHANGELOG.md

@@ -31,6 +31,13 @@ and this project adheres to [Semantic Versioning].
 
 ## [Unreleased] - XXXX-XX-XX
 
+## [0.9.1]
+### Added
+- `op_type` and `conflicts` kwargs on `IndexManager#reindex_from_local` and `#reindex_from_remote`. Defaults to `nil`, 
+  which omits them from the request body and preserves Elasticsearch defaults. Pass `op_type: 'create'` and
+  `conflicts: 'proceed'` for safe concurrent backfills - this avoids overwriting fresher live writes at the destination, and 
+  logs conflicts instead of aborting.
+
 ## [0.8.0] - 2022-08-19
 ### Added
 - Support for Ruby 3 (and keep support for 2.7).

data/CODEOWNERS

@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+* @carwow/platform @carwow/tech-leads

data/README.md

@@ -102,6 +102,45 @@ The `client` keyword argument to `Zelastic::IndexManager.new` is optional. It de
 passed to `Zelastic::Config.new`, if one client is passed, or the first client in the array, if an
 array is passed to `Zelastic::Config.new`.
 
+### Reindexing using Elasticsearch's Reindex API
+
+For large indices, you can use Elasticsearch's native reindex API instead of `populate_index`:
+
+```ruby
+index_manager = Zelastic::IndexManager.new(MyModelIndex)
+source_index = index_manager.current_read_index
+dest_index = index_manager.current_write_index  # or any target index name
+index_manager.reindex(source_index: source_index, dest_index: dest_index)
+```
+
+### Reindexing from a remote Elasticsearch instance
+
+If you have multiple Elasticsearch instances (e.g., primary and secondary), you can copy data
+from one to another using the `reindex_from_remote` method:
+
+```ruby
+# On the destination ES instance
+secondary_client = Elasticsearch::Client.new(url: ENV['SECONDARY_ELASTICSEARCH_URL'])
+index_manager = Zelastic::IndexManager.new(MyModelIndex, client: secondary_client)
+
+# Get the source index name from the primary instance
+primary_client = Elasticsearch::Client.new(url: ENV['ELASTICSEARCH_URL'])
+source_index = primary_client.indices.get_alias(name: MyModelIndex.read_alias).keys.first
+
+dest_index = index_manager.current_write_index
+index_manager.reindex_from_remote(
+  source_host: ENV['ELASTICSEARCH_URL'],
+  source_index: source_index,
+  dest_index: dest_index,
+  username: ENV['ELASTICSEARCH_USERNAME'],  # optional
+  password: ENV['ELASTICSEARCH_PASSWORD'],  # optional
+  wait_for_completion: false                # runs asynchronously by default
+)
+```
+
+Note: For remote reindexing to work, the destination Elasticsearch cluster must have the source
+host configured in `reindex.remote.whitelist` in `elasticsearch.yml`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/zelastic/index_manager.rb

@@ -19,9 +19,9 @@ module Zelastic
     def populate_index(unique_name = nil, batch_size: 3000, refresh: false)
       index_name = index_name_from_unique(unique_name)
 
-      config.data_source.find_in_batches(batch_size: batch_size).with_index do |batch, i|
-        logger.info(populate_index_log(batch_size: batch_size, batch_number: i + 1))
-        indexer.index_batch(batch, client: client, index_name: index_name, refresh: refresh)
+      config.data_source.find_in_batches(batch_size:).with_index do |batch, i|
+        logger.info(populate_index_log(batch_size:, batch_number: i + 1))
+        indexer.index_batch(batch, client:, index_name:, refresh:)
       end
     end
 
@@ -64,9 +64,9 @@ module Zelastic
                  )
 
       actions = other_write_indices.map do |index|
-        { remove: { index: index, alias: config.write_alias } }
+        { remove: { index:, alias: config.write_alias } }
       end
-      client.indices.update_aliases(body: { actions: actions })
+      client.indices.update_aliases(body: { actions: })
     end
 
     def cleanup_old_indices
@@ -92,12 +92,65 @@ module Zelastic
       client.indices.delete(index: indices_to_delete)
     end
 
+    def reindex_from_local(source_index:, dest_index:, wait_for_completion: false, op_type: nil, conflicts: nil)
+      logger.info("Reindexing from #{source_index} to #{dest_index}")
+      reindex(
+        source: { index: source_index }, dest_index:,
+        wait_for_completion:,
+        op_type:,
+        conflicts:
+      )
+    end
+
+    def reindex_from_remote(
+      source_host:,
+      source_index:,
+      dest_index:,
+      username: nil,
+      password: nil,
+      wait_for_completion: false,
+      op_type: nil,
+      conflicts: nil
+    )
+      logger.info("Reindexing from remote #{source_host}/#{source_index} to #{dest_index}")
+
+      remote = { host: source_host }
+      remote[:username] = username if username
+      remote[:password] = password if password
+
+      reindex(
+        source: { remote:, index: source_index },
+        dest_index:,
+        wait_for_completion:,
+        op_type:,
+        conflicts:
+      )
+    end
+
+    def current_read_index
+      client.indices.get_alias(name: config.read_alias).keys.first
+    end
+
+    def current_write_index
+      client.indices.get_alias(name: config.write_alias).keys.first
+    end
+
     private
 
     attr_reader :config, :client
 
     def_delegators :config, :logger
 
+    def reindex(source:, dest_index:, wait_for_completion:, op_type: nil, conflicts: nil)
+      dest = { index: dest_index }
+      dest[:op_type] = op_type if op_type
+
+      body = { source:, dest: }
+      body[:conflicts] = conflicts if conflicts
+
+      client.reindex(body:, wait_for_completion:)
+    end
+
     def indexer
       @indexer ||= Indexer.new(config)
     end

data/lib/zelastic/indexer.rb

@@ -13,15 +13,19 @@ module Zelastic
 
     extend Forwardable
 
+    VERSION_CONFLICT_ERROR_REGEXP =
+      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
+    private_constant :VERSION_CONFLICT_ERROR_REGEXP
+
     def initialize(config)
       @config = config
     end
 
     def index_batch(batch, client: nil, index_name: nil, refresh: false)
       version = current_version
-      execute_bulk(client: client, index_name: index_name, refresh: refresh) do |index|
+      execute_bulk(client:, index_name:, refresh:) do |index|
         batch.map do |record|
-          index_command(index: index, version: version, record: record)
+          index_command(index:, version:, record:)
         end
       end
     end
@@ -29,8 +33,8 @@ module Zelastic
     def index_record(record, refresh: false)
       version = current_version
 
-      execute_bulk(refresh: refresh) do |index_name|
-        [index_command(index: index_name, version: version, record: record)]
+      execute_bulk(refresh:) do |index_name|
+        [index_command(index: index_name, version:, record:)]
       end
     end
 
@@ -52,7 +56,7 @@ module Zelastic
       logger.info('ES: Deleting batch records')
 
       config.clients.map do |client|
-        client.delete_by_query(index: config.write_alias, body: { query: query })
+        client.delete_by_query(index: config.write_alias, body: { query: })
       end
     end
 
@@ -78,7 +82,7 @@ module Zelastic
           _index: index,
           _id: record.id,
           data: config.index_data(record),
-          version: version,
+          version:,
           version_type: :external
         }
       }
@@ -92,7 +96,7 @@ module Zelastic
 
         commands = indices.flat_map(&block)
 
-        current_client.bulk(body: commands, refresh: refresh).tap do |result|
+        current_client.bulk(body: commands, refresh:).tap do |result|
           check_errors!(result)
         end
       end
@@ -118,8 +122,5 @@ module Zelastic
       error['type'] == 'version_conflict_engine_exception' &&
         error['reason'] =~ VERSION_CONFLICT_ERROR_REGEXP
     end
-
-    VERSION_CONFLICT_ERROR_REGEXP =
-      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/.freeze
   end
 end

data/lib/zelastic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zelastic
-  VERSION = '0.8.0'
+  VERSION = '0.9.1'
 end

data/zelastic.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
 
   spec.add_development_dependency 'bundler', '~> 2'
-  spec.add_development_dependency 'carwow_rubocop', '~> 4'
+  spec.add_development_dependency 'carwow_rubocop', '~> 6'
   spec.add_development_dependency 'pry', '~> 0.14'
   spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: zelastic
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.1
 platform: ruby
 authors:
 - carwow Developers
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-19 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -78,14 +77,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -135,12 +134,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md
+- CODEOWNERS
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -160,7 +160,6 @@ homepage: https://github.com/carwow/zelastic
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -175,8 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.
 test_files: []

data/.circleci/config.yml

@@ -1,48 +0,0 @@
-version: 2.1
-
-jobs:
-  test:
-    parameters:
-      ruby_version:
-        type: string
-      elasticsearch_version:
-        type: string
-      elasticsearch_gem_version:
-        type: string
-    docker:
-      - image: "cimg/ruby:<< parameters.ruby_version >>"
-      - image: "elasticsearch:<< parameters.elasticsearch_version >>"
-        environment:
-          "discovery.type": single-node
-          "xpack.security.enabled": false
-    environment:
-      ELASTICSEARCH_GEM_VERSION: "<< parameters.elasticsearch_gem_version >>"
-    steps:
-      - checkout
-      - run: bin/setup
-      - run: bin/rubocop
-      - run: dockerize -wait http://localhost:9200 -timeout 1m
-      - run: bin/rspec
-
-workflows:
-  version: 2
-  test:
-    jobs:
-      - test:
-          name: "test with elasticsearch v8
-            and gem << matrix.elasticsearch_gem_version >>
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "8.3.3"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 8", "~> 7"]
-      - test:
-          name: "test with elasticsearch v7
-            and gem << matrix.elasticsearch_gem_version >>
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "7.17.5"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 7"]