zelastic 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b45ad7dd15f213cc571c3da4f021ff80129dcccb375ae98aedde7b22b759b05
-  data.tar.gz: '08328c62f6485a0b11300dc9c651ca3e8c99179c84ea2c3f5235a921ca676d9a'
+  metadata.gz: be9663f3819fc9057a838604106173e90e8b48219542a67eeac5f6493e696921
+  data.tar.gz: 67c3775b7911527728fc417c1978e223155d73bb7ade42629d8c595f2fcb0193
 SHA512:
-  metadata.gz: a3e38047bf9f47bc8e7353f8e52387a7bd6722e291f96291d09a50257cd28a43cf7fba2abcd9681abb3797aa5acce67f38b09cabb6ecbaf8e4638b2a7e6e21b6
-  data.tar.gz: 94572581d7fa5f6c6de6485cacd9232a310418379f981d12d0523183f6922f5bcbbeb290605e4452ccb51603f702aa5bf52a9540ed681002a1b2e389a2dff670
+  metadata.gz: 668853db1e94c690e8a86c32b1de997d11563d531cdef95b8f9dcab49df74ac412e36d2b79832ff12bcd82a8dc7740ca0afaa76acf82af2a83efb12e2d55f15d
+  data.tar.gz: e2e6f75c50b0c763654136fdf753e0734c0d79f5112f467c5f4fa6142cb332272595eecf1061120e82110bfd18ae08e0215fe205090cf35c23ea03b461fe6b98

data/.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    name: ES ${{ matrix.elasticsearch }} / Gem ${{ matrix.elasticsearch_gem }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        elasticsearch: ["8.3.3"]
+        elasticsearch_gem: ["~> 8", "~> 7"]
+        include:
+          - elasticsearch: "7.17.5"
+            elasticsearch_gem: "~> 7"
+
+    services:
+      elasticsearch:
+        image: elasticsearch:${{ matrix.elasticsearch }}
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: "false"
+        ports:
+          - 9200:9200
+        options: --health-cmd "curl -s http://localhost:9200" --health-interval 5s --health-timeout 3s --health-retries 20
+
+    env:
+      ELASTICSEARCH_GEM_VERSION: ${{ matrix.elasticsearch_gem }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.4"
+          bundler-cache: true
+
+      - run: bin/rubocop
+      - run: bin/rspec

data/CODEOWNERS

@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+* @carwow/platform @carwow/tech-leads

data/README.md

@@ -102,6 +102,45 @@ The `client` keyword argument to `Zelastic::IndexManager.new` is optional. It de
 passed to `Zelastic::Config.new`, if one client is passed, or the first client in the array, if an
 array is passed to `Zelastic::Config.new`.
 
+### Reindexing using Elasticsearch's Reindex API
+
+For large indices, you can use Elasticsearch's native reindex API instead of `populate_index`:
+
+```ruby
+index_manager = Zelastic::IndexManager.new(MyModelIndex)
+source_index = index_manager.current_read_index
+dest_index = index_manager.current_write_index  # or any target index name
+index_manager.reindex(source_index: source_index, dest_index: dest_index)
+```
+
+### Reindexing from a remote Elasticsearch instance
+
+If you have multiple Elasticsearch instances (e.g., primary and secondary), you can copy data
+from one to another using the `reindex_from_remote` method:
+
+```ruby
+# On the destination ES instance
+secondary_client = Elasticsearch::Client.new(url: ENV['SECONDARY_ELASTICSEARCH_URL'])
+index_manager = Zelastic::IndexManager.new(MyModelIndex, client: secondary_client)
+
+# Get the source index name from the primary instance
+primary_client = Elasticsearch::Client.new(url: ENV['ELASTICSEARCH_URL'])
+source_index = primary_client.indices.get_alias(name: MyModelIndex.read_alias).keys.first
+
+dest_index = index_manager.current_write_index
+index_manager.reindex_from_remote(
+  source_host: ENV['ELASTICSEARCH_URL'],
+  source_index: source_index,
+  dest_index: dest_index,
+  username: ENV['ELASTICSEARCH_USERNAME'],  # optional
+  password: ENV['ELASTICSEARCH_PASSWORD'],  # optional
+  wait_for_completion: false                # runs asynchronously by default
+)
+```
+
+Note: For remote reindexing to work, the destination Elasticsearch cluster must have the source
+host configured in `reindex.remote.whitelist` in `elasticsearch.yml`.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/zelastic/index_manager.rb

@@ -92,12 +92,50 @@ module Zelastic
       client.indices.delete(index: indices_to_delete)
     end
 
+    def reindex_from_local(source_index:, dest_index:, wait_for_completion: false)
+      logger.info("Reindexing from #{source_index} to #{dest_index}")
+      reindex(source: { index: source_index }, dest_index: dest_index,
+        wait_for_completion: wait_for_completion
+      )
+    end
+
+    def reindex_from_remote(source_host:, source_index:, dest_index:,
+      username: nil, password: nil, wait_for_completion: false)
+      logger.info("Reindexing from remote #{source_host}/#{source_index} to #{dest_index}")
+
+      remote = { host: source_host }
+      remote[:username] = username if username
+      remote[:password] = password if password
+
+      reindex(source: { remote: remote, index: source_index }, dest_index: dest_index,
+        wait_for_completion: wait_for_completion
+      )
+    end
+
+    def current_read_index
+      client.indices.get_alias(name: config.read_alias).keys.first
+    end
+
+    def current_write_index
+      client.indices.get_alias(name: config.write_alias).keys.first
+    end
+
     private
 
     attr_reader :config, :client
 
     def_delegators :config, :logger
 
+    def reindex(source:, dest_index:, wait_for_completion:)
+      client.reindex(
+        body: {
+          source: source,
+          dest: { index: dest_index }
+        },
+        wait_for_completion: wait_for_completion
+      )
+    end
+
     def indexer
       @indexer ||= Indexer.new(config)
     end

data/lib/zelastic/indexer.rb

@@ -13,6 +13,10 @@ module Zelastic
 
     extend Forwardable
 
+    VERSION_CONFLICT_ERROR_REGEXP =
+      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/
+    private_constant :VERSION_CONFLICT_ERROR_REGEXP
+
     def initialize(config)
       @config = config
     end
@@ -118,8 +122,5 @@ module Zelastic
       error['type'] == 'version_conflict_engine_exception' &&
         error['reason'] =~ VERSION_CONFLICT_ERROR_REGEXP
     end
-
-    VERSION_CONFLICT_ERROR_REGEXP =
-      /^\[\d+\]: version conflict, current version \[\d+\] is higher or equal to the one provided \[\d+\]$/.freeze
   end
 end

data/lib/zelastic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Zelastic
-  VERSION = '0.8.0'
+  VERSION = '0.9.0'
 end

data/zelastic.gemspec

@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
 
   spec.add_development_dependency 'bundler', '~> 2'
-  spec.add_development_dependency 'carwow_rubocop', '~> 4'
+  spec.add_development_dependency 'carwow_rubocop', '~> 6'
   spec.add_development_dependency 'pry', '~> 0.14'
   spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~> 3'

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: zelastic
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - carwow Developers
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-19 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -78,14 +77,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -135,12 +134,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".circleci/config.yml"
 - ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md
+- CODEOWNERS
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -160,7 +160,6 @@ homepage: https://github.com/carwow/zelastic
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -175,8 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.11
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Zero-downtime (re-)indexing of ActiveRecord models into Elasticsearch.
 test_files: []

data/.circleci/config.yml

@@ -1,48 +0,0 @@
-version: 2.1
-
-jobs:
-  test:
-    parameters:
-      ruby_version:
-        type: string
-      elasticsearch_version:
-        type: string
-      elasticsearch_gem_version:
-        type: string
-    docker:
-      - image: "cimg/ruby:<< parameters.ruby_version >>"
-      - image: "elasticsearch:<< parameters.elasticsearch_version >>"
-        environment:
-          "discovery.type": single-node
-          "xpack.security.enabled": false
-    environment:
-      ELASTICSEARCH_GEM_VERSION: "<< parameters.elasticsearch_gem_version >>"
-    steps:
-      - checkout
-      - run: bin/setup
-      - run: bin/rubocop
-      - run: dockerize -wait http://localhost:9200 -timeout 1m
-      - run: bin/rspec
-
-workflows:
-  version: 2
-  test:
-    jobs:
-      - test:
-          name: "test with elasticsearch v8
-            and gem << matrix.elasticsearch_gem_version >>
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "8.3.3"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 8", "~> 7"]
-      - test:
-          name: "test with elasticsearch v7
-            and gem << matrix.elasticsearch_gem_version >>
-            and ruby v<< matrix.ruby_version >>"
-          elasticsearch_version: "7.17.5"
-          matrix:
-            parameters:
-              ruby_version: ["3.1", "2.7"]
-              elasticsearch_gem_version: ["~> 7"]