elastomer-client 6.2.1 → 6.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49da91bf2c259b9b77ed76d66fa001aa4192432225090bb62ff9d74f8173f8dc
-  data.tar.gz: 2d519500cced0a8783a9fd5b77853d935cc3b99f35e8be981e5e1e23b55d9632
+  metadata.gz: 96e8ae346cfe62276c448747cca8b9959c835aa4d52b1681c1f5b52474f45778
+  data.tar.gz: e4c470563171f93f51c74b5f5fa0c3d696c3b0386183d949ad104ac46f95ef3b
 SHA512:
-  metadata.gz: 2f0f879414271cf6f861bfcf256ec6af824a60c2603dbb48dc2297ecffae55f969e7947e8c9cfc0469239c15fe4b9754721279886def527eb89658a549f085fb
-  data.tar.gz: 42cd80d2c67d119767451b524543f73cc1af3ffda4088186cfac8daa99970849655c05ed5382e7ab689a71e1a5c8e4c6b620fbaf66a91693e4afad7e186900ff
+  metadata.gz: 779b6def22ad36824d81dfdd1a2ee659aacba776fb3ea4c4530f88ac6e39dd1379c7414406e56b5edd11fdb0d8a4b528c26437204cae19eeaa62437b6fcdff63
+  data.tar.gz: 07fd71595562ff808e4a28ba22025397566c85b6ac36f272f8f71d769f08e956e9174564a8494d4d45091db79662fcd658a0b4390a538a10ca00e669c47f11b6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 6.2.2 (2025-02-19)
+- Add support for manually and automatically following indices for cross-cluster replication
+
 ## 6.2.1 (2024-12-10)
 - Add support for rethrottling reindex tasks
 

data/README.md

@@ -142,6 +142,11 @@ To run ES 5 and ES 8:
 docker compose --project-directory docker --profile all up
 ```
 
+To run in ES8 cross cluster replication mode:
+```
+script/setup-ccr up "{non-production license}"
+```
+
 To run only ES 8:
 ```
 docker compose --project-directory docker --profile es8 up
@@ -159,6 +164,11 @@ ES 8
 ES_PORT=9208 rake test
 ```
 
+CCR tests:
+```
+ES_PORT=9208 ES_REPLICA_PORT=9209 rake test
+```
+
 ES 5
 ```
 ES_PORT=9205 rake test

data/docker/compose.yaml

@@ -4,7 +4,7 @@ services:
   elasticsearch8.13:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.13.2
     container_name: es8.13
-    profiles: ["es8", "all"]
+    profiles: ["es8", "ccr", "all"]
     environment:
       - cluster.name=elastomer8.13
       - bootstrap.memory_lock=true
@@ -12,6 +12,7 @@ services:
       - xpack.security.enabled=false
       - xpack.watcher.enabled=false
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - node.roles=[master,data,remote_cluster_client]
     ulimits:
       memlock:
         soft: -1
@@ -28,6 +29,33 @@ services:
     ports:
       - 127.0.0.1:${ES_8_PORT:-9208}:9200
   
+  elasticsearchFollower:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.2
+    container_name: es-follow
+    profiles: ["ccr"]
+    environment:
+      - cluster.name=es-follow
+      - bootstrap.memory_lock=true
+      - discovery.type=single-node
+      - xpack.security.enabled=false
+      - xpack.watcher.enabled=false
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+      - node.roles=[master,data,remote_cluster_client]
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 2g
+    cap_add:
+      - IPC_LOCK
+    volumes:
+      - ./elasticsearch-follow.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+    ports:
+      - 127.0.0.1:${ES_8_PORT:-9209}:9201
+
   elasticsearch5.6:
     image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
     container_name: es5.6

data/docker/elasticsearch-follow.yml

@@ -0,0 +1,14 @@
+cluster.name: "es-follow"
+
+network.host: 0.0.0.0
+
+path:
+  data: /usr/share/elasticsearch/data
+  logs: /usr/share/elasticsearch/logs
+  repo: /usr/share/elasticsearch/repos
+
+transport.port: 9301
+http.port: 9201
+remote_cluster.port: 9444
+http.max_content_length: 50mb
+ingest.geoip.downloader.enabled: false

data/docker/elasticsearch8plus.yml

@@ -9,5 +9,6 @@ path:
 
 transport.port: 9300
 http.port: 9200
+remote_cluster.port: 9443
 http.max_content_length: 50mb
 ingest.geoip.downloader.enabled: false

data/lib/elastomer_client/client/ccr.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+module ElastomerClient
+  class Client
+
+    # Returns a CCR instance
+    def ccr
+      Ccr.new(self)
+    end
+
+    class Ccr
+      # Create a new Ccr for initiating requests for cross cluster replication.
+      # More context: https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-apis.html
+      #
+      # client     - ElastomerClient::Client used for HTTP requests to the server
+      def initialize(client)
+        @client = client
+      end
+
+      attr_reader :client
+
+      # Gets the parameters and status for each follower index.
+      #
+      # index_pattern - String name of the index pattern to get follower info for
+      # params        - Hash of query parameters
+      #
+      # Examples
+      #
+      #   ccr.get_follower_info("follower_index")
+      #   ccr.get_follower_info("*")
+      #
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html
+      def get_follower_info(index_pattern, params = {})
+        response = client.get "/#{index_pattern}/_ccr/info", params.merge(action: "get_follower_info", rest_api: "ccr")
+        response.body
+      end
+
+      # Creates a new follower index for the provided leader index on the remote cluster.
+      #
+      # follower_index - String name of the follower index to create
+      # body           - Hash of the request body
+      #   :remote_cluster - String name of the remote cluster. Required.
+      #   :leader_index   - String name of the leader index. Required.
+      # params         - Hash of query parameters
+      #
+      # Examples
+      #
+      #   ccr.follow("follower_index", { leader_index: "leader_index", remote_cluster: "leader" })
+      #
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-follow.html
+      def follow(follower_index, body, params = {})
+        response = client.put "/#{follower_index}/_ccr/follow", params.merge(body:, action: "follow", rest_api: "ccr")
+        response.body
+      end
+
+      # Creates a new auto-follow pattern for the provided remote cluster.
+      #
+      # pattern_name - String name of the auto-follow pattern to create
+      # body         - Hash of the request body
+      #   :remote_cluster - String name of the remote cluster. Required.
+      #   :leader_index_patterns - An array of simple index patterns to match against indices in the remote cluster
+      #   :leader_index_exclusion_patterns - An array of simple index patterns that can be used to exclude indices from being auto-followed.
+      #   :follow_index_pattern - The name of follower index. The template {{leader_index}} can be used to derive
+      #                           the name of the follower index from the name of the leader index.
+      # params       - Hash of query parameters
+
+      # Examples
+
+      #   ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["leader_index*"],
+      #                                         follow_index_pattern: "{{leader_index}}-follower" })
+
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html
+
+      def auto_follow(pattern_name, body, params = {})
+        response = client.put "/_ccr/auto_follow/#{pattern_name}", params.merge(body:, action: "create_auto_follow_pattern", rest_api: "ccr")
+        response.body
+      end
+
+      # Deletes the auto-follow pattern for the provided remote cluster.
+      #
+      # pattern_name - String name of the auto-follow pattern to delete
+      # params       - Hash of query parameters
+      #
+      # Examples
+      #
+      #   ccr.delete_auto_follow("follower_pattern")
+      #
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-delete-auto-follow-pattern.html
+
+      def delete_auto_follow(pattern_name, params = {})
+        response = client.delete "/_ccr/auto_follow/#{pattern_name}", params.merge(action: "delete_auto_follow_pattern", rest_api: "ccr")
+        response.body
+      end
+
+      # Gets cross-cluster replication auto-follow patterns
+      #
+      # params - Hash of query parameters
+      #   :pattern_name - (Optional) String name of the auto-follow pattern. Returns all patterns if not specified
+      # Examples
+      #
+      #   ccr.get_auto_follow
+      #
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-get-auto-follow-pattern.html
+
+      def get_auto_follow(params = {})
+        response = client.get "/_ccr/auto_follow{/pattern_name}", params.merge(action: "get_auto_follow_pattern", rest_api: "ccr")
+        response.body
+      end
+
+      # Pauses a follower index.
+      #
+      # follower_index - String name of the follower index to pause
+      # params         - Hash of the request body
+      #
+      # Examples
+      #   ccr.pause_follow("follower_index")
+      #
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-post-pause-follow.html
+      def pause_follow(follower_index, params = {})
+        response = client.post "/#{follower_index}/_ccr/pause_follow", params.merge(action: "pause_follow", rest_api: "ccr")
+        response.body
+      end
+
+      # Unfollows a leader index given a follower index.
+      # The follower index must be paused and closed before unfollowing.
+      #
+      # follower_index - String name of the follower index to unfollow
+      # params         - Hash of the request body
+      #
+      # Examples
+      #   ccr.unfollow("follower_index")
+      # See https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-post-unfollow.html
+      def unfollow(follower_index, params = {})
+        response = client.post "/#{follower_index}/_ccr/unfollow", params.merge(action: "unfollow", rest_api: "ccr")
+        response.body
+      end
+    end
+  end
+end

data/lib/elastomer_client/client/reindex.rb

@@ -20,12 +20,12 @@ module ElastomerClient
       attr_reader :client
 
       def reindex(body, params = {})
-        response = client.post "/_reindex", params.merge(params, body:, action: "reindex", rest_api: "reindex")
+        response = client.post "/_reindex", params.merge(body:, action: "reindex", rest_api: "reindex")
         response.body
       end
 
       def rethrottle(task_id, params = {})
-        response = client.post "/_reindex/#{task_id}/_rethrottle", params.merge(params, action: "rethrottle", rest_api: "reindex")
+        response = client.post "/_reindex/#{task_id}/_rethrottle", params.merge(action: "rethrottle", rest_api: "reindex")
         response.body
       end
 

data/lib/elastomer_client/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module ElastomerClient
-  VERSION = "6.2.1"
+  VERSION = "6.2.2"
 
   def self.version
     VERSION

data/script/setup-ccr

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Function to display help
+show_help() {
+  echo "Usage: $0 [option] [license]"
+  echo "Options:"
+  echo "  up      - Start the clusters and apply the license"
+  echo "  down    - Shut down the clusters"
+  echo "  help    - Display this help message"
+}
+
+# Function to apply the license to a cluster
+apply_license() {
+  local port=$1
+  local license="$2"
+  local response_file=$(mktemp)
+  local http_code
+  http_code=$(curl -s -o "$response_file" -w "%{http_code}" -X PUT "http://localhost:$port/_license?pretty" -H "Content-Type: application/json" -d "$license")
+  
+  if [ "$http_code" -ne 200 ]; then
+    echo "Failed to apply license to cluster on port $port. HTTP status code: $http_code"
+    echo "Error response: $(cat "$response_file")"
+    rm "$response_file"
+    exit 1
+  fi
+}
+
+# Function to shut down the clusters
+shutdown_clusters() {
+  docker compose --project-directory docker --profile ccr down
+  echo "Clusters shut down."
+}
+
+# Check for options
+case "$1" in
+  up)
+
+    # Get the directory of the current script
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+    # Start the clusters
+    docker compose --project-directory docker --profile ccr up -d
+
+    # Wait for both clusters to be online
+    export ES_PORT=9208
+    "$SCRIPT_DIR/poll-for-es"
+    export ES_PORT=9209
+    "$SCRIPT_DIR/poll-for-es"
+
+    # Apply the license to both clusters
+    LICENSE=$2
+    if [ -z "$LICENSE" ]; then
+      echo "License key is required as the second argument."
+      exit 1
+    fi
+
+    echo "Applying license to cluster on port 9208..."
+    apply_license 9208 "$LICENSE"
+    echo "Applying license to cluster on port 9209..."
+    apply_license 9209 "$LICENSE"
+    echo "License applied to both clusters."
+
+    # Set up the remote connection between the clusters
+    curl -X PUT "http://localhost:9209/_cluster/settings" -H "Content-Type: application/json" -d '{
+      "persistent": {
+        "cluster": {
+          "remote": {
+            "leader": {
+              "seeds": ["es8.13:9300"]
+            }
+          }
+        }
+      }
+    }'
+
+    echo "Clusters setup completed."
+    ;;
+  down)
+    shutdown_clusters
+    ;;
+  help)
+    show_help
+    ;;
+  *)
+    echo "Invalid option: $1"
+    show_help
+    exit 1
+    ;;
+esac

data/test/client/ccr_test.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+describe ElastomerClient::Client::Ccr do
+  before do
+    skip "Cannot test Ccr API without a replica cluster" unless $replica_client.available?
+
+    begin
+      ccr.delete_auto_follow("follower_pattern")
+    rescue StandardError
+      puts "No auto-follow pattern to delete"
+    end
+
+    @leader_index = $client.index("leader_index")
+    @follower_index = $replica_client.index("follower_index")
+    @auto_followed_index = $client.index("followed_index")
+    @auto_follower_index = $replica_client.index("followed_index-follower")
+
+    if @leader_index.exists?
+      @leader_index.delete
+    end
+    if @auto_followed_index.exists?
+      @auto_followed_index.delete
+    end
+    if @follower_index.exists?
+      @follower_index.delete
+    end
+    if @auto_follower_index.exists?
+      @auto_follower_index.delete
+    end
+
+    @leader_index.create(default_index_settings)
+    wait_for_index(@leader_index.name, "green")
+  end
+
+  after do
+    @leader_index.delete if @leader_index.exists?
+    @follower_index.delete if @follower_index.exists?
+    @auto_followed_index.delete if @auto_followed_index.exists?
+    @auto_follower_index.delete if @auto_follower_index.exists?
+
+    begin
+      ccr.delete_auto_follow("follower_pattern")
+    rescue StandardError
+      puts "No auto-follow pattern to delete"
+    end
+  end
+
+  def follow_index(follower_index_name, leader_index_name)
+    ccr = $replica_client.ccr
+    response = ccr.follow(follower_index_name, { leader_index: leader_index_name, remote_cluster: "leader" })
+    wait_for_index(follower_index_name, "green")
+    response
+  end
+
+  def pause_follow(follower_index_name)
+    ccr = $replica_client.ccr
+    response = ccr.pause_follow(follower_index_name)
+    wait_for_index(follower_index_name, "green")
+    response
+  end
+
+  def unfollow_index(follower_index_name)
+    ccr = $replica_client.ccr
+    response = ccr.unfollow(follower_index_name)
+    wait_for_index(follower_index_name, "green")
+    response
+  end
+
+  def create_document(index, type, document)
+    response = index.docs.index(document_wrapper(type, document))
+    index.refresh
+    response
+  end
+
+  it "successfully follows a leader index" do
+    create_document(@leader_index, "book", { _id: 1, title: "Book 1" })
+    follow_index(@follower_index.name, @leader_index.name)
+
+    doc = @follower_index.docs.get(id: 1, type: "book")
+
+    assert_equal "Book 1", doc["_source"]["title"]
+  end
+
+  it "successfully gets info for all follower indices" do
+    follow_index(@follower_index.name, @leader_index.name)
+
+    response = $replica_client.ccr.get_follower_info("*")
+
+    assert_equal response["follower_indices"][0]["follower_index"], @follower_index.name
+    assert_equal response["follower_indices"][0]["leader_index"], @leader_index.name
+  end
+
+  it "successfully pauses a follower index" do
+    follow_index(@follower_index.name, @leader_index.name)
+
+    response = pause_follow(@follower_index.name)
+
+    assert response["acknowledged"]
+
+    create_document(@leader_index, "book", { _id: 2, title: "Book 2" })
+
+    doc = @follower_index.docs.get(id: 2, type: "book")
+
+    refute doc["found"]
+  end
+
+  it "successfully unfollow a leader index" do
+    follow_index(@follower_index.name, @leader_index.name)
+
+    pause_follow(@follower_index.name)
+
+    @follower_index.close
+
+    response = unfollow_index(@follower_index.name)
+
+    assert response["acknowledged"]
+
+    @follower_index.open
+
+    wait_for_index(@follower_index.name, "green")
+
+    create_document(@leader_index, "book", { _id: 2, title: "Book 2" })
+
+    doc = @follower_index.docs.get(id: 2, type: "book")
+
+    refute doc["found"]
+  end
+
+  it "successfully implements an auto-follow policy" do
+    ccr = $replica_client.ccr
+
+    ccr.auto_follow("follower_pattern", { remote_cluster: "leader", leader_index_patterns: ["*"], follow_index_pattern: "{{leader_index}}-follower" })
+
+    @auto_followed_index.create(default_index_settings)
+    wait_for_index(@auto_followed_index.name, "green")
+
+    @auto_follower_index = $replica_client.index("followed_index-follower")
+    wait_for_index(@auto_follower_index.name, "green")
+
+    resp = ccr.get_auto_follow(pattern_name: "follower_pattern")
+
+    assert_equal "follower_pattern", resp["patterns"].first["name"]
+
+    assert_predicate @auto_follower_index, :exists?
+  end
+
+end

data/test/client/reindex_test.rb

@@ -57,15 +57,16 @@ describe ElastomerClient::Client::Reindex do
     }
     response = reindex.reindex(body, requests_per_second: 0.01, wait_for_completion: false)
     task_id = response["task"]
+    node_id = task_id.split(":").first
+    task_number = task_id.split(":").last.to_i
 
-    reindex.rethrottle(task_id, requests_per_second: 1)
+    response = reindex.rethrottle(task_id, requests_per_second: 1)
 
-    tasks = $client.tasks
-    node_id = task_id.split(":").first
-    task_id = task_id.split(":").last.to_i
+    assert_equal 1, response["nodes"][node_id]["tasks"][task_id]["status"]["requests_per_second"]
 
     # wait for the task to complete
-    tasks.wait_by_id(node_id, task_id, "30s")
+    tasks = $client.tasks
+    tasks.wait_by_id(node_id, task_number, "30s")
 
     # Verify that the document has been reindexed
     doc = @dest_index.docs.get(id: 1, type: "book")

data/test/test_helper.rb

@@ -46,6 +46,19 @@ raise "No server available at #{$client.url}" unless $client.available?
 
 puts "Elasticsearch version is #{$client.version}"
 
+# Create client instance for replica cluster
+$replica_client_params = {
+  port: ENV.fetch("ES_REPLICA_PORT", 9201),
+  read_timeout: 10,
+  open_timeout: 1,
+  opaque_id: false,
+  strict_params: true,
+  compress_body: true
+}
+$replica_client = ElastomerClient::Client.new(**$replica_client_params)
+
+puts "Replica server is unavailable at #{$replica_client.url}" unless $replica_client.available?
+
 # remove any lingering test indices from the cluster
 Minitest.after_run do
   $client.cluster.indices.keys.each do |name|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elastomer-client
 version: !ruby/object:Gem::Version
-  version: 6.2.1
+  version: 6.2.2
 platform: ruby
 authors:
 - Tim Pease
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-12-11 00:00:00.000000000 Z
+date: 2025-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -105,6 +105,7 @@ files:
 - README.md
 - Rakefile
 - docker/compose.yaml
+- docker/elasticsearch-follow.yml
 - docker/elasticsearch.yml
 - docker/elasticsearch8plus.yml
 - docs/README.md
@@ -121,6 +122,7 @@ files:
 - elastomer-client.gemspec
 - lib/elastomer_client/client.rb
 - lib/elastomer_client/client/bulk.rb
+- lib/elastomer_client/client/ccr.rb
 - lib/elastomer_client/client/cluster.rb
 - lib/elastomer_client/client/delete_by_query.rb
 - lib/elastomer_client/client/docs.rb
@@ -157,8 +159,10 @@ files:
 - script/console
 - script/generate-rest-api-spec
 - script/poll-for-es
+- script/setup-ccr
 - test/assertions.rb
 - test/client/bulk_test.rb
+- test/client/ccr_test.rb
 - test/client/cluster_test.rb
 - test/client/docs_test.rb
 - test/client/errors_test.rb
@@ -213,6 +217,7 @@ summary: A library for interacting with Elasticsearch
 test_files:
 - test/assertions.rb
 - test/client/bulk_test.rb
+- test/client/ccr_test.rb
 - test/client/cluster_test.rb
 - test/client/docs_test.rb
 - test/client/errors_test.rb