logstash-input-elasticsearch 4.22.0 → 5.0.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.22.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-04-07 00:00:00.000000000 Z
+date: 2024-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,9 +92,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
   name: elasticsearch
   type: :runtime
   prerelease: false
@@ -103,9 +100,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 7.17.9
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '9'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -278,26 +272,21 @@ files:
 - lib/logstash/helpers/loggable_try.rb
 - lib/logstash/inputs/elasticsearch.rb
 - lib/logstash/inputs/elasticsearch/aggregation.rb
-- lib/logstash/inputs/elasticsearch/cursor_tracker.rb
 - lib/logstash/inputs/elasticsearch/paginated_search.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_connections_selector.rb
 - lib/logstash/inputs/elasticsearch/patches/_elasticsearch_transport_http_manticore.rb
 - logstash-input-elasticsearch.gemspec
 - spec/es_helper.rb
-- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
-- spec/fixtures/test_certs/renew.sh
-- spec/inputs/cursor_tracker_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb
 - spec/inputs/paginated_search_spec.rb
-homepage: https://elastic.co/logstash
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
 metadata:
@@ -324,15 +313,11 @@ specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
 - spec/es_helper.rb
-- spec/fixtures/test_certs/GENERATED_AT
 - spec/fixtures/test_certs/ca.crt
 - spec/fixtures/test_certs/ca.der.sha256
 - spec/fixtures/test_certs/ca.key
-- spec/fixtures/test_certs/es.chain.crt
 - spec/fixtures/test_certs/es.crt
 - spec/fixtures/test_certs/es.key
-- spec/fixtures/test_certs/renew.sh
-- spec/inputs/cursor_tracker_spec.rb
 - spec/inputs/elasticsearch_spec.rb
 - spec/inputs/elasticsearch_ssl_spec.rb
 - spec/inputs/integration/elasticsearch_spec.rb

data/lib/logstash/inputs/elasticsearch/cursor_tracker.rb

@@ -1,58 +0,0 @@
-require 'fileutils'
-
-module LogStash; module Inputs; class Elasticsearch
-  class CursorTracker
-    include LogStash::Util::Loggable
-
-    attr_reader :last_value
-
-    def initialize(last_run_metadata_path:, tracking_field:, tracking_field_seed:)
-      @last_run_metadata_path = last_run_metadata_path
-      @last_value_hashmap = Java::java.util.concurrent.ConcurrentHashMap.new
-      @last_value = IO.read(@last_run_metadata_path) rescue nil || tracking_field_seed
-      @tracking_field = tracking_field
-      logger.info "Starting value for cursor field \"#{@tracking_field}\": #{@last_value}"
-      @mutex = Mutex.new
-    end
-
-    def checkpoint_cursor(intermediate: true)
-      @mutex.synchronize do
-        if intermediate
-          # in intermediate checkpoints pick the smallest
-          converge_last_value {|v1, v2| v1 < v2 ? v1 : v2}
-        else
-          # in the last search of a PIT choose the largest
-          converge_last_value {|v1, v2| v1 > v2 ? v1 : v2}
-          @last_value_hashmap.clear
-        end
-        IO.write(@last_run_metadata_path, @last_value)
-      end
-    end
-
-    def converge_last_value(&block)
-      return if @last_value_hashmap.empty?
-      new_last_value = @last_value_hashmap.reduceValues(1000, &block)
-      logger.debug? && logger.debug("converge_last_value: got #{@last_value_hashmap.values.inspect}. won: #{new_last_value}")
-      return if new_last_value == @last_value
-      @last_value = new_last_value
-      logger.info "New cursor value for field \"#{@tracking_field}\" is: #{new_last_value}"
-    end
-
-    def record_last_value(event)
-      value = event.get(@tracking_field)
-      logger.trace? && logger.trace("storing last_value if #{@tracking_field} for #{Thread.current.object_id}: #{value}")
-      @last_value_hashmap.put(Thread.current.object_id, value)
-    end
-
-    def inject_cursor(query_json)
-      # ":present" means "now - 30s" to avoid grabbing partially visible data in the PIT
-      result = query_json.gsub(":last_value", @last_value.to_s).gsub(":present", now_minus_30s)
-      logger.debug("inject_cursor: injected values for ':last_value' and ':present'", :query => result)
-      result
-    end
-
-    def now_minus_30s
-      Java::java.time.Instant.now.minusSeconds(30).to_s
-    end
-  end
-end; end; end

data/spec/fixtures/test_certs/GENERATED_AT

@@ -1 +0,0 @@
-2024-12-26T22:27:15+00:00

data/spec/fixtures/test_certs/es.chain.crt

@@ -1,38 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDIzCCAgugAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMA0xCzAJBgNVBAMTAmVzMIIBIjANBgkqhkiG
-9w0BAQEFAAOCAQ8AMIIBCgKCAQEArZLZvLSWDK7Ul+AaBnjU81dsfaow8zOjCC5V
-V21nXpYzQJoQbuWcvGYxwL7ZDs2ca4Wc8BVCj1NDduHuP7U+QIlUdQpl8kh5a0Zz
-36pcFw7UyF51/AzWixJrht/Azzkb5cpZtE22ZK0KhS4oCsjJmTN0EABAsGhDI9/c
-MjNrUC7iP0dvfOuzAPp7ufY83h98jKKXUYV24snbbvmqoWI6GQQNSG/sEo1+1UGH
-/z07/mVKoBAa5DVoNGvxN0fCE7vW7hkhT8+frJcsYFatAbnf6ql0KzEa8lN9u0gR
-hQNM3zcKKsjEMomBzVBc4SV3KXO0d/jGdDtlqsm2oXqlTMdtGwIDAQABo2cwZTAY
-BgNVHREEETAPgg1lbGFzdGljc2VhcmNoMAkGA1UdEwQCMAAwHQYDVR0OBBYEFFQU
-K+6Cg2kExRj1xSDzEi4kkgKXMB8GA1UdIwQYMBaAFMgkye5+2l+TE0I6RsXRHjGB
-wpBGMA0GCSqGSIb3DQEBCwUAA4IBAQB6cZ7IrDzcAoOZgAt9RlOe2yzQeH+alttp
-CSQVINjJotS1WvmtqjBB6ArqLpXIGU89TZsktNe/NQJzgYSaMnlIuHVLFdxJYmwU
-T1cP6VC/brmqP/dd5y7VWE7Lp+Wd5CxKl/WY+9chmgc+a1fW/lnPEJJ6pca1Bo8b
-byIL0yY2IUv4R2eh1IyQl9oGH1GOPLgO7cY04eajxYcOVA2eDSItoyDtrJfkFP/P
-UXtC1JAkvWKuujFEiBj0AannhroWlp3gvChhBwCuCAU0KXD6g8BE8tn6oT1+FW7J
-avSfHxAe+VHtYhF8sJ8jrdm0d7E4GKS9UR/pkLAL1JuRdJ1VkPx3
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIDFTCCAf2gAwIBAgIBATANBgkqhkiG9w0BAQsFADA0MTIwMAYDVQQDEylFbGFz
-dGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTAeFw0yNDEyMjYy
-MjI3MTVaFw0yNTEyMjYyMjI3MTVaMDQxMjAwBgNVBAMTKUVsYXN0aWMgQ2VydGlm
-aWNhdGUgVG9vbCBBdXRvZ2VuZXJhdGVkIENBMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEArUe66xG4Y2zO13gRC+rBwyvxe+c01pqV6ukw6isIbJIQWs1/
-QfEMhUwYwKs6/UXxK+VwardcA2zYwngXbGGEtms+mpUfH5CdJnrqW7lHz1BVK4yH
-90IzGE0GU4D90OW/L4QkGX0fv3VQbL8KGFKBoF04pXIaSGMStFN4wirutHtQboYv
-99X4kbLjVSIuubUpA/v9dUP1TNl8ar+HKUWRM96ijHkFTF3FR0NnZyt44gP5qC0h
-i4lUiR6Uo9D6WMFjeRYFF7GolCy/I1SzWBmmOnNhQLO5VxcNG4ldhBcapZeGwE98
-m/5lxLIwgFR9ZP8bXdxZTWLC58/LQ2NqOjA9mwIDAQABozIwMDAPBgNVHRMBAf8E
-BTADAQH/MB0GA1UdDgQWBBTIJMnuftpfkxNCOkbF0R4xgcKQRjANBgkqhkiG9w0B
-AQsFAAOCAQEAhfg/cmXc4Uh90yiXU8jOW8saQjTsq4ZMDQiLfJsNmNNYmHFN0vhv
-lJRI1STdy7+GpjS5QbrMjQIxWSS8X8xysE4Rt81IrWmLuao35TRFyoiE1seBQ5sz
-p/BxZUe57JvWi9dyzv2df4UfWFdGBhzdr80odZmz4i5VIv6qCKJKsGikcuLpepmp
-E/UKnKHeR/dFWsxzA9P2OzHTUNBMOOA2PyAUL49pwoChwJeOWN/zAgwMWLbuHFG0
-IN0u8swAmeH98QdvzbhiOatGNpqfTNvQEDc19yVjfXKpBVZQ79WtronYSqrbrUa1
-T2zD8bIVP7CdddD/UmpT1SSKh4PJxudy5Q==
------END CERTIFICATE-----

data/spec/fixtures/test_certs/renew.sh

@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-cd "$(dirname "$0")"
-
-openssl x509 -x509toreq -in ca.crt -copy_extensions copyall -signkey ca.key -out ca.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in ca.csr -set_serial 0x01 -signkey ca.key -out ca.crt && rm ca.csr
-openssl x509 -in ca.crt -outform der | sha256sum | awk '{print $1}' > ca.der.sha256
-
-openssl x509 -x509toreq -in es.crt -copy_extensions copyall -signkey es.key -out es.csr
-openssl x509 -req -copy_extensions copyall -days 365 -in es.csr -set_serial 0x01 -CA ca.crt -CAkey ca.key -out es.crt && rm es.csr
-cat es.crt ca.crt > es.chain.crt
-
-# output ISO8601 timestamp to file
-date -Iseconds > GENERATED_AT

data/spec/inputs/cursor_tracker_spec.rb

@@ -1,72 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/devutils/rspec/shared_examples"
-require "logstash/inputs/elasticsearch"
-require "logstash/inputs/elasticsearch/cursor_tracker"
-
-describe LogStash::Inputs::Elasticsearch::CursorTracker do
-
-  let(:last_run_metadata_path) { Tempfile.new('cursor_tracker_testing').path }
-  let(:tracking_field_seed) { "1980-01-01T23:59:59.999999999Z" }
-  let(:options) do
-    {
-      :last_run_metadata_path => last_run_metadata_path,
-      :tracking_field => "my_field",
-      :tracking_field_seed => tracking_field_seed
-    }
-  end
-
-  subject { described_class.new(**options) }
-
-  it "creating a class works" do
-    expect(subject).to be_a described_class
-  end
-
-  describe "checkpoint_cursor" do
-    before(:each) do
-      subject.checkpoint_cursor(intermediate: false) # store seed value
-      [
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-03T23:59:59.999999999Z")) },
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-01T23:59:59.999999999Z")) },
-        Thread.new(subject) {|subject| subject.record_last_value(LogStash::Event.new("my_field" => "2025-01-02T23:59:59.999999999Z")) },
-      ].each(&:join)
-    end
-    context "when doing intermediate checkpoint" do
-      it "persists the smallest value" do
-        subject.checkpoint_cursor(intermediate: true)
-        expect(IO.read(last_run_metadata_path)).to eq("2025-01-01T23:59:59.999999999Z")
-      end
-    end
-    context "when doing non-intermediate checkpoint" do
-      it "persists the largest value" do
-        subject.checkpoint_cursor(intermediate: false)
-        expect(IO.read(last_run_metadata_path)).to eq("2025-01-03T23:59:59.999999999Z")
-      end
-    end
-  end
-
-  describe "inject_cursor" do
-    let(:new_value) { "2025-01-03T23:59:59.999999999Z" }
-    let(:fake_now) { "2026-09-19T23:59:59.999999999Z" }
-
-    let(:query) do
-      %q[
-      { "query": { "range": { "event.ingested": { "gt": :last_value, "lt": :present}}}, "sort": [ { "event.ingested": {"order": "asc", "format": "strict_date_optional_time_nanos", "numeric_type" : "date_nanos" } } ] }
-      ]
-    end
-
-    before(:each) do
-      subject.record_last_value(LogStash::Event.new("my_field" => new_value))
-      subject.checkpoint_cursor(intermediate: false)
-      allow(subject).to receive(:now_minus_30s).and_return(fake_now)
-    end
-
-    it "injects the value of the cursor into json query if it contains :last_value" do
-      expect(subject.inject_cursor(query)).to match(/#{new_value}/)
-    end
-
-    it "injects current time into json query if it contains :present" do
-      expect(subject.inject_cursor(query)).to match(/#{fake_now}/)
-    end
-  end
-end