logstash-input-elasticsearch 4.3.2 → 4.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0323fd0462661ab2568f90769666a59f17574aa5bfb9ed6e3c280ba32d6e52e
-  data.tar.gz: c65aabfa5514d75b7414d32cf2d7411d1690f52721193467b3d97316f28cd30d
+  metadata.gz: 72f28226e8297df8d4eda501ef9a99a67c11bab0574711f553797c678cd5e9a8
+  data.tar.gz: d141edd1c0664b1f77f4ddc757cda564781aeaa2ada1ebee828ac2f76268c1de
 SHA512:
-  metadata.gz: cd91e70ac02af9d834337649dc11ba13b4a39ce9165b19419b7a1922d6907b240dbf735436700709cffc0747708238d943ddcc5a4bb9a8b7b390a82d65242c57
-  data.tar.gz: 9b9d390c39ef7394c87aa799c38e0039af8955696b1c8fc98f360e8e1331091f33e583d6192f57e0a957df9a6436778b59f1366fdb87ce94fc97fcb104ed7b72
+  metadata.gz: 468f91f896d037895c7479e7ae377cee60c4b0c22b05918695f35b6e56bdfa405524fd04d03440133ac40a71fa4ed00251ad27e38792f7c30a8b736ffaa186c9
+  data.tar.gz: 5ba240ae691f622ab6bf410b7603ebbb959ee5c50cddc90748dd26a1a349fcc36057e9147a15d0501ea1e4f0bbc2861f3042c0addc070568ec805cdd6668d999

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.3.3
+  - Loosen restrictions on Elasticsearch gem [#110](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/110)
+
 ## 4.3.2
   - Fixed broken link to Elasticsearch Reference  [#106](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/106)
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.3.2'
+  s.version         = '4.3.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
-  s.add_runtime_dependency 'elasticsearch', ['>= 5.0.3', '< 6.0.0']
+  s.add_runtime_dependency 'elasticsearch', '>= 5.0.3'
 
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-codec-plain'

data/spec/es_helper.rb

@@ -0,0 +1,40 @@
+module ESHelper
+  def self.get_host_port
+    return "http://elasticsearch:9200" if ENV["INTEGRATION"] == "true"
+    raise "This setting is only used for integration tests"
+  end
+
+  def self.get_client
+    Elasticsearch::Client.new(:hosts => [get_host_port])
+  end
+
+  def self.doc_type
+    if ESHelper.es_version_satisfies?(">=8")
+      nil
+    elsif ESHelper.es_version_satisfies?(">=7")
+      "_doc"
+    else
+      "doc"
+    end
+  end
+
+  def self.index_doc(es, params)
+    type = doc_type
+    params[:type] = doc_type unless type.nil?
+    es.index(params)
+  end
+
+  def self.es_version
+    ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+  end
+
+  def self.es_version_satisfies?(*requirement)
+    es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION']
+    if es_version.nil?
+      puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
+      return false
+    end
+    es_release_version = Gem::Version.new(es_version).release
+    Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
+  end
+end

data/spec/inputs/elasticsearch_spec.rb

@@ -375,15 +375,13 @@ describe LogStash::Inputs::Elasticsearch do
       end
 
       it 'merges the values if the `docinfo_target` already exist in the `_source` document' do
-        metadata_field = 'metadata_with_hash'
-
         config_metadata_with_hash = %Q[
             input {
               elasticsearch {
                 hosts => ["localhost"]
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
-                docinfo_target => '#{metadata_field}'
+                docinfo_target => 'metadata_with_hash'
               }
             }
         ]
@@ -392,33 +390,23 @@ describe LogStash::Inputs::Elasticsearch do
           queue.pop
         end
 
-        expect(event.get("[#{metadata_field}][_index]")).to eq('logstash-2014.10.12')
-        expect(event.get("[#{metadata_field}][_type]")).to eq('logs')
-        expect(event.get("[#{metadata_field}][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
-        expect(event.get("[#{metadata_field}][awesome]")).to eq("logstash")
+        expect(event.get("[metadata_with_hash][_index]")).to eq('logstash-2014.10.12')
+        expect(event.get("[metadata_with_hash][_type]")).to eq('logs')
+        expect(event.get("[metadata_with_hash][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
+        expect(event.get("[metadata_with_hash][awesome]")).to eq("logstash")
       end
 
-      it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
-        metadata_field = 'metadata_with_string'
-
-        config_metadata_with_string = %Q[
-            input {
-              elasticsearch {
-                hosts => ["localhost"]
-                query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-                docinfo => true
-                docinfo_target => '#{metadata_field}'
-              }
-            }
-        ]
-
-        pipeline = new_pipeline_from_string(config_metadata_with_string)
-        queue = Queue.new
-        pipeline.instance_eval do
-          @output_func = lambda { |event| queue << event }
+      context 'if the `docinfo_target` exist but is not of type hash' do
+        let (:config) { {
+            "hosts" => ["localhost"],
+            "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+            "docinfo" => true,
+            "docinfo_target" => 'metadata_with_string'
+        } }
+        it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
+          plugin.register
+          expect { plugin.run([]) }.to raise_error(Exception, /incompatible event/)
         end
-
-        expect { pipeline.run }.to raise_error(Exception, /incompatible event/)
       end
 
       it "should move the document info to the @metadata field" do

data/spec/inputs/integration/elasticsearch_spec.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/plugin"
+require "logstash/inputs/elasticsearch"
+require_relative "../../../spec/es_helper"
+
+describe LogStash::Inputs::Elasticsearch, :integration => true do
+
+  let(:config)   { { 'hosts' => [ESHelper.get_host_port],
+                     'index' => 'logs',
+                     'query' => '{ "query": { "match": { "message": "Not found"} }}' } }
+  let(:plugin) { described_class.new(config) }
+  let(:event)  { LogStash::Event.new({}) }
+
+  before(:each) do
+    @es = ESHelper.get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    10.times do
+      ESHelper.index_doc(@es, :index => 'logs', :body => { :response => 404, :message=> 'Not Found'})
+    end
+    @es.indices.refresh
+    plugin.register
+  end
+
+  after(:each) do
+    @es.indices.delete_template(:name => "*")
+    @es.indices.delete(:index => "*") rescue nil
+  end
+
+  describe 'smoke test' do
+    it "should retrieve json event from elasticseach" do
+      queue = []
+      plugin.run(queue)
+      event = queue.pop
+      expect(event).to be_a(LogStash::Event)
+      expect(event.get("response")).to eql(404)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.3.2
+  version: 4.3.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-31 00:00:00.000000000 Z
+date: 2019-11-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -36,9 +36,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
   name: elasticsearch
   prerelease: false
   type: :runtime
@@ -47,9 +44,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 5.0.3
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 6.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -179,7 +173,9 @@ files:
 - docs/index.asciidoc
 - lib/logstash/inputs/elasticsearch.rb
 - logstash-input-elasticsearch.gemspec
+- spec/es_helper.rb
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -207,4 +203,6 @@ signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster
 test_files:
+- spec/es_helper.rb
 - spec/inputs/elasticsearch_spec.rb
+- spec/inputs/integration/elasticsearch_spec.rb