logstash-input-elasticsearch 3.0.2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5b8d06824d9ce5dbbee6d8d903776a936cd868a
-  data.tar.gz: eb4cae9d990982b54d9892942c5c39fa14e543c1
+  metadata.gz: 99cc10fa5daee63fcb318c325f8edaa86c443f22
+  data.tar.gz: 0dd1acac2ff9314ab1decf1b57782c1ad37c390e
 SHA512:
-  metadata.gz: 3d49cd543e144854ee7cede1aa77fd0010c243b7a06f70911beebfb4f61d210b20486446adbd1c7e165a5caf08ac221810ce92dbb4de53c02e573c07a814af1a
-  data.tar.gz: df04ca58458f17d663913804b2aa4de4656b055943471ba8cf6398f297fba39bd36bff30164f2a5d10e1439af66a5bc8da597cbebb91892c2206ee903944904e
+  metadata.gz: c2b22763cfc5521cbbff86c11d7c01030f4e9d88df57d7d6c02813eaf97524e34e641e2ee109c6d2aa14f270e04a98c2c4d1818d8880fd9d779be6fd982a6146
+  data.tar.gz: c89c0da4e80ef31a41d1ac15c13fe34ba5d4eb3ed0ec6cff356b23fd29aeec7b61d65f6d7562bcc663eb05f67a3ca2ccbb4888772715c1fa31467d0eb83bf820

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 4.0.0
+  - Remove `scan` from list of options as this is no longer allowed in
+    Elasticsearch 5.0.
+  - Change default query to sort by \_doc, as this replicates the `scan`
+    behavior
+  - Improve documentation to show sort by \_doc, and how to add it to custom
+    queries.
+    
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -12,7 +12,7 @@ require "base64"
 #       # Read all documents from Elasticsearch matching the given query
 #       elasticsearch {
 #         hosts => "localhost"
-#         query => '{ "query": { "match": { "statuscode": 200 } } }'
+#         query => '{ "query": { "match": { "statuscode": 200 } }, "sort": [ "_doc" ] }'
 #       }
 #     }
 #
@@ -23,7 +23,8 @@ require "base64"
 #         "match": {
 #           "statuscode": 200
 #         }
-#       }
+#       },
+#       "sort": [ "_doc" ]
 #     }'
 #
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
@@ -40,20 +41,16 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config :index, :validate => :string, :default => "logstash-*"
 
   # The query to be executed. Read the Elasticsearch query DSL documentation
-  # for more info 
+  # for more info
   # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html
-  config :query, :validate => :string, :default => '{"query": { "match_all": {} } }'
-
-  # Enable the Elasticsearch "scan" search type.  This will disable
-  # sorting but increase speed and performance.
-  config :scan, :validate => :boolean, :default => true
+  config :query, :validate => :string, :default => '{ "sort": [ "_doc" ] }'
 
   # This allows you to set the maximum number of hits returned per scroll.
   config :size, :validate => :number, :default => 1000
 
   # This parameter controls the keepalive time in seconds of the scrolling
   # request and initiates the scrolling process. The timeout applies per
-  # round trip (i.e. between the previous scan scroll request, to the next).
+  # round trip (i.e. between the previous scroll request, to the next).
   config :scroll, :validate => :string, :default => "1m"
 
   # If set, include Elasticsearch document information such as index, type, and
@@ -61,7 +58,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   #
   # It might be important to note, with regards to metadata, that if you're
   # ingesting documents with the intent to re-index them (or just update them)
-  # that the `action` option in the elasticsearch output want's to know how to
+  # that the `action` option in the elasticsearch output wants to know how to
   # handle those things. It can be dynamically assigned with a field
   # added to the metadata.
   #
@@ -117,8 +114,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
       :size => @size
     }
 
-    @options[:search_type] = 'scan' if @scan
-
     transport_options = {}
 
     if @user && @password
@@ -146,14 +141,8 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     # get first wave of data
     r = @client.search(@options)
 
-    # since 'scan' doesn't return data on the search call, do an extra scroll
-    if @scan
-      r = process_next_scroll(output_queue, r['_scroll_id'])
-      has_hits = r['has_hits']
-    else # not a scan, process the response
-      r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
-      has_hits = r['hits']['hits'].any?
-    end
+    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    has_hits = r['hits']['hits'].any?
 
     while has_hits && !stop?
       r = process_next_scroll(output_queue, r['_scroll_id'])

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '3.0.2'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read from an Elasticsearch cluster, based on search query results"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -28,4 +28,3 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/inputs/elasticsearch_spec.rb

@@ -28,7 +28,6 @@ describe LogStash::Inputs::Elasticsearch do
       input {
         elasticsearch {
           hosts => ["localhost"]
-          scan => false
           query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
         }
       }
@@ -74,63 +73,6 @@ describe LogStash::Inputs::Elasticsearch do
     insist { event.get("message") } == [ "ohayo" ]
   end
 
-  it "should retrieve json event from elasticseach with scan" do
-    config = %q[
-      input {
-        elasticsearch {
-          hosts => ["localhost"]
-          scan => true
-          query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
-        }
-      }
-    ]
-
-    scan_response = {
-      "_scroll_id" => "DcrY3G1xff6SB",
-    }
-
-    scroll_responses = [
-      {
-        "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
-        "took" => 27,
-        "timed_out" => false,
-        "_shards" => {
-          "total" => 169,
-          "successful" => 169,
-          "failed" => 0
-        },
-        "hits" => {
-          "total" => 1,
-          "max_score" => 1.0,
-          "hits" => [ {
-            "_index" => "logstash-2014.10.12",
-            "_type" => "logs",
-            "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
-            "_score" => 1.0,
-            "_source" => { "message" => ["ohayo"] }
-          } ]
-        }
-      },
-      {
-        "_scroll_id" => "r453Wc1jh0caLJhSDg",
-        "hits" => { "hits" => [] }
-      }
-    ]
-
-    client = Elasticsearch::Client.new
-    expect(Elasticsearch::Client).to receive(:new).with(any_args).and_return(client)
-    expect(client).to receive(:search).with(any_args).and_return(scan_response)
-    expect(client).to receive(:scroll).with({ :body => "DcrY3G1xff6SB", :scroll => "1m" }).and_return(scroll_responses.first)
-    expect(client).to receive(:scroll).with({ :body=> "cXVlcnlUaGVuRmV0Y2g", :scroll => "1m" }).and_return(scroll_responses.last)
-
-    event = input(config) do |pipeline, queue|
-      queue.pop
-    end
-
-    insist { event }.is_a?(LogStash::Event)
-    insist { event.get("message") } == [ "ohayo" ]
-  end
-
   context "with Elasticsearch document information" do
     let!(:response) do
       {
@@ -150,7 +92,7 @@ describe LogStash::Inputs::Elasticsearch do
             "_type" => "logs",
             "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
             "_score" => 1.0,
-            "_source" => { 
+            "_source" => {
               "message" => ["ohayo"],
               "metadata_with_hash" => { "awesome" => "logstash" },
               "metadata_with_string" => "a string"
@@ -181,7 +123,6 @@ describe LogStash::Inputs::Elasticsearch do
             input {
               elasticsearch {
                 hosts => ["localhost"]
-                scan => false
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
               }
@@ -196,7 +137,6 @@ describe LogStash::Inputs::Elasticsearch do
             input {
               elasticsearch {
                 hosts => ["localhost"]
-                scan => false
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
                 docinfo_target => '#{metadata_field}'
@@ -213,7 +153,7 @@ describe LogStash::Inputs::Elasticsearch do
         expect(event.get("[#{metadata_field}][_id]")).to eq('C5b2xLQwTZa76jBmHIbwHQ')
         expect(event.get("[#{metadata_field}][awesome]")).to eq("logstash")
       end
-      
+
       it 'thows an exception if the `docinfo_target` exist but is not of type hash' do
         metadata_field = 'metadata_with_string'
 
@@ -221,7 +161,6 @@ describe LogStash::Inputs::Elasticsearch do
             input {
               elasticsearch {
                 hosts => ["localhost"]
-                scan => false
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
                 docinfo_target => '#{metadata_field}'
@@ -253,7 +192,6 @@ describe LogStash::Inputs::Elasticsearch do
             input {
               elasticsearch {
                 hosts => ["localhost"]
-                scan => false
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
                 docinfo_target => 'meta'
@@ -275,7 +213,6 @@ describe LogStash::Inputs::Elasticsearch do
             input {
               elasticsearch {
                 hosts => ["localhost"]
-                scan => false
                 query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
                 docinfo => true
                 docinfo_fields => #{fields}
@@ -299,7 +236,6 @@ describe LogStash::Inputs::Elasticsearch do
           input {
             elasticsearch {
               hosts => ["localhost"]
-              scan => false
               query => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }'
             }
           }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -115,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Read from an Elasticsearch cluster, based on search query results