logstash-input-elasticsearch 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 110f1639b58075d9b13cea8d31511765b3dfab53
-  data.tar.gz: e8c9ed5e0f4519793a127710ba3452e2f78cdd8a
+  metadata.gz: f8380a3a71bd40498ef5838731f767fdcd095b75
+  data.tar.gz: e41dc06f80647729e588a1c36549269ca8aad48c
 SHA512:
-  metadata.gz: bf3dabbc420ca79eca5464eb59734a28808cf9f65f5f621345cced1b8c0687696e7e8560b1220baff445db7830e41cc77cb94018b94d5bc39370def325d27692
-  data.tar.gz: b8f5b58078b43ca6975882f99dde9ee6a040b8f13facd244a3f9ea1d4ba6ff79f973e49194f5c710dbf008d419bf700b746a187d7f88943f02be4512deffcb19
+  metadata.gz: 6e8802b10b181d74e845ca63ac6479c7d8d56bc012d0c163f7b84d10733b9acbd494b2de94c8960aa64d9eb6739b2d11babec180b0d64fa6c3a8d26528c08198
+  data.tar.gz: dd2988ee5fd12baf2b1753330cb4716797f62e97b9f34f9961615f60790fadaa19c7f09090b6aafa9b3d10f61446ff30f3f60e0f00176fec20d7a8bea8c0d574

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 1.0.1
+ - refactor request logic into own method (better memory gc perf)

data/README.md

@@ -1,15 +1,15 @@
 # Logstash Plugin
 
-This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
 
 - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
 
 ## Need Help?
 
@@ -83,4 +83,4 @@ Programming is not a required skill. Whatever you've seen about open source and
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/elasticsearch.rb

@@ -11,7 +11,7 @@ require "base64"
 #     input {
 #       # Read all documents from Elasticsearch matching the given query
 #       elasticsearch {
-#         host => "localhost"
+#         hosts => "localhost"
 #         query => '{ "query": { "match": { "statuscode": 200 } } }'
 #       }
 #     }
@@ -32,11 +32,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   default :codec, "json"
 
   # List of elasticsearch hosts to use for querying.
+  # each host can be either IP, HOST, IP:port or HOST:port
+  # port defaults to 9200
   config :hosts, :validate => :array
 
-  # The HTTP port of your Elasticsearch server's REST interface.
-  config :port, :validate => :number, :default => 9200
-
   # The index or alias to search.
   config :index, :validate => :string, :default => "logstash-*"
 
@@ -68,7 +67,7 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # [source, ruby]
   #     input {
   #       elasticsearch {
-  #         host => "es.production.mysite.org"
+  #         hosts => "es.production.mysite.org"
   #         index => "mydata-2018.09.*"
   #         query => "*"
   #         size => 500
@@ -127,7 +126,10 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     end
 
     hosts = if @ssl then
-      @hosts.map { |h| { :host => h, :scheme => 'https' } }
+      @hosts.map do |h|
+        host, port = h.split(":")
+        { :host => host, :scheme => 'https', :port => port }
+      end
     else
       @hosts
     end
@@ -138,6 +140,32 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
   end
+
+  private
+  def run_next(output_queue, scroll_id)
+    r = scroll_request(scroll_id)
+    r['hits']['hits'].each do |hit|
+      event = LogStash::Event.new(hit['_source'])
+      decorate(event)
+
+      if @docinfo
+        event[@docinfo_target] ||= {}
+
+        unless event[@docinfo_target].is_a?(Hash)
+          @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the `@metadata` field in the `_source` document, expected a hash got:", :metadata_type => event[@docinfo_target].class)
+
+          raise Exception.new("Elasticsearch input: incompatible event")
+        end
+
+        @docinfo_fields.each do |field|
+          event[@docinfo_target][field] = hit[field]
+        end
+      end
+      output_queue << event
+    end
+
+    {:has_hits => r['hits']['hits'].any?, :scroll_id => r['_scroll_id']}
+  end
   
   public
   def run(output_queue)
@@ -147,31 +175,11 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
 
     # since 'scan' doesn't return data on the search call, do an extra scroll
     if @scan
-      r = scroll_request(r['_scroll_id'])
+      resp = run_next(output_queue, r['_scroll_id'])
     end
 
-    while r['hits']['hits'].any? do
-      r['hits']['hits'].each do |hit|
-        event = LogStash::Event.new(hit['_source'])
-        decorate(event)
-
-        if @docinfo
-          event[@docinfo_target] ||= {}
-
-          unless event[@docinfo_target].is_a?(Hash)
-            @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the `@metadata` field in the `_source` document, expected a hash got:", :metadata_type => event[@docinfo_target].class)
-
-            raise Exception.new("Elasticsearch input: incompatible event") 
-          end
-
-          @docinfo_fields.each do |field|
-            event[@docinfo_target][field] = hit[field]
-          end
-        end
-
-        output_queue << event
-      end
-      r = scroll_request(r['_scroll_id'])
+    while resp[:has_hits] do
+      resp = run_next(output_queue, resp[:scroll_id])
     end
   end # def run
 

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read from an Elasticsearch cluster, based on search query results"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-24 00:00:00.000000000 Z
+date: 2015-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core