logstash-input-http_poller 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27ad585ea744dd3feb40d3acfe744dabbfb5a122
-  data.tar.gz: 74ecba4a7d356cf7386c62469d6c16091f34f794
+  metadata.gz: 265b825999ea3786b93cc2c6ceff4c2a50800fcd
+  data.tar.gz: 56bb9dc953d316e0d6023e39a6653062fb6ccb23
 SHA512:
-  metadata.gz: 664d8bf1546d7b866c623b14d6a7dfeb799becb06e223a51510aed06cc2d2c3e3a0ca6c1e512e14e74dc96799624a34b83f1ab643d02a7ce1112b162866be04d
-  data.tar.gz: fd6716eaa478426b08494312a7e2939537d652780fa6455f58ef1bac3f01547f01a380d425fea9b7867535cd58277ff120f0f17e0775e4f11e6a0b664e10420f
+  metadata.gz: d97e0bcafd842865eaa04d9d64e18c6d90414daa8213f620797406023965a58b19e652910cb2cc1e3000440b3f4ef6769f7f892cdb307bbc0c2a9de68ce6f968
+  data.tar.gz: 7ba774a15dafbb3f193989c7f11fefac339b99b763eeff997e616cf32ffda7224a2d3821a2c5d13375a3ff2b0b25713c1a298e115fbaef00adc5295db98e168b

data/CHANGELOG.md

@@ -1,2 +1,5 @@
+* 1.0.1
+  - Add 'target' option
+  
 * 1.0.0
   - Initial release

data/examples/ha_proxy.conf

@@ -22,7 +22,7 @@ filter {
   }
 
   csv {
-    columns => [ pxname,svname,qcur,qmax,scur,smax,slim,stot,bin,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime ]
+    columns => [ pxname,svname,qcur,qmax,scur,smax,slim,stot,binhttp://localhost:9201/haproxy?stats;csvm,bout,dreq,dresp,ereq,econ,eresp,wretr,wredis,status,weight,act,bck,chkfail,chkdown,lastchg,downtime,qlimit,pid,iid,sid,throttle,lbtot,tracked,type,rate,rate_lim,rate_max,check_status,check_code,check_duration,hrsp_1xx,hrsp_2xx,hrsp_3xx,hrsp_4xx,hrsp_5xx,hrsp_other,hanafail,req_rate,req_rate_max,req_tot,cli_abrt,srv_abrt,comp_in,comp_out,comp_byp,comp_rsp,lastsess,last_chk,last_agt,qtime,ctime,rtime,ttime ]
   }
 
   # This is a pointless CSV header
@@ -35,4 +35,9 @@ output {
   stdout {
     codec => rubydebug
   }
+
+  elasticsearch {
+    protocol => "http"
+    cluster => "avc-cluster"
+  }
 }

data/examples/local_elasticsearch.conf

@@ -1,28 +1,10 @@
 input {
-  http_poller {
-    urls => {
-      test1 => "http://localhost:9200"
-      test2 => {
-        # Supports all options supported by ruby's Manticore HTTP client
-        method => get
-        url => "http://localhost:9200/_cluster/health"
-        headers => {
-          Accept => "application/json"
-        }
-        auth => {
-          user => "AzureDiamond"
-          pass => "hunter2"
-        }
-      }
-    }
-    request_timeout => 60
-    interval => 60
-    codec => "json"
-    # A hash of request metadata info (timing, response headers, etc.) will be sent here
-    metadata_target => "_http_poller_metadata"
-  }
+  stdin {}
 }
 
 output {
-  elasticsearch {}
+  elasticsearch {
+    protocol => http
+    host => "127.0.0.1"
+  }
 }

data/lib/logstash/inputs/http_poller.rb

@@ -56,6 +56,9 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   # How often  (in seconds) the urls will be called
   config :interval, :validate => :number, :required => true
 
+  # Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
+  config :target, :validate => :string
+
   # If you'd like to work with the request/response metadata
   # Set this value to the name of the field you'd like to store a nested
   # hash of metadata.
@@ -132,21 +135,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
       request_async(queue, name, request)
     end
 
-    # TODO: Remove this once our patch to manticore is accepted. The real callback should work
-    # Some exceptions are only returned here! There is no callback,
-    # for example, if there is a bad port number.
-    # https://github.com/cheald/manticore/issues/22
-    # This issue is now fixed, but we'll leave this code in for older versions
-    # of manticore. Once the plugin ecosystem is more updated we can kill this.
-    client.execute!.each_with_index do |resp, i|
-      if resp.is_a?(java.lang.Exception) || resp.is_a?(StandardError)
-        name = @requests.keys[i]
-        request = @requests[name]
-        # We can't report the time here because this is as slow as the slowest request
-        # This is all temporary code anyway
-        handle_failure(queue, name, request, resp, nil)
-      end
-    end
+    client.execute!
   end
 
   private
@@ -165,7 +154,8 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   def handle_success(queue, name, request, response, execution_time)
     @codec.decode(response.body) do |decoded|
-      handle_decoded_event(queue, name, request, response, decoded, execution_time)
+      event = @target ? LogStash::Event.new(@target => decoded.to_hash) : decoded
+      handle_decoded_event(queue, name, request, response, event, execution_time)
     end
   end
 
@@ -184,6 +174,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   end
 
   private
+  # Beware, on old versions of manticore some uncommon failures are not handled
   def handle_failure(queue, name, request, exception, execution_time)
     event = LogStash::Event.new
     apply_metadata(event, name, request)
@@ -196,14 +187,16 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
       "url" => @urls[name], # We want the exact parameter they passed in
       "name" => name,
       "error" => exception.to_s,
+      "backtrace" => exception.backtrace,
       "runtime_seconds" => execution_time
    }
 
     queue << event
   rescue StandardError, java.lang.Exception => e
       @logger.error? && @logger.error("Cannot read URL or send the error as an event!",
-                                      :exception => exception,
-                                      :exception_message => exception.message,
+                                      :exception => e,
+                                      :exception_message => e.message,
+                                      :exception_backtrace => e.backtrace,
                                       :name => name,
                                       :url => request
       )

data/logstash-input-http_poller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-http_poller'
-  s.version         = '1.0.0'
+  s.version         = '1.0.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Poll HTTP endpoints with Logstash."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program."

data/spec/inputs/http_poller_spec.rb

@@ -209,7 +209,6 @@ describe LogStash::Inputs::HTTP_Poller do
       end
     end
 
-
     describe "a valid request and decoded response" do
       let(:payload) { {"a" => 2, "hello" => ["a", "b", "c"]} }
       let(:opts) { default_opts }
@@ -280,15 +279,18 @@ describe LogStash::Inputs::HTTP_Poller do
           expect(event.to_hash).to include(payload)
         end
       end
-    end
-  end
-
-
-
-
-
-
-
 
+      context "with a specified target" do
+        let(:target) { "mytarget" }
+        let(:opts) { default_opts.merge("target" => target) }
 
+        it "should store the event info in the target" do
+          # When events go through the pipeline they are java-ified
+          # this normalizes the payload to java types
+          payload_normalized = LogStash::Json.load(LogStash::Json.dump(payload))
+          expect(event[target]).to include(payload_normalized)
+        end
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - andrewvc
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-15 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core