logstash-input-httpclient 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dfffe002abaff65c15fae28f60bdb26c96a49296
+  data.tar.gz: a0ccb1d87c692aa0a408d1565000a7f758fc28d6
+SHA512:
+  metadata.gz: a2d781b3211acb8ccf50e73c1fa8d64fc3edcd8e15d176df3f1787c5596277a3e8cdad396cafb82129858715742709b623284191f670475ca596ca779290bf6e
+  data.tar.gz: 41cf27bfe287f24232626b2ee3d82a3a63f53adb5d0dcba29ea836b1e87a4a3254ecbfa978ba60d5f52b13f9df386a652aa330ca9d526be949efac2ef9c3adaf

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+Gemfile.lock
+Gemfile.bak
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elastic/logstash", :branch => "1.4"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,28 @@
+# Logstash HTTP Client Input Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+This plugin will open up a http keep-alive connection to any http server and periodically execute GET requests. The response body will be added to the `message` field of the logstash event.
+
+This plugin also adds a `X-Logstash-Avg-Queue-Secs` header to every GET request. This indicates the average amount of "wait time" an event is waiting to be added to logstash's intenal bounded/blocking input queue. The server could use this value to determine how many events to send to logstash based on the throughput/wait time.
+
+## Installing
+`{logstash_dir}/bin/plugin install logstash-input-httpclient`
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/inputs/httpclient.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require "net/http"
+
+# Query a http server with GET requests at a regular interval. 
+#
+#[%hardbreaks]
+# HTTP GET requests are performed using the provided url (which may contain URL parameters).
+# The HTTP Response body will be added to "message" after being decoded by the provided codec.
+# There is an additional header named `X-Logstash-Avg-Queue-Secs` added to each HTTP request. This is the
+# average seconds it took for the past 20 events to be accepted by Logstash's bounded input queue. 
+# The HTTP server could use this as a indication of how many events the Logstash instance is able to process (throughput).
+
+
+class LogStash::Inputs::HttpClient < LogStash::Inputs::Base
+  config_name "httpclient"
+
+  # If undefined, Logstash will complain, even if codec is unused.
+  default :codec, "plain" 
+
+  #[%hardbreaks]
+  # The full url to execute GET requests on. 
+  # It may contain URL parameters (you are responsible for URL encoding them)
+  # e.g. `https://mywebservice.int/`
+  # e.g. `http://mywebservice.int?queue=logstash_events&numevents=10`
+  #
+  # IMPORTANT: make sure you end with a slash (if not using url params). `http://mywebservice.int` is not valid, it must be `http://myswebservice.int/`
+  # todo: auto-append slash if user forgets
+  config :url, :validate => :string, :required => true
+
+  # Set how frequently we should query the url for messages
+  #
+  # The default, `10`, means send a message every 10 seconds.
+  config :interval, :validate => :number, :default => 10
+  
+  # The name of the object that http response meta-data (headers, response code, etc) should be added to.
+  config :response_object_name, :validate => :string, :default => "http_response"
+  
+  # Should the http headers be added to the `response_object_name`?
+  #
+  # If true, there will be a "headers" with all the response headers/values.
+  config :include_response_headers, :validate => :boolean, :default => false
+  
+  # Should the http response code be added to the `response_object_name`?
+  #
+  # If true, there will be a "code" with all the response headers/values.
+  config :include_response_code, :validate => :boolean, :default => true
+  
+  # Should the http request elapsed time be added to the `response_object_name`?
+  #
+  # If true, there will be a "took_secs" with the time it took for the http request to complete
+  config :include_http_request_time, :validate => :boolean, :default => true
+  
+  public
+  def register
+    @uri = URI(@url)
+  end # def register
+
+  def run(queue)
+    queue_times = Array.new #track an average of how long it's taking to queue events into logstash
+    #::start creates a connection to the HTTP server and keeps it alive for the duration
+    Net::HTTP.start @uri.host, @uri.port, :use_ssl => @uri.scheme == 'https' do |http|
+      Stud.interval(@interval) do
+        begin
+          http_start = Time.now
+          request = Net::HTTP::Get.new(@uri.path)          
+          request ["X-Logstash-Avg-Queue-Secs"] = arr_avg(queue_times, 20, 3)
+          response = http.request request  # Net::HTTPResponse object
+          http_elapsed = Time.now - http_start
+        rescue => e
+          @logger.warn("Http request failed, will retry", :exception => e)
+          @logger.warn(e.backtrace)
+          sleep(@interval)
+          retry
+        end
+        #event = LogStash::Event.new("message" => response.body)
+        @codec.decode(response.body) do |event|
+          event[@response_object_name] = {}
+          if @include_response_headers 
+            event[@response_object_name]["headers"] = response
+          end
+          if @include_response_code 
+            event[@response_object_name]["code"] = response.code
+          end
+          if @include_http_request_time 
+            event[@response_object_name]["took_secs"] = http_elapsed
+          end
+          decorate(event)
+          queue_start = Time.now
+          queue << event
+          queue_elapsed = Time.now - http_start
+          queue_times.push queue_elapsed
+        end
+      end #interval loop
+    end #HTTP keepalive
+  end # def run
+  
+  def arr_avg(arr,cutoff,precision)
+    if(arr.size == 0)
+      return 0
+    end
+    if(arr.size > cutoff)
+      arr.drop(arr.length-cutoff) #removes from beginning of array
+    end
+    #get the average
+    (arr.inject{ |sum, el| sum + el }.to_f / arr.size).round(precision)
+  end
+end # class LogStash::Inputs::Example

data/logstash-input-httpclient.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-httpclient'
+  s.version         = '0.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "This input queries a http url at a regular interval and sends the responses to logstash."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["bradvido"]
+  s.email = 'bradvido+logstash-input-httpclient@bradvido.com'
+  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/inputs/httpclient_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,116 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-httpclient
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- bradvido
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-06-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: bradvido+logstash-input-httpclient@bradvido.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/inputs/httpclient.rb
+- logstash-input-httpclient.gemspec
+- spec/inputs/httpclient_spec.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.6
+signing_key:
+specification_version: 4
+summary: This input queries a http url at a regular interval and sends the responses to logstash.
+test_files:
+- spec/inputs/httpclient_spec.rb