logstash-input-burrow 1.0.0

data/lib/logstash/inputs/burrow.rb

@@ -0,0 +1,280 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/plugin_mixins/http_client"
+require "socket" # for Socket.gethostname
+require "manticore"
+require "rufus/scheduler"
+
+class LogStash::Inputs::Burrow < LogStash::Inputs::Base
+  include LogStash::PluginMixins::HttpClient
+
+  config_name "burrow"
+
+  # The Burrow Client configuration, with at least url
+  # client_config => {
+  #         url => "http://localhost:8000"
+  # }
+  config :client_config, :validate => :hash, :required => true
+  # The Burrow API version
+  config :api_version, :default => "v3"
+  # Schedule of when to periodically poll from the url
+  # Format: A hash with
+  #   + key: "cron" | "every" | "in" | "at"
+  #   + value: string
+  # Examples:
+  #   a) { "every" => "1h" }
+  #   b) { "cron" => "* * * * * UTC" }
+  # See: rufus/scheduler for details about different schedule options and value string format
+  config :schedule, :validate => :hash, :default => %w(every 30s)
+
+  default :codec, "json"
+
+  Schedule_types = %w(cron every at in)
+
+  def register
+    @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
+
+    @logger.info("Registering burrow Input", :type => @type, :schedule => @schedule, :timeout => @timeout)
+
+    if @api_version != "v3"
+      raise LogStash::ConfigurationError, "at the moment only Burrow API version v3 is supported."
+    end
+
+    if @codec.class.config_name != "json"
+      raise LogStash::ConfigurationError, "this plugin needs codec to be json."
+    end
+
+    setup_request!
+  end
+
+  def stop
+    Stud.stop!(@interval_thread) if @interval_thread
+    @scheduler.stop if @scheduler
+  end
+
+  private
+
+  def setup_request!
+    @request = normalize_request(client_config)
+  end
+
+  def normalize_request(client_spec)
+    if client_spec.is_a?(String)
+      res = [client_spec + ('/' unless client_spec.end_with?('/')) + @api_version << "/kafka"]
+    elsif client_spec.is_a?(Hash)
+      # The client will expect keys / values
+      spec = Hash[client_spec.clone.map {|k, v| [k.to_sym, v]}] # symbolize keys
+
+      # method and url aren't really part of the options, so we pull them out
+      spec.delete(:method)
+      url = spec.delete(:url)
+
+      raise LogStash::ConfigurationError, "Invalid URL #{url}" unless URI::DEFAULT_PARSER.regexp[:ABS_URI].match(url)
+
+      # Manticore wants auth options that are like {:auth => {:user => u, :pass => p}}
+      # We allow that because earlier versions of this plugin documented that as the main way to
+      # to do things, but now prefer top level "user", and "password" options
+      # So, if the top level user/password are defined they are moved to the :auth key for manticore
+      # if those attributes are already in :auth they still need to be transformed to symbols
+      auth = spec[:auth]
+      user = spec.delete(:user) || (auth && auth["user"])
+      password = spec.delete(:password) || (auth && auth["password"])
+
+      if user.nil? ^ password.nil?
+        raise LogStash::ConfigurationError, "'user' and 'password' must both be specified for input HTTP poller!"
+      end
+
+      if user && password
+        spec[:auth] = {
+            user: user,
+            pass: password,
+            eager: true
+        }
+      end
+      res = [url + ('/' unless url.end_with?('/')) + @api_version << "/kafka", spec]
+    else
+      raise LogStash::ConfigurationError, "Invalid URL or request spec: '#{client_spec}', expected a String or Hash!"
+    end
+
+    validate_request!(res)
+    res
+  end
+
+  private
+
+  def validate_request!(request)
+    url, spec = request
+
+    raise LogStash::ConfigurationError, "Invalid URL #{url}" unless URI::DEFAULT_PARSER.regexp[:ABS_URI].match(url)
+
+    raise LogStash::ConfigurationError, "No URL provided for request! #{url}" unless url
+    if spec && spec[:auth]
+      unless spec[:auth][:user]
+        raise LogStash::ConfigurationError, "Auth was specified, but 'user' was not!"
+      end
+      unless spec[:auth][:pass]
+        raise LogStash::ConfigurationError, "Auth was specified, but 'password' was not!"
+      end
+    end
+
+    request
+  end
+
+  public
+
+  def run(queue)
+    setup_schedule(queue)
+  end
+
+  def setup_schedule(queue)
+    #schedule hash must contain exactly one of the allowed keys
+    msg_invalid_schedule = "Invalid config. schedule hash must contain " +
+        "exactly one of the following keys - cron, at, every or in"
+    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length != 1
+    schedule_type = @schedule.keys.first
+    schedule_value = @schedule[schedule_type]
+    raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
+
+    @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
+    #as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
+    opts = schedule_type == "every" ? {:first_in => 0.01} : {}
+    @scheduler.send(schedule_type, schedule_value, opts) {run_once(queue)}
+    @scheduler.join
+  end
+
+  def run_once(queue)
+    request(queue, @request)
+  end
+
+
+  private
+
+  def request(queue, request)
+    @logger.debug? && @logger.debug?("Fetching URL", :client_config => request)
+    started = Time.now
+
+    url, spec = request
+    client.get(url, spec).
+        on_success do |cluster_response|
+      body = cluster_response.body
+      if body && body.size > 0
+        @codec.decode(body) do |clusters|
+          cluster_list = clusters.get('clusters')
+          @logger.debug? && @logger.debug?("found clusters", :cluster_list => cluster_list)
+          cluster_list.each {|cluster|
+            consumer_url = url + "/" << cluster + "/consumer"
+            client.get(consumer_url, spec).
+                on_success do |consumers_response|
+              body = consumers_response.body
+              if body && body.size > 0
+                @codec.decode(body) do |consumers|
+                  consumer_list = consumers.get('consumers')
+                  @logger.debug? && @logger.debug?("found consumers", :consumer_list => consumer_list)
+                  consumer_list.each {|consumer|
+                    lag_url = url + "/" + cluster + "/consumer" + "/" + consumer + "/lag"
+                    client.get(lag_url, spec).
+                        on_success do |consumer_lag_response|
+                      handle_success(queue, [lag_url, spec], consumer_lag_response, Time.now - started)
+                    end.
+                        on_failure do |exception|
+                      handle_failure(queue, [lag_url, spec], exception, Time.now - started)
+                    end.call
+                  } unless consumer_list.nil?
+                end
+              end
+            end.
+                on_failure do |exception|
+              handle_failure(queue, [consumer_url, spec], exception, Time.now - started)
+            end.call
+          } unless cluster_list.nil?
+        end
+      end
+    end.
+        on_failure do |exception|
+      handle_failure(queue, request, exception, Time.now - started)
+    end.call
+  end
+
+  private
+
+  def handle_success(queue, request, response, execution_time)
+    body = response.body
+    # If there is a usable response. HEAD request are `nil` and empty get
+    # responses come up as "" which will cause the codec to not yield anything
+    if body && body.size > 0
+      decode_and_flush(@codec, body) do |decoded|
+        event = decoded
+        handle_decoded_event(queue, request, response, event, execution_time)
+      end
+    else
+      event = ::LogStash::Event.new
+      handle_decoded_event(queue, request, response, event, execution_time)
+    end
+  end
+
+  private
+
+  def decode_and_flush(codec, body, &yielder)
+    codec.decode(body, &yielder)
+    codec.flush(&yielder)
+  end
+
+  private
+
+  def handle_decoded_event(queue, request, response, event, _execution_time)
+    decorate(event)
+    queue << event
+  rescue StandardError, java.lang.Exception => e
+    @logger.error? && @logger.error("Error eventifying response!",
+                                    :exception => e,
+                                    :exception_message => e.message,
+                                    :client_config => request,
+                                    :response => response
+    )
+  end
+
+  private
+
+  # Beware, on old versions of manticore some uncommon failures are not handled
+  def handle_failure(queue, request, exception, execution_time)
+    event = LogStash::Event.new
+
+    event.tag("_http_request_failure")
+
+    # This is also in the metadata, but we send it anyone because we want this
+    # persisted by default, whereas metadata isn't. People don't like mysterious errors
+    event.set("http_request_failure", {
+        "request" => structure_request(request),
+        "error" => exception.to_s,
+        "backtrace" => exception.backtrace,
+        "runtime_seconds" => execution_time
+    })
+
+    queue << event
+  rescue StandardError, java.lang.Exception => e
+    @logger.error? && @logger.error("Cannot read URL or send the error as an event!",
+                                    :exception => e,
+                                    :exception_message => e.message,
+                                    :exception_backtrace => e.backtrace)
+
+    # If we are running in debug mode we can display more information about the
+    # specific request which could give more details about the connection.
+    @logger.debug? && @logger.debug("Cannot read URL or send the error as an event!",
+                                    :exception => e,
+                                    :exception_message => e.message,
+                                    :exception_backtrace => e.backtrace,
+                                    :client_config => request)
+  end
+
+  private
+
+  # Turn [method, url, spec] request into a hash for friendlier logging / ES indexing
+  def structure_request(request)
+    url, spec = request
+    # Flatten everything into the 'spec' hash, also stringify any keys to normalize
+    Hash[(spec || {}).merge({
+                                "url" => url,
+                            }).map {|k, v| [k.to_s, v]}]
+  end
+end

data/logstash-input-burrow.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+  s.name        = 'logstash-input-burrow'
+  s.version         = '1.0.0'
+  s.licenses    = ['Apache License (2.0)']
+  s.summary     = "Decodes the output of Burrow HTTP API into events"
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+  s.authors     = [ "markush81"]
+  s.email       = 'info@markushelbig.de'
+  s.homepage    = "https://github.com/markush81/logstash-input-burrow"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-mixin-http_client', ">= 6.0.0", "< 7.0.0"
+  s.add_runtime_dependency 'stud', "~> 0.0.22"
+  s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
+
+  s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'logstash-codec-line'
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'flores'
+  s.add_development_dependency 'timecop'
+end

data/spec/inputs/burrow_spec.rb

@@ -0,0 +1,281 @@
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/inputs/burrow'
+require 'flores/random'
+require "timecop"
+# Workaround for the bug reported in https://github.com/jruby/jruby/issues/4637
+require 'rspec/matchers/built_in/raise_error.rb'
+
+describe LogStash::Inputs::Burrow do
+  let(:queue) {Queue.new}
+  let(:default_schedule) {
+    {"every" => "30s"}
+  }
+
+  let(:default_url) {"http://localhost:8000"}
+  let(:default_client_config) {{"url" => default_url}}
+  let(:default_opts) {
+    {
+        "schedule" => default_schedule,
+        "client_config" => default_client_config
+    }
+  }
+  let(:klass) {LogStash::Inputs::Burrow}
+
+  describe "instances" do
+    subject {klass.new(default_opts)}
+
+    before do
+      subject.register
+    end
+
+    describe "#run" do
+      it "should setup a scheduler" do
+        runner = Thread.new do
+          subject.run(double("queue"))
+          expect(subject.instance_variable_get("@scheduler")).to be_a_kind_of(Rufus::Scheduler)
+        end
+        runner.kill
+        runner.join
+      end
+    end
+
+    describe "#run_once" do
+      it "should issue an request for each url" do
+        normalized_request = subject.send(:normalize_request, default_client_config)
+        expect(subject).to receive(:request).with(queue, normalized_request).once
+
+        subject.send(:run_once, queue) # :run_once is a private method
+      end
+    end
+
+    describe "normalizing a request spec" do
+      shared_examples "a normalized request" do
+        it "should set the options to the URL string" do
+          expect(normalized[0]).to eql(spec_url + "/v3/kafka")
+        end
+      end
+
+      let(:normalized) {subject.send(:normalize_request, client_config)}
+
+      describe "a string URL" do
+        let(:client_config) {default_client_config}
+        let(:spec_url) {default_url}
+
+        include_examples("a normalized request")
+      end
+
+      describe "URL specs" do
+
+        context "missing an URL" do
+          let(:client_config) {}
+
+          it "should raise an error" do
+            expect {normalized}.to raise_error(LogStash::ConfigurationError)
+          end
+        end
+
+        shared_examples "auth" do
+          context "with auth enabled but no pass" do
+            let(:auth) {{"user" => "foo"}}
+
+            it "should raise an error" do
+              expect {normalized}.to raise_error(LogStash::ConfigurationError)
+            end
+          end
+
+          context "with auth enabled, a path, but no user" do
+            let(:client_config) {{"auth" => {"password" => "bar"}}}
+            it "should raise an error" do
+              expect {normalized}.to raise_error(LogStash::ConfigurationError)
+            end
+          end
+          context "with auth enabled correctly" do
+            let(:auth) {{"user" => "foo", "password" => "bar"}}
+
+            it "should raise an error" do
+              expect {normalized}.not_to raise_error
+            end
+
+            it "should properly set the auth parameter" do
+              expect(normalized[1][:auth]).to eq({
+                                                     :user => auth["user"],
+                                                     :pass => auth["password"],
+                                                     :eager => true
+                                                 })
+            end
+          end
+        end
+
+        # The new 'right' way to do things
+        describe "auth with direct auth options" do
+          let(:client_config) {{"url" => "http://localhost", "user" => auth["user"], "password" => auth["password"]}}
+
+          include_examples("auth")
+        end
+      end
+    end
+
+    describe "#structure_request" do
+      it "Should turn a simple request into the expected structured request" do
+        expected = {"url" => "http://example.net"}
+        expect(subject.send(:structure_request, ["http://example.net"])).to eql(expected)
+      end
+
+      it "should turn a complex request into the expected structured one" do
+        headers = {
+            "X-Fry" => " Like a balloon, and... something bad happens! "
+        }
+        expected = {
+            "url" => "http://example.net",
+            "headers" => headers
+        }
+        expect(subject.send(:structure_request, ["http://example.net", {"headers" => headers}])).to eql(expected)
+      end
+    end
+  end
+
+  describe "events" do
+
+    shared_examples "unprocessable_requests" do
+      let(:burow) {LogStash::Inputs::Burrow.new(settings)}
+      subject(:event) {
+        burow.send(:run_once, queue)
+        queue.pop(true)
+      }
+
+      before do
+        burow.register
+        allow(burow).to receive(:handle_failure).and_call_original
+        allow(burow).to receive(:handle_success)
+        event # materialize the subject
+      end
+
+      it "should enqueue a message" do
+        expect(event).to be_a(LogStash::Event)
+      end
+
+      it "should enqueue a message with 'http_request_failure' set" do
+        expect(event.get("http_request_failure")).to be_a(Hash)
+      end
+
+      it "should tag the event with '_http_request_failure'" do
+        expect(event.get("tags")).to include('_http_request_failure')
+      end
+
+      it "should invoke handle failure exactly once" do
+        expect(burow).to have_received(:handle_failure).once
+      end
+
+      it "should not invoke handle success at all" do
+        expect(burow).not_to have_received(:handle_success)
+      end
+    end
+
+    context "with a non responsive server" do
+      context "due to a non-existant host" do # Fail with handlers
+        let(:url) {"http://thouetnhoeu89ueoueohtueohtneuohn"}
+        let(:code) {nil} # no response expected
+
+        let(:settings) {default_opts.merge("client_config" => {"url" => url})}
+
+        include_examples("unprocessable_requests")
+      end
+
+      context "due to a bogus port number" do # fail with return?
+        let(:invalid_port) {Flores::Random.integer(65536..1000000)}
+
+        let(:url) {"http://127.0.0.1:#{invalid_port}"}
+        let(:settings) {default_opts.merge("client_config" => {"url" => url})}
+        let(:code) {nil} # No response expected
+
+        include_examples("unprocessable_requests")
+      end
+    end
+
+    describe "a valid request and decoded response" do
+      let(:cluster) {{"clusters" => ["default"]}}
+      let(:consumer) {{"consumers" => ["console-1"]}}
+      let(:lag) {{"message" => "consumer status returned"}}
+
+      let(:opts) {default_opts}
+      let(:instance) {
+        klass.new(opts)
+      }
+      let(:code) {202}
+
+      subject(:event) {
+        queue.pop(true)
+      }
+
+      before do
+        instance.register
+        instance.client.stub(default_url + "/v3/kafka",
+                             :body => LogStash::Json.dump(cluster),
+                             :code => code)
+
+        instance.client.stub(default_url + "/v3/kafka/default/consumer",
+                             :body => LogStash::Json.dump(consumer),
+                             :code => code)
+
+        instance.client.stub(default_url + "/v3/kafka/default/consumer/console-1/lag",
+                             :body => LogStash::Json.dump(lag),
+                             :code => code)
+
+        allow(instance).to receive(:decorate)
+        instance.send(:run_once, queue)
+      end
+
+      it "should have a matching message" do
+        expect(event.to_hash).to include(lag)
+      end
+
+      it "should decorate the event" do
+        expect(instance).to have_received(:decorate).once
+      end
+
+      context "with empty cluster" do
+        let(:cluster) {{ "clusters" => [] }}
+
+        it "should have no event" do
+          expect(instance).to have_received(:decorate).exactly(0).times
+        end
+      end
+
+      context "with empty consumers" do
+        let(:consumer) {{ "consumers" => [] }}
+
+        it "should have no event" do
+          expect(instance).to have_received(:decorate).exactly(0).times
+        end
+      end
+
+      context "with a complex URL spec" do
+        let(:client_config) {
+          {
+              "url" => default_url,
+              "headers" => {
+                  "X-Fry" => "I'm having one of those things, like a headache, with pictures..."
+              }
+          }
+        }
+        let(:opts) {
+          {
+              "schedule" => {
+                  "cron" => "* * * * * UTC"
+              },
+              "client_config" => client_config
+          }
+        }
+
+        it "should have a matching message" do
+          expect(event.to_hash).to include(lag)
+        end
+      end
+    end
+  end
+
+  describe "stopping" do
+    let(:config) {default_opts}
+    it_behaves_like "an interruptible input plugin"
+  end
+end