logstash-output-prometheus 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 667c219d828f53a1a0a6e3949297b5a7ff9f2561db62332f7d286c4abde345fd
+  data.tar.gz: 6043bc410be20a1dcc2722134d2cccb15d4d2715488e881098e52741bae63057
+SHA512:
+  metadata.gz: a842d58037e05203313574bd5c9180c4a4255918a9f4321c0618728b8d38c34154d16d093db7c9bbf2475fd45cc49b2b44881a5df0adb4c38eaeb78aaa1bc4d3
+  data.tar.gz: ad7f9ddc0b44a250223994af94cc285ce5d8ec191707c9f29b08b908fc3b7d4da57a84ab422fcdc92053a24a1b03dd60ff4ef58e6b6a9178e72faaf35bae715a

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 0.1.0
+  - Plugin created with the logstash plugin generator

data/CONTRIBUTORS

@@ -0,0 +1,10 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Spencer Malone - smalone@rsglab.com
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-output-prometheus
+Example output plugin. This should help bootstrap your effort to write your own output plugin!

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+gem 'logstash-core'
+gem 'logstash-core-plugin-api'
+
+gem 'prometheus-client', "0.10.0.pre.alpha.2"
+gem "rack", ">= 1.6.11"
+
+gem 'logstash-devutils', ">= 1.3.1", :group => [:development]

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,100 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+This plugin allows you to expose metrics from logstash to a prometheus exporter, hosted by your logstash instance.
+
+
+## Building
+
+### Requirements
+- JRuby
+- JDK
+- Git
+- bundler
+
+### Build steps
+
+`./script/build`
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
+
+## Examples
+
+```
+logstash -e 'input { stdin { } } 
+filter {
+	mutate {
+		add_field => {
+			"timer" => "%{[message]}"
+		}
+	}
+	mutate {
+		convert      => { "timer" => "float" }
+	}
+}
+output {
+	prometheus {
+		timer => {
+			histogramtest => {
+				description => "This is my histogram"
+				value => "%{[timer]}"
+				type => "histogram"
+				buckets => [0.1, 1, 5, 10]
+				labels => {
+					mylabel => "testlabel" 
+				}
+			}
+		}
+	}
+	prometheus {
+		port => 9641 # You can run multiple ports if you want different outputs scraped by different prom instances.
+		timer => {
+			summarytest => {
+				description => "This is my summary"
+				value => "%{[timer]}"
+				type => "summary"
+			}
+		}
+	}
+}'
+```
+
+```
+logstash -e 'input { stdin { } } 
+output {
+	prometheus {
+		increment => {
+			mycounter => {
+				description => "This is my test counter"
+				labels => {
+					value => "%{[message]}" 
+				}
+				type => "counter"
+			}
+		}
+	}
+
+	prometheus {
+		increment => {
+			totaleventscustom => {
+				description => "This is my second test counter"
+			}
+		}
+	}
+}'
+```
+
+## Things to keep in mind
+
+As outlined in 

data/lib/logstash/outputs/prometheus.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require 'rack'
+require 'prometheus/middleware/exporter'
+require 'prometheus/middleware/collector'
+require 'prometheus/client'
+
+# An prometheus output that does nothing.
+class LogStash::Outputs::Prometheus < LogStash::Outputs::Base
+  config_name "prometheus"
+  concurrency :shared
+
+  config :port, :validate => :number, :default => 9640
+
+  config :increment, :validate => :hash, :default => {}
+  # Decrement is only available for gauges
+  config :decrement, :validate => :hash, :default => {}
+  # Decrement is only available for gauges
+  config :set, :validate => :hash, :default => {}
+
+  config :timer, :validate => :hash, :default => {}
+
+  public
+  def register
+    $prom_servers ||= {}
+    $metrics ||= {}
+
+    if $prom_servers[@port].nil?
+      $prom_servers[@port] = Prometheus::Client::Registry.new
+      prom_server = $prom_servers[@port]
+
+      app =
+      Rack::Builder.new(@port) do
+        use ::Rack::Deflater
+        use ::Prometheus::Middleware::Exporter, registry: prom_server
+
+        run ->(_) { [200, {'Content-Type' => 'text/html'}, ['Please access /metrics to see exposed metrics for this Logstash instance.']] }
+      end.to_app
+
+      @thread = Thread.new do
+        Rack::Handler::WEBrick.run(app, Port: @port, BindAddress: "0.0.0.0", Host: "0.0.0.0")
+      end
+    end
+
+    prom_server = $prom_servers[@port]
+
+    @increment.each do |metric_name, val|
+    	val = setup_registry_labels(val)
+      if $metrics[port.to_s + metric_name].nil?
+        if val['type'] == "gauge"
+          metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys)
+        else
+          metric = prom_server.counter(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys)
+        end
+        $metrics[port.to_s + metric_name] = metric
+      end
+    end
+
+    @decrement.each do |metric_name, val|
+      val = setup_registry_labels(val)
+      if $metrics[port.to_s + metric_name].nil?
+        metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys)
+
+        $metrics[port.to_s + metric_name] = metric
+      end
+    end
+
+    @set.each do |metric_name, val|
+      val = setup_registry_labels(val)
+      if $metrics[port.to_s + metric_name].nil?
+        metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys)
+
+        $metrics[port.to_s + metric_name] = metric
+      end
+    end
+
+    @timer.each do |metric_name, val|
+      val = setup_registry_labels(val)
+
+      if val['type'] == "histogram"
+        metric = prom_server.histogram(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys, buckets: val['buckets'])
+      else
+        metric = prom_server.summary(metric_name.to_sym, labels: val['labels'].keys, docstring: val['description'])
+      end
+
+      $metrics[port.to_s + metric_name] = metric
+    end
+  end # def register
+
+  def kill_thread()
+  	@thread.kill
+  	$prom_servers[@port] = nil
+  end
+
+  protected
+  def setup_registry_labels(val)
+    if val['labels'].nil?
+      val['labels'] = {}
+    end
+
+    val['labels'].keys.each do |key|
+      val['labels'][(key.to_sym rescue key) || key] = val['labels'].delete(key)
+    end
+
+    return val     
+  end
+
+  public
+  def receive(event)
+    @increment.each do |metric_name, val|
+      labels = setup_event_labels(val, event)
+      $metrics[port.to_s + metric_name].increment(labels: labels)
+    end
+
+    @decrement.each do |metric_name, val|
+      labels = setup_event_labels(val, event)
+      $metrics[port.to_s + metric_name].decrement(labels: labels)
+    end
+
+    @set.each do |metric_name, val|
+      labels = setup_event_labels(val, event)
+      $metrics[port.to_s + metric_name].set(event.sprintf(val['value']).to_f,labels: labels)
+    end
+
+    @timer.each do |metric_name, val|
+      labels = setup_event_labels(val, event)
+      $metrics[port.to_s + metric_name].observe(event.sprintf(val['value']).to_f,labels: labels)
+    end
+  end # def event
+
+  protected
+  def setup_event_labels(val, event)
+    labels = {}
+    val['labels'].each do |label, lval|
+      labels[label] = event.sprintf(lval)
+    end
+
+    return labels     
+  end
+end # class LogStash::Outputs::Prometheus

data/logstash-output-prometheus.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-output-prometheus'
+  s.version       = '0.1.0'
+  s.licenses      = ['Apache-2.0']
+  s.summary       = 'Output logstash data to a prometheus exporter'
+#  s.homepage      = 'Nada'
+  s.authors       = ['Spencer Malone']
+  s.email         = 'spencer@mailchimp.com'
+  s.require_paths = ['lib']
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "prometheus-client", "0.10.0.pre.alpha.2"
+  s.add_runtime_dependency "rack", ">= 1.6.11"
+
+  s.add_development_dependency "logstash-devutils", "~> 1.3", ">= 1.3.1"
+end

data/spec/outputs/prometheus_spec.rb

@@ -0,0 +1,239 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/prometheus"
+require "logstash/codecs/plain"
+require "logstash/event"
+require 'net/http'
+
+describe LogStash::Outputs::Prometheus do
+  let(:port) { rand(2000..10000) }
+  let(:output) { LogStash::Outputs::Prometheus.new(properties) }
+
+  before do
+    output.register
+  end
+
+  after do
+    output.kill_thread
+  end
+
+  let(:event) do
+    LogStash::Event.new(
+      properties
+    )
+  end
+
+  shared_examples "it should expose data" do |*values|
+    it "should expose data" do
+      output.receive(event)
+
+      url = URI.parse("http://localhost:#{port}/metrics")
+      req = Net::HTTP::Get.new(url.to_s)
+
+      attempts = 0
+
+      begin
+        res = Net::HTTP.start(url.host, url.port) {|http|
+            http.request(req)
+        }
+      rescue
+        attempts++
+        sleep(0.1)
+        if attempts < 10
+          retry
+        end
+      end
+
+      values.each do |value|
+        expect(res.body).to include(value)
+      end
+    end
+  end
+
+  describe "counter behavior" do
+    let(:properties) {
+      { 
+        "port" => port,
+        "increment" => { 
+          "basic_counter" => { 
+            "description" => "Test",
+            "labels" => {
+              "mylabel" => "hi" 
+            }
+          }
+        }
+      }
+    }
+
+    include_examples "it should expose data", 'basic_counter{mylabel="hi"} 1', "# TYPE basic_counter counter", "# HELP basic_counter Test"
+  end
+
+  describe "gauge behavior" do
+    describe "increment" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "increment" => { 
+            "basic_gauge" => { 
+              "description" => "Test1",
+              "type" => "gauge"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", "basic_gauge 1.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Test1"
+    end
+
+    describe "decrement" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "decrement" => { 
+            "basic_gauge" => { 
+              "description" => "Testone",
+              "type" => "gauge"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", "basic_gauge -1.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Testone"
+    end
+
+    describe "set" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "set" => { 
+            "basic_gauge" => { 
+              "description" => "Testone",
+              "type" => "gauge",
+              "value" => "123"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", "basic_gauge 123.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Testone"
+    end
+  end
+
+  describe "summary behavior" do
+    let(:properties) {
+      { 
+        "port" => port,
+        "timer" => { 
+          "huh" => { 
+            "description" => "noway",
+            "type" => "summary",
+            "value" => "11"
+          }
+        }
+      }
+    }
+    include_examples "it should expose data", "huh_sum 11.0", "huh_count 1.0", "# TYPE huh summary", "# HELP huh noway"
+  end
+
+  describe "histogram behavior" do
+    describe "description" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "0"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", "# TYPE history histogram", "# HELP history abe"
+    end
+
+    describe "sum and count" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "111"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", "history_sum 111.0", "history_count 1.0"
+    end
+
+    describe "minimum histogram" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "0"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", 'history_bucket{le="1"} 1.0', 'history_bucket{le="5"} 1.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0'
+    end
+
+    describe "middle histogram" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "5"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0'
+    end
+
+    describe "max histogram" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "9"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0'
+    end
+
+    describe "beyond max histogram" do
+      let(:properties) {
+        { 
+          "port" => port,
+          "timer" => { 
+            "history" => { 
+              "description" => "abe",
+              "type" => "histogram",
+              "buckets" => [1, 5, 10],
+              "value" => "100"
+            }
+          }
+        }
+      }
+      include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 0.0', 'history_bucket{le="+Inf"} 1.0'
+    end
+
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-prometheus
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Spencer Malone
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.0.pre.alpha.2
+  name: prometheus-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.0.pre.alpha.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.11
+  name: rack
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.11
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+description:
+email: spencer@mailchimp.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/prometheus.rb
+- logstash-output-prometheus.gemspec
+- spec/outputs/prometheus_spec.rb
+homepage:
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
+specification_version: 4
+summary: Output logstash data to a prometheus exporter
+test_files:
+- spec/outputs/prometheus_spec.rb