fluent-plugin-prometheus 0.1.0

data/misc/nginx_proxy.conf

@@ -0,0 +1,22 @@
+log_format ltsv 'time:$time_iso8601\t'
+                'remote_addr:$remote_addr\t'
+                'request_method:$request_method\t'
+                'request_length:$request_length\t'
+                'request_uri:$request_uri\t'
+                'uri:$uri\t'
+                'status:$status\t'
+                'bytes_sent:$bytes_sent\t'
+                'body_bytes_sent:$body_bytes_sent\t'
+                'referer:$http_referer\t'
+                'useragent:$http_user_agent\t'
+                'request_time:$request_time\t'
+                'upstream_response_time:$upstream_response_time';
+
+server {
+  access_log /var/log/nginx/access_proxy.log ltsv;
+  listen 9999;
+  location / {
+    proxy_pass https://www.google.com;
+  }
+}
+

data/misc/prometheus.conf

@@ -0,0 +1,10 @@
+# A job to scrape an endpoint of Fluentd running on localhost.
+job: {
+  name: "fluentd"
+  scrape_interval: "5s"
+
+  target_group: {
+    target: "http://localhost:24231/metrics"
+  }
+}
+

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require 'fluent/plugin/filter_prometheus'
+require_relative 'shared'
+
+describe Fluent::PrometheusFilter do
+  let(:tag) { 'prometheus.test' }
+  let(:driver) { Fluent::Test::FilterTestDriver.new(Fluent::PrometheusFilter, tag).configure(config, true) }
+  let(:registry) { ::Prometheus::Client.registry }
+
+  describe '#configure' do
+    it_behaves_like 'output configuration'
+  end
+
+  describe '#run' do
+    let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100} }
+    let(:es) { driver.run { driver.emit(message, Time.now) }.filtered }
+
+    context 'simple config' do
+      include_context 'simple_config'
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to include(name)
+        es
+        expect(registry.metrics.map(&:name)).to include(name)
+      end
+
+      it 'should keep original message' do
+        expect(es.first[1]).to eq(message)
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+end if defined?(Fluent::PrometheusFilter)

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require 'fluent/plugin/out_prometheus'
+require_relative 'shared'
+
+describe Fluent::PrometheusOutput do
+  let(:tag) { 'prometheus.test' }
+  let(:driver) { Fluent::Test::OutputTestDriver.new(Fluent::PrometheusOutput, tag).configure(config) }
+  let(:registry) { ::Prometheus::Client.registry }
+
+  describe '#configure' do
+    it_behaves_like 'output configuration'
+  end
+
+  describe '#run' do
+    let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100} }
+    let(:es) { driver.run { driver.emit(message, Time.now) } }
+
+    context 'simple config' do
+      include_context 'simple_config'
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to include(name)
+        es
+        expect(registry.metrics.map(&:name)).to include(name)
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+end

data/spec/fluent/plugin/prometheus_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+
+require 'net/http'
+
+describe Fluent::PrometheusInput do
+  CONFIG = %[
+  type prometheus
+]
+
+  LOCAL_CONFIG = %[
+  type prometheus
+  bind 127.0.0.1
+]
+
+  let(:config) { CONFIG }
+  let(:port) { 24231 }
+  let(:driver) { Fluent::Test::InputTestDriver.new(Fluent::PrometheusInput).configure(config) }
+
+  describe '#configure' do
+    describe 'bind' do
+      let(:config) { CONFIG + %[
+  bind 127.0.0.1
+] }
+      it 'should be configurable' do
+        expect(driver.instance.bind).to eq('127.0.0.1')
+      end
+    end
+
+    describe 'port' do
+      let(:config) { CONFIG + %[
+  port 8888
+] }
+      it 'should be configurable' do
+        expect(driver.instance.port).to eq(8888)
+      end
+    end
+
+    describe 'metrics_path' do
+      let(:config) { CONFIG + %[
+  metrics_path /_test
+] }
+      it 'should be configurable' do
+        expect(driver.instance.metrics_path).to eq('/_test')
+      end
+    end
+  end
+
+  describe '#run' do
+    context '/metrics' do
+      let(:config) { LOCAL_CONFIG }
+      it 'returns 200' do
+        driver.run do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/metrics")
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context '/foo' do
+      let(:config) { LOCAL_CONFIG }
+      it 'does not return 200' do
+        driver.run do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/foo")
+            res = http.request(req)
+            expect(res.code).not_to eq('200')
+          end
+        end
+      end
+    end
+  end
+end

data/spec/fluent/plugin/shared.rb

@@ -0,0 +1,210 @@
+
+BASE_CONFIG = %[
+  type prometheus
+]
+
+
+SIMPLE_CONFIG = BASE_CONFIG + %[
+  type prometheus
+  <metric>
+    name simple_foo
+    type counter
+    desc Something foo.
+    key foo
+  </metric>
+]
+
+FULL_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name full_foo
+    type counter
+    desc Something foo.
+    key foo
+    <labels>
+      key foo1
+    </labels>
+  </metric>
+  <metric>
+    name full_bar
+    type gauge
+    desc Something bar.
+    key bar
+    <labels>
+      key foo2
+    </labels>
+  </metric>
+  <metric>
+    name full_baz
+    type summary
+    desc Something baz.
+    key baz
+    <labels>
+      key foo3
+    </labels>
+  </metric>
+  <labels>
+    test_key test_value
+  </labels>
+]
+
+PLACEHOLDER_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name placeholder_foo
+    type counter
+    desc Something foo.
+    key foo
+    <labels>
+      foo ${foo}
+    </labels>
+  </metric>
+  <labels>
+    tag ${tag}
+    hostname ${hostname}
+  </labels>
+]
+
+COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name without_key_foo
+    type counter
+    desc Something foo.
+  </metric>
+]
+
+shared_context 'simple_config' do
+  let(:orig_name) { 'simple_foo' }
+  let(:config) { SIMPLE_CONFIG.gsub(orig_name, name.to_s) }
+  let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
+  let(:counter) { registry.get(name) }
+end
+
+shared_context 'full_config' do
+  let(:config) { FULL_CONFIG }
+  let(:counter) { registry.get(:full_foo) }
+  let(:gauge) { registry.get(:full_bar) }
+  let(:summary) { registry.get(:full_baz) }
+end
+
+shared_context 'placeholder_config' do
+  let(:orig_name) { 'placeholder_foo' }
+  let(:config) { PLACEHOLDER_CONFIG.gsub(orig_name, name.to_s) }
+  let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
+  let(:counter) { registry.get(name) }
+end
+
+shared_context 'counter_without_key_config' do
+  let(:orig_name) { 'without_key_foo' }
+  let(:config) { COUNTER_WITHOUT_KEY_CONFIG.gsub(orig_name, name.to_s) }
+  let(:name) { "#{orig_name}_#{Time.now.to_f}".to_sym }
+  let(:counter) { registry.get(name) }
+end
+
+shared_examples_for 'output configuration' do
+  context 'base config' do
+    let(:config) { BASE_CONFIG }
+    it 'does not raise error' do
+      expect{driver}.not_to raise_error
+    end
+  end
+
+  describe 'configure simple configuration' do
+    include_context 'simple_config'
+    it { expect{driver}.not_to raise_error }
+  end
+
+  describe 'configure full configuration' do
+    include_context 'full_config'
+    it { expect{driver}.not_to raise_error }
+  end
+
+  describe 'configure placeholder configuration' do
+    include_context 'placeholder_config'
+    it { expect{driver}.not_to raise_error }
+  end
+
+  describe 'configure counter without key configuration' do
+    include_context 'counter_without_key_config'
+    it { expect{driver}.not_to raise_error }
+  end
+
+  context 'unknown type' do
+    let(:config) { BASE_CONFIG + %[
+<metric>
+  type foo
+</metric>
+] }
+    it 'raises ConfigError' do
+      expect{driver}.to raise_error Fluent::ConfigError
+    end
+  end
+end
+
+shared_examples_for 'instruments record' do
+  context 'full config' do
+    include_context 'full_config'
+
+    before :each do
+      es
+    end
+
+    it 'adds all metrics' do
+      expect(registry.metrics.map(&:name)).to include(:full_foo)
+      expect(registry.metrics.map(&:name)).to include(:full_bar)
+      expect(registry.metrics.map(&:name)).to include(:full_baz)
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
+      expect(summary).to be_kind_of(::Prometheus::Client::Metric)
+    end
+
+    it 'instruments counter metric' do
+      expect(counter.type).to eq(:counter)
+      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Integer)
+    end
+
+    it 'instruments gauge metric' do
+      expect(gauge.type).to eq(:gauge)
+      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+    end
+
+    it 'instruments summary metric' do
+      expect(summary.type).to eq(:summary)
+      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
+    end
+  end
+
+  context 'placeholder config' do
+    include_context 'placeholder_config'
+
+    before :each do
+      es
+    end
+
+    it 'expands placeholders with record values' do
+      expect(registry.metrics.map(&:name)).to include(name)
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      key, _ = counter.values.find {|k,v| v ==  100 }
+      expect(key).to be_kind_of(Hash)
+      expect(key[:tag]).to eq(tag)
+      expect(key[:hostname]).to be_kind_of(String)
+      expect(key[:hostname]).not_to eq("${hostname}")
+      expect(key[:hostname]).not_to be_empty
+      expect(key[:foo]).to eq("100")
+    end
+  end
+
+  context 'counter_without config' do
+    include_context 'counter_without_key_config'
+
+    before :each do
+      es
+    end
+
+    it 'just increments by 1' do
+      expect(registry.metrics.map(&:name)).to include(name)
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      _, value = counter.values.find {|k,v| k == {} }
+      expect(value).to eq(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'fluent/test'
+require 'fluent/plugin/prometheus'
+
+# Disable Test::Unit
+Test::Unit::AutoRunner.need_auto_run = false
+
+Fluent::Test.setup

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-prometheus
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Masahiro Sano
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluent plugin that collects metrics and exposes for Prometheus.
+email:
+- sabottenda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.fluentd.0.10
+- LICENSE.txt
+- README.md
+- Rakefile
+- fluent-plugin-prometheus.gemspec
+- lib/fluent/plugin/filter_prometheus.rb
+- lib/fluent/plugin/in_prometheus.rb
+- lib/fluent/plugin/in_prometheus_monitor.rb
+- lib/fluent/plugin/out_prometheus.rb
+- lib/fluent/plugin/prometheus.rb
+- misc/fluentd_sample.conf
+- misc/nginx_proxy.conf
+- misc/prometheus.conf
+- spec/fluent/plugin/filter_prometheus_spec.rb
+- spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_spec.rb
+- spec/fluent/plugin/shared.rb
+- spec/spec_helper.rb
+homepage: https://github.com/kazegusuri/fluent-plugin-prometheus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.3
+signing_key: 
+specification_version: 4
+summary: A fluent plugin that collects metrics and exposes for Prometheus.
+test_files:
+- spec/fluent/plugin/filter_prometheus_spec.rb
+- spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus_spec.rb
+- spec/fluent/plugin/shared.rb
+- spec/spec_helper.rb