fluent-plugin-prometheus-smarter 1.8.4

data/spec/fluent/plugin/prometheus_metrics_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PromMetricsAggregator do
+
+  metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+]
+
+  metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
+# HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+# TYPE fluentd_output_status_buffer_total_bytes gauge
+# HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+]
+
+  metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  describe 'add_metrics' do
+    context '1st_metrics' do
+      it 'adds all fields' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1)
+      end
+    end
+    context '2nd_metrics' do
+      it 'append new metrics' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
+      end
+    end
+
+    context '3rd_metrics' do
+      it 'append existing metrics in the right place' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        all_metrics.add_metrics(metrics_worker_3)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
+      end
+    end
+  end
+end

data/spec/fluent/plugin/shared.rb

@@ -0,0 +1,249 @@
+
+BASE_CONFIG = %[
+  @type prometheus
+]
+
+SIMPLE_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name simple_foo
+    type counter
+    desc Something foo.
+    key foo
+  </metric>
+]
+
+FULL_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name full_foo
+    type counter
+    desc Something foo.
+    key foo
+    <labels>
+      key foo1
+    </labels>
+  </metric>
+  <metric>
+    name full_bar
+    type gauge
+    desc Something bar.
+    key bar
+    <labels>
+      key foo2
+    </labels>
+  </metric>
+  <metric>
+    name full_baz
+    type summary
+    desc Something baz.
+    key baz
+    <labels>
+      key foo3
+    </labels>
+  </metric>
+  <metric>
+    name full_qux
+    type histogram
+    desc Something qux.
+    key qux
+    buckets 0.1, 1, 5, 10
+    <labels>
+      key foo4
+    </labels>
+  </metric>
+  <metric>
+    name full_accessor1
+    type summary
+    desc Something with accessor.
+    key $.foo
+    <labels>
+      key foo5
+    </labels>
+  </metric>
+  <metric>
+    name full_accessor2
+    type counter
+    desc Something with accessor.
+    key $.foo
+    <labels>
+      key foo6
+    </labels>
+  </metric>
+  <labels>
+    test_key test_value
+  </labels>
+]
+
+PLACEHOLDER_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name placeholder_foo
+    type counter
+    desc Something foo.
+    key foo
+    <labels>
+      foo ${foo}
+    </labels>
+  </metric>
+  <labels>
+    tag ${tag}
+    hostname ${hostname}
+  </labels>
+]
+
+ACCESSOR_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name accessor_foo
+    type counter
+    desc Something foo.
+    key foo
+    <labels>
+      foo $.foo
+    </labels>
+  </metric>
+]
+
+COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
+  <metric>
+    name without_key_foo
+    type counter
+    desc Something foo.
+  </metric>
+]
+
+shared_examples_for 'output configuration' do
+  context 'base config' do
+    let(:config) { BASE_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  context 'with simple configuration' do
+    let(:config) { SIMPLE_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  context 'with full configuration' do
+    let(:config) { FULL_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  context 'with placeholder configuration' do
+    let(:config) { PLACEHOLDER_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  context 'with accessor configuration' do
+    let(:config) {  ACCESSOR_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  describe 'with counter without key configuration' do
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    it { expect { driver }.not_to raise_error }
+  end
+
+  context 'with unknown type' do
+    let(:config) do
+      BASE_CONFIG + %[
+      <metric>
+        type foo
+      </metric>
+      ]
+    end
+    it { expect { driver }.to raise_error(Fluent::ConfigError) }
+  end
+end
+
+shared_examples_for 'instruments record' do
+  before do
+    driver.run(default_tag: tag) { driver.feed(event_time, message) }
+  end
+
+  context 'full config' do
+    let(:config) { FULL_CONFIG }
+    let(:counter) { registry.get(:full_foo) }
+    let(:gauge) { registry.get(:full_bar) }
+    let(:summary) { registry.get(:full_baz) }
+    let(:histogram) { registry.get(:full_qux) }
+    let(:summary_with_accessor) { registry.get(:full_accessor1) }
+    let(:counter_with_accessor) { registry.get(:full_accessor2) }
+
+    it 'adds all metrics' do
+      expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2])
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
+      expect(summary).to be_kind_of(::Prometheus::Client::Metric)
+      expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
+      expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
+      expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
+    end
+
+    it 'instruments counter metric' do
+      expect(counter.type).to eq(:counter)
+      expect(counter.get({test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
+      expect(counter_with_accessor.get({test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
+    end
+
+    it 'instruments gauge metric' do
+      expect(gauge.type).to eq(:gauge)
+      expect(gauge.get({test_key: 'test_value', key: 'foo2'})).to eq(100)
+    end
+
+    it 'instruments summary metric' do
+      expect(summary.type).to eq(:summary)
+      expect(summary.get({test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
+      expect(summary.get({test_key: 'test_value', key: 'foo3'})[0.99]).to eq(100)
+      expect(summary_with_accessor.get({test_key: 'test_value', key: 'foo5'})[0.99]).to eq(100)
+    end
+
+    it 'instruments histogram metric' do
+      driver.run(default_tag: tag) do
+        4.times { driver.feed(event_time, message) }
+      end
+
+      expect(histogram.type).to eq(:histogram)
+      expect(histogram.get({test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
+      expect(histogram.get({test_key: 'test_value', key: 'foo4'})[10]).to eq(5) # 4 + `es` in before
+    end
+  end
+
+  context 'placeholder config' do
+    let(:config) { PLACEHOLDER_CONFIG }
+    let(:counter) { registry.get(:placeholder_foo) }
+
+    it 'expands placeholders with record values' do
+      expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      key, _ = counter.values.find {|k,v| v ==  100 }
+      expect(key).to be_kind_of(Hash)
+      expect(key[:tag]).to eq(tag)
+      expect(key[:hostname]).to be_kind_of(String)
+      expect(key[:hostname]).not_to eq("${hostname}")
+      expect(key[:hostname]).not_to be_empty
+      expect(key[:foo]).to eq("100")
+    end
+  end
+
+  context 'accessor config' do
+    let(:config) { ACCESSOR_CONFIG }
+    let(:counter) { registry.get(:accessor_foo) }
+
+    it 'expands accessor with record values' do
+      expect(registry.metrics.map(&:name)).to eq([:accessor_foo])
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      key, _ = counter.values.find {|k,v| v ==  100 }
+      expect(key).to be_kind_of(Hash)
+      expect(key[:foo]).to eq(100)
+    end
+  end
+
+  context 'counter_without config' do
+    let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
+    let(:counter) { registry.get(:without_key_foo) }
+
+    it 'just increments by 1' do
+      expect(registry.metrics.map(&:name)).to eq([:without_key_foo])
+      expect(counter).to be_kind_of(::Prometheus::Client::Metric)
+      _, value = counter.values.find {|k,v| k == {} }
+      expect(value).to eq(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/plugin/prometheus'
+
+# Disable Test::Unit
+Test::Unit::AutoRunner.need_auto_run = false
+
+Fluent::Test.setup
+include Fluent::Test::Helpers

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-prometheus-smarter
+version: !ruby/object:Gem::Version
+  version: 1.8.4
+platform: ruby
+authors:
+- Masahiro Sano
+- Josh Minor
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A fluent plugin that collects metrics and exposes for Prometheus.
+email:
+- sabottenda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-prometheus.gemspec
+- lib/fluent/plugin/filter_prometheus.rb
+- lib/fluent/plugin/in_prometheus.rb
+- lib/fluent/plugin/in_prometheus_monitor.rb
+- lib/fluent/plugin/in_prometheus_output_monitor.rb
+- lib/fluent/plugin/in_prometheus_tail_monitor.rb
+- lib/fluent/plugin/metric_prometheus.rb
+- lib/fluent/plugin/out_prometheus.rb
+- lib/fluent/plugin/prometheus.rb
+- lib/fluent/plugin/prometheus/placeholder_expander.rb
+- lib/fluent/plugin/prometheus_metrics.rb
+- misc/fluentd_sample.conf
+- misc/nginx_proxy.conf
+- misc/prometheus.yaml
+- misc/prometheus_alerts.yaml
+- spec/fluent/plugin/filter_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
+- spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
+- spec/fluent/plugin/shared.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jishminor/fluent-plugin-prometheus
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A fluent plugin that collects metrics and exposes for Prometheus.
+test_files:
+- spec/fluent/plugin/filter_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_monitor_spec.rb
+- spec/fluent/plugin/in_prometheus_spec.rb
+- spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb
+- spec/fluent/plugin/out_prometheus_spec.rb
+- spec/fluent/plugin/prometheus/placeholder_expander_spec.rb
+- spec/fluent/plugin/prometheus_metrics_spec.rb
+- spec/fluent/plugin/shared.rb
+- spec/spec_helper.rb