phihos-fluent-plugin-prometheus 2.0.3

data/lib/fluent/plugin/prometheus_metrics.rb

@@ -0,0 +1,77 @@
+module Fluent::Plugin
+
+  ##
+  # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
+  # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
+
+
+  class PrometheusMetrics
+    def initialize
+      @comments = []
+      @metrics = []
+    end
+
+    def to_string
+      (@comments + @metrics).join("\n")
+    end
+
+    def add_comment(comment)
+      @comments << comment
+    end
+
+    def add_metric_value(value)
+      @metrics << value
+    end
+
+    attr_writer :comments, :metrics
+  end
+
+  class PromMetricsAggregator
+    def initialize
+      @metrics = {}
+    end
+
+    def get_metric_name_from_comment(line)
+      tokens = line.split(' ')
+      if ['HELP', 'TYPE'].include?(tokens[1])
+        tokens[2]
+      else
+        ''
+      end
+    end
+
+    def add_metrics(metrics)
+      current_metric = ''
+      new_metric = false
+      lines = metrics.split("\n")
+      for line in lines
+        if line[0] == '#'
+          # Metric comment (# TYPE, # HELP)
+          parsed_metric = get_metric_name_from_comment(line)
+          if parsed_metric != ''
+            if parsed_metric != current_metric
+              # Starting a new metric comment block
+              new_metric = !@metrics.key?(parsed_metric)
+              if new_metric
+                @metrics[parsed_metric] = PrometheusMetrics.new()
+              end
+              current_metric = parsed_metric
+            end
+
+            if new_metric && parsed_metric == current_metric
+              # New metric, inject comments (# TYPE, # HELP)
+              @metrics[parsed_metric].add_comment(line)
+            end
+          end
+        else
+          # Metric value, simply append line
+          @metrics[current_metric].add_metric_value(line)
+        end
+      end
+    end
+
+    def get_metrics
+      @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
+    end
+  end
+end

data/misc/fluentd_sample.conf

@@ -0,0 +1,170 @@
+## Prometheus Input Plugin Configuration
+
+# input plugin that exports metrics
+<source>
+  @type prometheus
+</source>
+
+<source>
+  @type monitor_agent
+</source>
+
+<source>
+  @type forward
+</source>
+
+# input plugin that collects metrics from MonitorAgent
+<source>
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for output plugin
+<source>
+  @type prometheus_output_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+# input plugin that collects metrics for in_tail plugin
+<source>
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+  </labels>
+</source>
+
+## Nginx Access Log Configuration
+
+<source>
+  @type tail
+  format nginx
+  tag nginx
+  path /var/log/nginx/access.log
+  pos_file /tmp/fluent_nginx.pos
+  types size:integer
+</source>
+
+<filter nginx>
+  @type prometheus
+
+  # You can use counter type with specifying a key,
+  # and increments counter by the value
+  <metric>
+    name nginx_size_counter_bytes
+    type counter
+    desc nginx bytes sent
+    key size
+    <labels>
+       host ${hostname}
+       foo bar
+    </labels>
+  </metric>
+
+  # You can use counter type without specifying a key
+  # This just increments counter by 1
+  <metric>
+    name nginx_record_counts
+    type counter
+    desc the number of emited records
+    <labels>
+       host ${hostname}
+    </labels>
+  </metric>
+</filter>
+
+<match nginx>
+  @type copy
+  # for MonitorAgent sample
+  <store>
+    @id test_forward
+    @type forward
+    buffer_type memory
+    flush_interval 1s
+    max_retry_wait 2s
+    <buffer>
+    # max_retry_wait 10s
+    flush_interval 1s
+    # retry_type periodic
+    disable_retry_limit
+</buffer>
+    # retry_limit 3
+    disable_retry_limit
+   <server>
+     host 127.0.0.1
+     port 20000
+   </server>
+  </store>
+  <store>
+    @type stdout
+  </store>
+</match>
+
+## Nginx Proxy Log Configuration
+
+<source>
+  @type tail
+  format ltsv
+  tag nginx_proxy
+  path /var/log/nginx/access_proxy.log
+  pos_file /tmp/fluent_nginx_proxy.pos
+  types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
+</source>
+
+<filter nginx_proxy>
+  @type prometheus
+
+  # common labels for all metrics
+  <labels>
+     host ${hostname}
+     method ${request_method}
+     status ${status}
+  </labels>
+
+  <metric>
+    name nginx_proxy_request_length_total_bytes
+    type counter
+    desc nginx proxy request length bytes
+    key request_length
+  </metric>
+  <metric>
+    name nginx_proxy_bytes_sent_total_bytes
+    type counter
+    desc nginx proxy bytes sent
+    key bytes_sent
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_total_milliseconds
+    type counter
+    desc nginx proxy request time
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_response_duration_total_milliseconds
+    type counter
+    desc nginx proxy upstream response time
+    key upstream_response_time
+  </metric>
+  <metric>
+    name nginx_proxy_request_duration_milliseconds
+    type summary
+    desc nginx proxy request duration summary
+    key request_time
+  </metric>
+  <metric>
+    name nginx_proxy_upstream_duration_milliseconds
+    type summary
+    desc nginx proxy upstream response duration summary
+    key upstream_response_time
+  </metric>
+</filter>
+
+<match nginx_proxy>
+  @type copy
+  <store>
+    @type stdout
+  </store>
+</match>

data/misc/nginx_proxy.conf

@@ -0,0 +1,22 @@
+log_format ltsv 'time:$time_iso8601\t'
+                'remote_addr:$remote_addr\t'
+                'request_method:$request_method\t'
+                'request_length:$request_length\t'
+                'request_uri:$request_uri\t'
+                'uri:$uri\t'
+                'status:$status\t'
+                'bytes_sent:$bytes_sent\t'
+                'body_bytes_sent:$body_bytes_sent\t'
+                'referer:$http_referer\t'
+                'useragent:$http_user_agent\t'
+                'request_time:$request_time\t'
+                'upstream_response_time:$upstream_response_time';
+
+server {
+  access_log /var/log/nginx/access_proxy.log ltsv;
+  listen 9999;
+  location / {
+    proxy_pass https://www.google.com;
+  }
+}
+

data/misc/prometheus.yaml

@@ -0,0 +1,13 @@
+# A job to scrape an endpoint of Fluentd running on localhost.
+scrape_configs:
+- job_name: 'prometheus'
+  scrape_interval: 5s
+  static_configs:
+    - targets:
+      - 'localhost:9090'
+- job_name: fluentd
+  scrape_interval: 5s
+  static_configs:
+    - targets:
+      - 'localhost:24231'
+  metrics_path: /metrics

data/misc/prometheus_alerts.yaml

@@ -0,0 +1,59 @@
+ALERT FluentdNodeDown
+  IF up{job="fluentd"} == 0
+  FOR 10m
+  LABELS {
+    service = "fluentd",
+    severity = "warning"
+  }
+  ANNOTATIONS {
+    summary = "fluentd cannot be scraped",
+    description = "Prometheus could not scrape {{ $labels.job }} for more than 10 minutes",
+  }
+  
+ALERT FluentdNodeDown
+  IF up{job="fluentd"} == 0
+  FOR 30m
+  LABELS {
+    service = "fluentd",
+    severity = "critical"
+  }
+  ANNOTATIONS {
+    summary = "fluentd cannot be scraped",
+    description = "Prometheus could not scrape {{ $labels.job }} for more than 30 minutes",
+  }
+  
+ALERT FluentdQueueLength
+  IF rate(fluentd_status_buffer_queue_length[5m]) > 0.3
+  FOR 1m
+  LABELS {
+    service = "fluentd",
+    severity = "warning"
+  }
+  ANNOTATIONS {
+    summary = "fluentd node are failing",
+    description = "In the last 5 minutes, fluentd queues increased 30%. Current value is {{ $value }} ",
+  }
+
+ALERT FluentdQueueLength
+  IF rate(fluentd_status_buffer_queue_length[5m]) > 0.5
+  FOR 1m
+  LABELS {
+    service = "fluentd",
+    severity = "critical"
+  }
+  ANNOTATIONS {
+    summary = "fluentd node are critical",
+    description = "In the last 5 minutes, fluentd queues increased 50%. Current value is {{ $value }} ",
+  }
+
+ALERT FluentdRecordsCountsHigh
+  IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
+  FOR 1m
+  LABELS {
+    service = "fluentd",
+    severity = "critical"
+  }
+  ANNOTATIONS {
+    summary = "fluentd records count are critical",
+    description = "In the last 5m, records counts increased 3 times, comparing to the latest 15 min.",
+  }

data/spec/fluent/plugin/filter_prometheus_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_prometheus'
+require_relative 'shared'
+
+describe Fluent::Plugin::PrometheusFilter do
+  let(:tag) { 'prometheus.test' }
+  let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
+
+  describe '#configure' do
+    it_behaves_like 'output configuration'
+  end
+
+  describe '#run' do
+    let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+
+      it 'should keep original message' do
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(driver.filtered_records.first).to eq(message)
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+
+  describe '#run with retention' do
+    let(:message) { { "foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10 } }
+
+    context 'config with retention 1' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+            <labels>
+              bar ${bar}
+              baz ${baz}
+              qux ${qux}
+            </labels>
+            retention 1
+            retention_check_interval 1
+          </metric>
+        )
+      }
+
+      it 'expires metric after max 2s' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) {
+          driver.feed(event_time, message)
+          expect(registry.metrics[0].get(labels: { :bar => 100, :baz => 100, :qux => 10 })).to eq(100)
+          sleep(2)
+          expect(registry.metrics[0].get(labels: { :bar => 100, :baz => 100, :qux => 10 })).to eq(0.0)
+        }
+      end
+    end
+  end
+
+  describe '#run with topk' do
+    let(:message1) { { "foo" => 200, "bar" => "a" } }
+    let(:message2) { { "foo" => 300, "bar" => "b" } }
+    let(:message3) { { "foo" => 100, "bar" => "c" } }
+
+    context 'config with retention 1' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+            <labels>
+              bar ${bar}
+            </labels>
+            topk 2
+          </metric>
+        )
+      }
+
+      it 'expires metric after max 2s' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) {
+          driver.feed(event_time, message1)
+          driver.feed(event_time, message2)
+          driver.feed(event_time, message3)
+        }
+        expect(registry.metrics[0].values).to eq({
+                                                   { :bar => "a" } => 200,
+                                                   { :bar => "b" } => 300,
+                                                 })
+      end
+    end
+  end
+end

data/spec/fluent/plugin/in_prometheus_monitor_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus_monitor'
+require 'fluent/test/driver/input'
+
+describe Fluent::Plugin::PrometheusMonitorInput do
+  MONITOR_CONFIG = %[
+  @type prometheus_monitor
+  <labels>
+    host ${hostname}
+    foo bar
+  </labels>
+]
+
+  INVALID_MONITOR_CONFIG = %[
+  @type prometheus_monitor
+
+  <labels>
+    host ${hostname}
+    foo bar
+    invalid_use1 $.foo.bar
+    invalid_use2 $[0][1]
+  </labels>
+]
+
+  let(:config) { MONITOR_CONFIG }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
+
+  describe '#configure' do
+    describe 'valid' do
+      it 'does not raise error' do
+        expect{driver}.not_to raise_error
+      end
+    end
+
+    describe 'invalid' do
+      let(:config) { INVALID_MONITOR_CONFIG }
+      it 'expect raise error' do
+        expect{driver}.to raise_error
+      end
+    end
+  end
+end