phihos-fluent-plugin-prometheus 2.0.3

data/spec/fluent/plugin/in_prometheus_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PrometheusInput do
+  CONFIG = %[
+  @type prometheus
+]
+
+  LOCAL_CONFIG = %[
+  @type prometheus
+  bind 127.0.0.1
+]
+
+  let(:config) { CONFIG }
+  let(:port) { 24231 }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput).configure(config) }
+
+  describe '#configure' do
+    describe 'bind' do
+      let(:config) { CONFIG + %[
+  bind 127.0.0.1
+] }
+      it 'should be configurable' do
+        expect(driver.instance.bind).to eq('127.0.0.1')
+      end
+    end
+
+    describe 'port' do
+      let(:config) { CONFIG + %[
+  port 8888
+] }
+      it 'should be configurable' do
+        expect(driver.instance.port).to eq(8888)
+      end
+    end
+
+    describe 'metrics_path' do
+      let(:config) { CONFIG + %[
+  metrics_path /_test
+] }
+      it 'should be configurable' do
+        expect(driver.instance.metrics_path).to eq('/_test')
+      end
+    end
+  end
+
+  describe '#start' do
+    context 'with transport section' do
+      let(:config) do
+        %[
+           @type prometheus
+           bind 127.0.0.1
+           <transport tls>
+             insecure true
+           </transport>
+         ]
+      end
+
+      it 'returns 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+            req = Net::HTTP::Get.new('/metrics')
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context 'old parameters are given' do
+      context 'when extra_conf is used' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
+            </ssl>
+         ]
+        end
+
+        it 'uses webrick' do
+          expect(driver.instance).to receive(:start_webrick).once
+          driver.run(timeout: 1)
+        end
+
+        it 'returns 200' do
+          driver.run(timeout: 1) do
+            Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
+              req = Net::HTTP::Get.new('/metrics')
+              res = http.request(req)
+              expect(res.code).to eq('200')
+            end
+          end
+        end
+      end
+
+      context 'cert_path and private_key_path combination' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              certificate_path path
+              private_key_path path1
+            </ssl>
+          ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'insecure and ca_path' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              ca_path path
+            </ssl>
+           ]
+        end
+
+        it 'converts them into new transport section' do
+          expect(driver.instance).to receive(:http_server_create_http_server).with(
+                                       :in_prometheus_server,
+                                       addr: anything,
+                                       logger: anything,
+                                       port: anything,
+                                       proto: :tls,
+                                       tls_opts: { 'ca_path' => 'path', 'insecure' => true }
+                                     ).once
+
+          driver.run(timeout: 1)
+        end
+      end
+
+      context 'when only private_key_path is geven' do
+        let(:config) do
+          %[
+            @type prometheus
+            bind 127.0.0.1
+            <ssl>
+              enable true
+              private_key_path path
+            </ssl>
+           ]
+        end
+
+        it 'raises ConfigError' do
+          expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
+        end
+      end
+    end
+  end
+
+  describe '#run' do
+    context '/metrics' do
+      let(:config) { LOCAL_CONFIG }
+      it 'returns 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/metrics")
+            res = http.request(req)
+            expect(res.code).to eq('200')
+          end
+        end
+      end
+    end
+
+    context '/foo' do
+      let(:config) { LOCAL_CONFIG }
+      it 'does not return 200' do
+        driver.run(timeout: 1) do
+          Net::HTTP.start("127.0.0.1", port) do |http|
+            req = Net::HTTP::Get.new("/foo")
+            res = http.request(req)
+            expect(res.code).not_to eq('200')
+          end
+        end
+      end
+    end
+  end
+
+  describe '#run_multi_workers' do
+    context '/metrics' do
+      Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+        let(:config) { CONFIG + %[
+          port #{port - 2}
+        ] }
+
+        it 'should configure port using sequential number' do
+          driver = Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput)
+          driver.instance.instance_eval{ @_fluentd_worker_id = 2 }
+          driver.configure(config)
+          expect(driver.instance.port).to eq(port)
+          driver.run(timeout: 1) do
+            Net::HTTP.start("127.0.0.1", port) do |http|
+              req = Net::HTTP::Get.new("/metrics")
+              res = http.request(req)
+              expect(res.code).to eq('200')
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus_tail_monitor'
+require 'fluent/test/driver/input'
+
+describe Fluent::Plugin::PrometheusTailMonitorInput do
+  MONITOR_CONFIG = %[
+  @type prometheus_tail_monitor
+  <labels>
+    host ${hostname}
+    foo bar
+  </labels>
+]
+
+  INVALID_MONITOR_CONFIG = %[
+  @type prometheus_tail_monitor
+
+  <labels>
+    host ${hostname}
+    foo bar
+    invalid_use1 $.foo.bar
+    invalid_use2 $[0][1]
+  </labels>
+]
+
+  let(:config) { MONITOR_CONFIG }
+  let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusTailMonitorInput).configure(config) }
+
+  describe '#configure' do
+    describe 'valid' do
+      it 'does not raise error' do
+        expect { driver }.not_to raise_error
+      end
+    end
+
+    describe 'invalid' do
+      let(:config) { INVALID_MONITOR_CONFIG }
+      it 'expect raise error' do
+        expect { driver }.to raise_error
+      end
+    end
+  end
+end

data/spec/fluent/plugin/out_prometheus_spec.rb

@@ -0,0 +1,139 @@
+require 'spec_helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_prometheus'
+require_relative 'shared'
+
+describe Fluent::Plugin::PrometheusOutput do
+  let(:tag) { 'prometheus.test' }
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::PrometheusOutput).configure(config) }
+  let(:registry) { ::Prometheus::Client::Registry.new }
+
+  before do
+    allow(Prometheus::Client).to receive(:registry).and_return(registry)
+  end
+
+  describe '#configure' do
+    it_behaves_like 'output configuration'
+  end
+
+  describe '#run' do
+    let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+
+  describe '#run with symbolized keys' do
+    let(:message) { {:foo => 100, :bar => 100, :baz => 100, :qux => 10} }
+
+    context 'simple config' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+          </metric>
+        )
+      }
+
+      it 'adds a new counter metric' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) { driver.feed(event_time, message) }
+        expect(registry.metrics.map(&:name)).to eq([:simple])
+      end
+    end
+
+    it_behaves_like 'instruments record'
+  end
+
+  describe '#run with retention' do
+    let(:message) { { "foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10 } }
+    let(:labels) { { :bar => 100, :baz => 100, :qux => 10 } }
+
+    context 'config with retention 1' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+            <labels>
+              bar ${bar}
+              baz ${baz}
+              qux ${qux}
+            </labels>
+            retention 1
+            retention_check_interval 1
+          </metric>
+        )
+      }
+
+      it 'expires metric after max 2s' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) {
+          driver.feed(event_time, message)
+          expect(registry.metrics[0].get(labels: labels)).to eq(100)
+          sleep(2)
+          expect(registry.metrics[0].get(labels: labels)).to eq(0.0)
+        }
+      end
+    end
+  end
+
+  describe '#run with topk' do
+    let(:message1) { { "foo" => 200, "bar" => "a" } }
+    let(:message2) { { "foo" => 300, "bar" => "b" } }
+    let(:message3) { { "foo" => 100, "bar" => "c" } }
+
+    context 'config with retention 1' do
+      let(:config) {
+        BASE_CONFIG + %(
+          <metric>
+            name simple
+            type counter
+            desc Something foo.
+            key foo
+            <labels>
+              bar ${bar}
+            </labels>
+            topk 2
+          </metric>
+        )
+      }
+
+      it 'expires metric after max 2s' do
+        expect(registry.metrics.map(&:name)).not_to eq([:simple])
+        driver.run(default_tag: tag) {
+          driver.feed(event_time, message1)
+          driver.feed(event_time, message2)
+          driver.feed(event_time, message3)
+        }
+        expect(registry.metrics[0].values).to eq({
+                                                   { :bar => "a" } => 200,
+                                                   { :bar => "b" } => 300,
+                                                 })
+      end
+    end
+  end
+end

data/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb

@@ -0,0 +1,110 @@
+require 'logger'
+
+require 'spec_helper'
+require 'fluent/plugin/prometheus/placeholder_expander'
+require_relative '../shared'
+
+describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
+  let(:log) do
+    Logger.new('/dev/null')
+  end
+
+  let(:builder) do
+    Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
+  end
+
+  describe '#expand' do
+    context 'with static placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'tag' => '1.2.3',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}')).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}')).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}')).to eq('val1')
+
+        expect(expander.expand('${tag}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}')).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}')).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}')).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create new expander' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).never
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}')
+        expander.expand('${hostname}')
+      end
+
+      context 'when not found placeholder' do
+        it 'prints wanring log and as it is' do
+          expect(log).to receive(:warn).with('unknown placeholder `${tag_prefix[100]}` found').once
+
+          expander = builder.build(static_placeholder)
+          expect(expander.expand('${tag_prefix[100]}')).to eq('${tag_prefix[100]}')
+        end
+      end
+    end
+
+    context 'with dynamic placeholder' do
+      let(:static_placeholder) do
+        {
+          'hostname' => 'host_value',
+          'ary_value' => ['1', '2', '3'],
+          'hash_value' => { 'key1' => 'val1' },
+        }
+      end
+
+      let(:dynamic_placeholder) do
+        { 'tag' => '1.2.3'}
+      end
+
+      it 'expands values' do
+        expander = builder.build(static_placeholder)
+        expect(expander.expand('${hostname}', dynamic_placeholders: dynamic_placeholder)).to eq('host_value')
+        expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${hash_value["key1"]}', dynamic_placeholders: dynamic_placeholder)).to eq('val1')
+
+        expect(expander.expand('${tag}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
+        expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.1.2.1.2.3')
+        expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('3.2.3.1.2.3')
+      end
+
+      it 'does not create expander twice if given the same placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).once.and_call_original
+        expander = builder.build(static_placeholder)
+        placeholder = { 'tag' => 'val.test' }
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: placeholder)
+      end
+
+      it 'creates new expander for each placeholder' do
+        builder # cached before mock
+
+        expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).twice.and_call_original
+        expander = builder.build(static_placeholder)
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test' })
+        expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test2' })
+      end
+    end
+  end
+end

data/spec/fluent/plugin/prometheus_metrics_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+require 'fluent/plugin/in_prometheus'
+require 'fluent/test/driver/input'
+
+require 'net/http'
+
+describe Fluent::Plugin::PromMetricsAggregator do
+
+  metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+]
+
+  metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
+# HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+# TYPE fluentd_output_status_buffer_total_bytes gauge
+# HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
+fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
+]
+
+  metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
+# HELP fluentd_status_buffer_queue_length Current buffer queue length.
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE fluentd_status_buffer_total_bytes gauge
+# HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
+fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
+# TYPE log_counter counter
+# HELP log_counter the number of received logs
+log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
+# HELP empty_metric A metric with no data
+# TYPE empty_metric gauge
+# HELP http_request_duration_seconds The HTTP request latencies in seconds.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
+http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
+http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
+http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
+http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
+http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
+]
+
+  describe 'add_metrics' do
+    context '1st_metrics' do
+      it 'adds all fields' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1)
+      end
+    end
+    context '2nd_metrics' do
+      it 'append new metrics' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
+      end
+    end
+
+    context '3rd_metrics' do
+      it 'append existing metrics in the right place' do
+        all_metrics = Fluent::Plugin::PromMetricsAggregator.new
+        all_metrics.add_metrics(metrics_worker_1)
+        all_metrics.add_metrics(metrics_worker_2)
+        all_metrics.add_metrics(metrics_worker_3)
+        result_str = all_metrics.get_metrics
+
+        expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
+      end
+    end
+  end
+end