fluentd 1.13.0 → 1.14.0.rc

data/test/plugin/test_in_tcp.rb

@@ -5,21 +5,33 @@ require 'fluent/plugin/in_tcp'
 class TcpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @port = unused_port
   end
 
-  PORT = unused_port
-  BASE_CONFIG = %[
-    port #{PORT}
-    tag tcp
-  ]
-  CONFIG = BASE_CONFIG + %[
-    bind 127.0.0.1
-    format none
-  ]
-  IPv6_CONFIG = BASE_CONFIG + %[
-    bind ::1
-    format none
-  ]
+  def teardown
+    @port = nil
+  end
+
+  def base_config
+    %[
+      port #{@port}
+      tag tcp
+    ]
+  end
+
+  def ipv4_config
+    base_config + %[
+      bind 127.0.0.1
+      format none
+    ]
+  end
+
+  def ipv6_config
+    base_config + %[
+      bind ::1
+      format none
+    ]
+  end
 
   def create_driver(conf)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::TcpInput).configure(conf)
@@ -35,22 +47,23 @@ class TcpInputTest < Test::Unit::TestCase
 
 
   data(
-    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
-    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+    'ipv4' => ['127.0.0.1', :ipv4],
+    'ipv6' => ['::1', :ipv6],
   )
   test 'configure' do |data|
-    conf, bind, protocol = data
+    bind, protocol = data
+    conf = send("#{protocol}_config")
     omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
 
     d = create_driver(conf)
-    assert_equal PORT, d.instance.port
+    assert_equal @port, d.instance.port
     assert_equal bind, d.instance.bind
     assert_equal "\n", d.instance.delimiter
   end
 
   test ' configure w/o parse section' do
     assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
-      create_driver(BASE_CONFIG)
+      create_driver(base_config)
     }
   end
 
@@ -82,10 +95,10 @@ class TcpInputTest < Test::Unit::TestCase
     payloads = data['payloads']
     expecteds = data['expecteds']
 
-    d = create_driver(BASE_CONFIG + "format #{format}")
+    d = create_driver(base_config + "format #{format}")
     d.run(expect_records: 2) do
       payloads.each do |payload|
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send(payload, 0)
         end
       end
@@ -105,9 +118,9 @@ class TcpInputTest < Test::Unit::TestCase
     payloads = data['payloads']
     expecteds = data['expecteds']
 
-    d = create_driver(BASE_CONFIG + "format #{format}")
+    d = create_driver(base_config + "format #{format}")
     d.run(expect_records: 2) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         payloads.each do |payload|
           sock.send(payload, 0)
         end
@@ -123,13 +136,13 @@ class TcpInputTest < Test::Unit::TestCase
   end
 
   test 'source_hostname_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_hostname_key host
     !)
     hostname = nil
     d.run(expect_records: 1) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         sock.do_not_reverse_lookup = false
         hostname = sock.peeraddr[2]
         sock.send("test\n", 0)
@@ -144,13 +157,13 @@ class TcpInputTest < Test::Unit::TestCase
   end
 
   test 'source_address_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_address_key addr
     !)
     address = nil
     d.run(expect_records: 1) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         address = sock.peeraddr[3]
         sock.send("test\n", 0)
       end
@@ -165,7 +178,7 @@ class TcpInputTest < Test::Unit::TestCase
 
   sub_test_case '<security>' do
     test 'accept from allowed client' do
-      d = create_driver(CONFIG + %!
+      d = create_driver(ipv4_config + %!
         <security>
           <client>
             network 127.0.0.1
@@ -173,7 +186,7 @@ class TcpInputTest < Test::Unit::TestCase
         </security>
       !)
       d.run(expect_records: 1) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
@@ -185,7 +198,7 @@ class TcpInputTest < Test::Unit::TestCase
     end
 
     test 'deny from disallowed client' do
-      d = create_driver(CONFIG + %!
+      d = create_driver(ipv4_config + %!
         <security>
           <client>
             network 200.0.0.0
@@ -193,7 +206,7 @@ class TcpInputTest < Test::Unit::TestCase
         </security>
       !)
       d.run(shutdown: false, expect_records: 1, timeout: 2) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
@@ -205,7 +218,7 @@ class TcpInputTest < Test::Unit::TestCase
 
   sub_test_case '<extract>' do
     test 'extract tag from record field' do
-      d = create_driver(BASE_CONFIG + %!
+      d = create_driver(base_config + %!
         <parse>
           @type json
         </parse>
@@ -214,7 +227,7 @@ class TcpInputTest < Test::Unit::TestCase
         </extract>
       !)
       d.run(expect_records: 1) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           data = {'msg' => 'hello', 'tag' => 'helper_test'}
           sock.send("#{data.to_json}\n", 0)
         end

data/test/plugin/test_in_udp.rb

@@ -5,21 +5,33 @@ require 'fluent/plugin/in_udp'
 class UdpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @port = unused_port
   end
 
-  PORT = unused_port
-  BASE_CONFIG = %[
-    port #{PORT}
-    tag udp
-  ]
-  CONFIG = BASE_CONFIG + %!
-    bind 127.0.0.1
-    format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
-  !
-  IPv6_CONFIG = BASE_CONFIG + %!
-    bind ::1
-    format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
-  !
+  def teardown
+    @port = nil
+  end
+
+  def base_config
+    %[
+      port #{@port}
+      tag udp
+    ]
+  end
+
+  def ipv4_config
+    base_config + %!
+      bind 127.0.0.1
+      format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
+    !
+  end
+
+  def ipv6_config
+    base_config + %!
+      bind ::1
+      format /^\\[(?<time>[^\\]]*)\\] (?<message>.*)/
+    !
+  end
 
   def create_driver(conf)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::UdpInput).configure(conf)
@@ -45,15 +57,16 @@ class UdpInputTest < Test::Unit::TestCase
   end
 
   data(
-    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
-    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+    'ipv4' => ['127.0.0.1', :ipv4],
+    'ipv6' => ['::1', :ipv6],
   )
   test 'configure' do |data|
-    conf, bind, protocol = data
+    bind, protocol = data
+    conf = send("#{protocol}_config")
     omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
 
     d = create_driver(conf)
-    assert_equal PORT, d.instance.port
+    assert_equal @port, d.instance.port
     assert_equal bind, d.instance.bind
     assert_equal 4096, d.instance.message_length_limit
     assert_equal nil, d.instance.receive_buffer_size
@@ -61,16 +74,17 @@ class UdpInputTest < Test::Unit::TestCase
 
   test ' configure w/o parse section' do
     assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
-      create_driver(BASE_CONFIG)
+      create_driver(base_config)
     }
   end
 
   data(
-    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
-    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+    'ipv4' => ['127.0.0.1', :ipv4],
+    'ipv6' => ['::1', :ipv6],
   )
   test 'time_format' do |data|
-    conf, bind, protocol = data
+    bind, protocol = data
+    conf = send("#{protocol}_config")
     omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
 
     d = create_driver(conf)
@@ -81,7 +95,7 @@ class UdpInputTest < Test::Unit::TestCase
     ]
 
     d.run(expect_records: 2) do
-      create_udp_socket(bind, PORT) do |u|
+      create_udp_socket(bind, @port) do |u|
         tests.each do |test|
           u.send(test['msg'], 0)
         end
@@ -100,7 +114,7 @@ class UdpInputTest < Test::Unit::TestCase
   )
   test 'message_length_limit/body_size_limit compatibility' do |param|
 
-    d = create_driver(CONFIG + param)
+    d = create_driver(ipv4_config + param)
     assert_equal 2048, d.instance.message_length_limit
   end
 
@@ -141,9 +155,9 @@ class UdpInputTest < Test::Unit::TestCase
     payloads = data['payloads']
     expecteds = data['expecteds']
 
-    d = create_driver(BASE_CONFIG + "format #{format}")
+    d = create_driver(base_config + "format #{format}")
     d.run(expect_records: 2) do
-      create_udp_socket('127.0.0.1', PORT) do |u|
+      create_udp_socket('127.0.0.1', @port) do |u|
         payloads.each do |payload|
           u.send(payload, 0)
         end
@@ -159,13 +173,13 @@ class UdpInputTest < Test::Unit::TestCase
   end
 
   test 'remove_newline' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       remove_newline false
     !)
     payloads = ["test1\n", "test2\n"]
     d.run(expect_records: 2) do
-      create_udp_socket('127.0.0.1', PORT) do |u|
+      create_udp_socket('127.0.0.1', @port) do |u|
         payloads.each do |payload|
           u.send(payload, 0)
         end
@@ -182,13 +196,13 @@ class UdpInputTest < Test::Unit::TestCase
   end
 
   test 'source_hostname_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_hostname_key host
     !)
     hostname = nil
     d.run(expect_records: 1) do
-      create_udp_socket('127.0.0.1', PORT) do |u|
+      create_udp_socket('127.0.0.1', @port) do |u|
         u.send("test", 0)
         hostname = u.peeraddr[2]
       end
@@ -201,13 +215,13 @@ class UdpInputTest < Test::Unit::TestCase
   end
 
   test 'source_address_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_address_key addr
     !)
     address = nil
     d.run(expect_records: 1) do
-      create_udp_socket('127.0.0.1', PORT) do |u|
+      create_udp_socket('127.0.0.1', @port) do |u|
         u.send("test", 0)
         address = u.peeraddr[3]
       end
@@ -223,7 +237,7 @@ class UdpInputTest < Test::Unit::TestCase
     # doesn't check exact value because it depends on platform and condition
 
     # check if default socket and in_udp's one without receive_buffer_size have same size buffer
-    d0 = create_driver(BASE_CONFIG + %!
+    d0 = create_driver(base_config + %!
       format none
     !)
     d0.run do
@@ -237,7 +251,7 @@ class UdpInputTest < Test::Unit::TestCase
     end
 
     # check if default socket and in_udp's one with receive_buffer_size have different size buffer
-    d1 = create_driver(BASE_CONFIG + %!
+    d1 = create_driver(base_config + %!
       format none
       receive_buffer_size 1001
     !)

data/test/plugin/test_input.rb

@@ -85,6 +85,17 @@ class InputTest < Test::Unit::TestCase
     end
   end
 
+  test 'can use metrics plugins and fallback methods' do
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    %w[emit_size_metrics emit_records_metrics].each do |metric_name|
+      assert_true @p.instance_variable_get(:"@#{metric_name}").is_a?(Fluent::Plugin::Metrics)
+    end
+
+    assert_equal 0, @p.emit_size
+    assert_equal 0, @p.emit_records
+  end
+
   test 'are not available with multi workers configuration in default' do
     assert_false @p.multi_workers_ready?
   end

data/test/plugin/test_metrics.rb

@@ -0,0 +1,294 @@
+require_relative '../helper'
+require 'fluent/plugin/metrics'
+require 'fluent/plugin/base'
+require 'fluent/system_config'
+
+class BareMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('bare', self)
+
+  private
+
+  # Just override for tests.
+  def has_methods_for_counter?
+    false
+  end
+end
+
+class BasicCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedCounterMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :set, :set_counter
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def add(value)
+    @data += value
+  end
+  def set_counter(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class BasicGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub(value)
+    @data -= value
+  end
+  def set(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class AliasedGaugeMetrics < Fluent::Plugin::Metrics
+  Fluent::Plugin.register_metrics('example', self)
+
+  attr_reader :data
+
+  def initialize
+    super
+    @data = 0
+  end
+  def configure(conf)
+    super
+    class << self
+      alias_method :dec, :dec_gauge
+      alias_method :set, :set_gauge
+      alias_method :sub, :sub_gauge
+    end
+  end
+  def get
+    @data
+  end
+  def inc
+    @data +=1
+  end
+  def dec_gauge
+    @data -=1
+  end
+  def add(value)
+    @data += value
+  end
+  def sub_gauge(value)
+    @data -= value
+  end
+  def set_gauge(value)
+    @data = value
+  end
+  def close
+    @data = 0
+    super
+  end
+end
+
+class StorageTest < Test::Unit::TestCase
+  sub_test_case 'BareMetrics' do
+    setup do
+      @m = BareMetrics.new
+      @m.configure(config_element())
+    end
+
+    test 'is configured with plugin information and system config' do
+      m = BareMetrics.new
+      m.configure(config_element('metrics', '', {}))
+
+      assert_false m.use_gauge_metric
+      assert_false m.has_methods_for_counter
+      assert_false m.has_methods_for_gauge
+    end
+
+    test 'all bare operations are not defined yet' do
+      assert_raise NotImplementedError do
+        @m.get
+      end
+      assert_raise NotImplementedError do
+        @m.inc
+      end
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+      assert_raise NotImplementedError do
+        @m.add(10)
+      end
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+      assert_raise NotImplementedError do
+        @m.set(123)
+      end
+    end
+  end
+
+  sub_test_case 'BasicCounterMetric' do
+    setup do
+      @m = BasicCounterMetrics.new
+      @m.configure(config_element('metrics', '', {'@id' => '1'}))
+    end
+
+    test 'all basic counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'AliasedCounterMetric' do
+    setup do
+      @m = AliasedCounterMetrics.new
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased counter operations work well' do
+      assert_true @m.has_methods_for_counter
+      assert_false @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      assert_raise NotImplementedError do
+        @m.dec
+      end
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      assert_raise NotImplementedError do
+        @m.sub(11)
+      end
+    end
+  end
+
+  sub_test_case 'BasicGaugeMetric' do
+    setup do
+      @m = BasicGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all basic gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+
+  sub_test_case 'AliasedGaugeMetric' do
+    setup do
+      @m = AliasedGaugeMetrics.new
+      @m.use_gauge_metric = true
+      @m.configure(config_element('metrics', '', {}))
+    end
+
+    test 'all aliased gauge operations work well' do
+      assert_false @m.has_methods_for_counter
+      assert_true @m.has_methods_for_gauge
+
+      assert_equal 0, @m.get
+      assert_equal 1, @m.inc
+
+      @m.add(20)
+      assert_equal 21, @m.get
+      @m.dec
+      assert_equal 20, @m.get
+
+      @m.set(100)
+      assert_equal 100, @m.get
+      @m.sub(11)
+      assert_equal 89, @m.get
+    end
+  end
+end