fluentd 1.13.0 → 1.14.0.rc

data/test/plugin_helper/test_socket.rb

@@ -6,17 +6,24 @@ require 'socket'
 require 'openssl'
 
 class SocketHelperTest < Test::Unit::TestCase
-  PORT = unused_port
   CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/without_ca')
   CA_CERT_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
   CERT_CHAINS_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/cert_chains')
 
+  def setup
+    @port = unused_port
+  end
+
+  def teardown
+    @port = nil
+  end
+
   class SocketHelperTestPlugin < Fluent::Plugin::TestBase
     helpers :socket
   end
 
   class EchoTLSServer
-    def initialize(host = '127.0.0.1', port = PORT, cert_path: nil, private_key_path: nil, ca_path: nil)
+    def initialize(port, host: '127.0.0.1', cert_path: nil, private_key_path: nil, ca_path: nil)
       server = TCPServer.open(host, port)
       ctx = OpenSSL::SSL::SSLContext.new
       ctx.cert = OpenSSL::X509::Certificate.new(File.open(cert_path)) if cert_path
@@ -91,8 +98,8 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.join(CERT_DIR, 'cert.pem')
     private_key_path = File.join(CERT_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -105,8 +112,8 @@ class SocketHelperTest < Test::Unit::TestCase
 
     ca_cert_path = File.join(CA_CERT_DIR, 'ca-cert.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_paths: [ca_cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_paths: [ca_cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -121,8 +128,8 @@ class SocketHelperTest < Test::Unit::TestCase
     client_cert_path = File.join(CERT_CHAINS_DIR, 'cert.pem')
     client_private_key_path = File.join(CERT_CHAINS_DIR, 'cert-key.pem')
 
-    EchoTLSServer.new(cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
-      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
+    EchoTLSServer.new(@port, cert_path: cert_path, private_key_path: private_key_path, ca_path: client_ca_cert_path).start do
+      client = SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, verify_fqdn: false, cert_path: client_cert_path, private_key_path: client_private_key_path, cert_paths: [cert_path])
       client.write('hello')
       assert_equal 'hello', client.readpartial(100)
       client.close
@@ -133,7 +140,7 @@ class SocketHelperTest < Test::Unit::TestCase
     cert_path = File.expand_path(File.dirname(__FILE__) + '/data/cert/empty.pem')
 
     assert_raise Fluent::ConfigError do
-      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', PORT, cert_path: cert_path)
+      SocketHelperTestPlugin.new.socket_create_tls('127.0.0.1', @port, cert_path: cert_path)
     end
   end
 end

data/test/test_event_time.rb

@@ -64,8 +64,8 @@ class EventTimeTest < Test::Unit::TestCase
 
   test 'Oj.dump' do
     time = Fluent::EventTime.new(100)
-    require 'fluent/env'
-    Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+    require 'fluent/oj_options'
+    Fluent::OjOptions.load_env
     assert_equal('{"time":100}', Oj.dump({'time' => time}))
     assert_equal('["tag",100,{"key":"value"}]', Oj.dump(["tag", time, {"key" => "value"}], mode: :compat))
   end

data/test/test_oj_options.rb

@@ -0,0 +1,55 @@
+require_relative 'helper'
+require 'fluent/test'
+require 'fluent/oj_options'
+
+class OjOptionsTest < ::Test::Unit::TestCase
+  begin
+    require 'oj'
+    @@oj_is_avaibale = true
+  rescue LoadError
+    @@oj_is_avaibale = false
+  end
+
+  setup do
+    @orig_env = {}
+    ENV.each do |key, value|
+      @orig_env[key] = value if key.start_with?("FLUENT_OJ_OPTION_")
+    end
+  end
+
+  teardown do
+    ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+    @orig_env.each { |key, value| ENV[key] = value }
+  end
+
+  test "available?" do
+    assert_equal(@@oj_is_avaibale, Fluent::OjOptions.available?)
+  end
+
+  sub_test_case "set by environment variable" do
+    test "when no env vars set, returns default options" do
+      ENV.delete_if { |key| key.start_with?("FLUENT_OJ_OPTION_") }
+      defaults = Fluent::OjOptions::DEFAULTS
+      assert_equal(defaults, Fluent::OjOptions.load_env)
+      assert_equal(defaults, Oj.default_options.slice(*defaults.keys)) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with valid value, default is overridden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":bigdecimal"
+      assert_equal(:bigdecimal, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:bigdecimal, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "valid env var passed with invalid value, default is not overriden" do
+      ENV["FLUENT_OJ_OPTION_BIGDECIMAL_LOAD"] = ":conor"
+      assert_equal(:float, Fluent::OjOptions.load_env[:bigdecimal_load])
+      assert_equal(:float, Oj.default_options[:bigdecimal_load]) if @@oj_is_avaibale
+    end
+
+    test "invalid env var passed, nothing done with it" do
+      ENV["FLUENT_OJ_OPTION_CONOR"] = ":conor"
+      assert_equal(nil, Fluent::OjOptions.load_env[:conor])
+      assert_equal(nil, Oj.default_options[:conor]) if @@oj_is_avaibale
+    end
+  end
+end

data/test/test_plugin_classes.rb

@@ -5,11 +5,78 @@ require 'fluent/plugin/bare_output'
 require 'fluent/plugin/filter'
 
 module FluentTest
+  class FluentTestCounterMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_counter', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data +=1
+    end
+    def add(value)
+      @data += value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
+  class FluentTestGaugeMetrics < Fluent::Plugin::Metrics
+    Fluent::Plugin.register_metrics('test_gauge', self)
+
+    attr_reader :data
+
+    def initialize
+      super
+      @data = 0
+    end
+    def get
+      @data
+    end
+    def inc
+      @data += 1
+    end
+    def dec
+      @data -=1
+    end
+    def add(value)
+      @data += value
+    end
+    def sub(value)
+      @data -= value
+    end
+    def set(value)
+      @data = value
+    end
+    def close
+      @data = 0
+      super
+    end
+  end
+
   class FluentTestInput < ::Fluent::Plugin::Input
     ::Fluent::Plugin.register_input('test_in', self)
 
     attr_reader :started
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -28,6 +95,13 @@ module FluentTest
 
     config_param :num, :integer, default: 10000
 
+    def initialize
+      super
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def start
       super
       @started = true
@@ -49,6 +123,15 @@ module FluentTest
     def initialize
       super
       @events = Hash.new { |h, k| h[k] = [] }
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :events
@@ -168,6 +251,19 @@ module FluentTest
   class FluentTestErrorOutput < ::Fluent::Plugin::Output
     ::Fluent::Plugin.register_output('test_out_error', self)
 
+    def initialize
+      super
+      # stub metrics instances
+      @num_errors_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
+      @write_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @rollback_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @flush_time_count_metrics = FluentTest::FluentTestCounterMetrics.new
+      @slow_flush_count_metrics = FluentTest::FluentTestCounterMetrics.new
+    end
+
     def format(tag, time, record)
       raise "emit error!"
     end
@@ -184,6 +280,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num
@@ -213,6 +312,9 @@ module FluentTest
       super()
       @num = 0
       @field = field
+      # stub metrics instances
+      @emit_records_metrics = FluentTest::FluentTestCounterMetrics.new
+      @emit_size_metrics = FluentTest::FluentTestCounterMetrics.new
     end
 
     attr_reader :num

data/test/test_root_agent.rb

@@ -16,7 +16,8 @@ class RootAgentTest < ::Test::Unit::TestCase
 
   data(
     'suppress interval' => [{'emit_error_log_interval' => 30}, {:@suppress_emit_error_log_interval => 30}],
-    'without source' => [{'without_source' => true}, {:@without_source => true}]
+    'without source' => [{'without_source' => true}, {:@without_source => true}],
+    'enable input metrics' => [{'enable_input_metrics' => true}, {:@enable_input_metrics => true}],
     )
   def test_initialize_with_opt(data)
     opt, expected = data
@@ -109,6 +110,34 @@ EOC
       end
     end
 
+    test 'raises configuration error for label without name' do
+      conf = <<-EOC
+<label>
+  @type test_out
+</label>
+EOC
+      errmsg = "Missing symbol argument on <label> directive"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
+    test 'raises configuration error for <label @ROOT>' do
+      conf = <<-EOC
+<source>
+  @type test_in
+  @label @ROOT
+</source>
+<label @ROOT>
+  @type test_out
+</label>
+EOC
+      errmsg = "@ROOT for <label> is not permitted, reserved for getting root router"
+      assert_raise Fluent::ConfigError.new(errmsg) do
+        configure_ra(conf)
+      end
+    end
+
     test 'raises configuration error if there are not match sections in label section' do
       conf = <<-EOC
 <source>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.14.0.rc
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-29 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,7 +107,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -117,7 +117,7 @@ dependencies:
         version: 0.5.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: sigdump
   requirement: !ruby/object:Gem::Requirement
@@ -373,9 +373,9 @@ files:
 - ".deepsource.toml"
 - ".drone.yml"
 - ".github/ISSUE_TEMPLATE.md"
-- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
 - ".github/ISSUE_TEMPLATE/config.yml"
-- ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux-test.yaml"
@@ -519,6 +519,7 @@ files:
 - lib/fluent/match.rb
 - lib/fluent/mixin.rb
 - lib/fluent/msgpack_factory.rb
+- lib/fluent/oj_options.rb
 - lib/fluent/output.rb
 - lib/fluent/output_chain.rb
 - lib/fluent/parser.rb
@@ -568,6 +569,8 @@ files:
 - lib/fluent/plugin/in_udp.rb
 - lib/fluent/plugin/in_unix.rb
 - lib/fluent/plugin/input.rb
+- lib/fluent/plugin/metrics.rb
+- lib/fluent/plugin/metrics_local.rb
 - lib/fluent/plugin/multi_output.rb
 - lib/fluent/plugin/out_copy.rb
 - lib/fluent/plugin/out_exec.rb
@@ -632,6 +635,7 @@ files:
 - lib/fluent/plugin_helper/http_server/server.rb
 - lib/fluent/plugin_helper/http_server/ssl_context_builder.rb
 - lib/fluent/plugin_helper/inject.rb
+- lib/fluent/plugin_helper/metrics.rb
 - lib/fluent/plugin_helper/parser.rb
 - lib/fluent/plugin_helper/record_accessor.rb
 - lib/fluent/plugin_helper/retry_state.rb
@@ -664,6 +668,7 @@ files:
 - lib/fluent/test/driver/multi_output.rb
 - lib/fluent/test/driver/output.rb
 - lib/fluent/test/driver/parser.rb
+- lib/fluent/test/driver/storage.rb
 - lib/fluent/test/driver/test_event_router.rb
 - lib/fluent/test/filter_test.rb
 - lib/fluent/test/formatter_test.rb
@@ -689,12 +694,14 @@ files:
 - templates/new_gem/lib/fluent/plugin/input.rb.erb
 - templates/new_gem/lib/fluent/plugin/output.rb.erb
 - templates/new_gem/lib/fluent/plugin/parser.rb.erb
+- templates/new_gem/lib/fluent/plugin/storage.rb.erb
 - templates/new_gem/test/helper.rb.erb
 - templates/new_gem/test/plugin/test_filter.rb.erb
 - templates/new_gem/test/plugin/test_formatter.rb.erb
 - templates/new_gem/test/plugin/test_input.rb.erb
 - templates/new_gem/test/plugin/test_output.rb.erb
 - templates/new_gem/test/plugin/test_parser.rb.erb
+- templates/new_gem/test/plugin/test_storage.rb.erb
 - templates/plugin_config_formatter/param.md-compact.erb
 - templates/plugin_config_formatter/param.md-table.erb
 - templates/plugin_config_formatter/param.md.erb
@@ -790,6 +797,8 @@ files:
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
 - test/plugin/test_metadata.rb
+- test/plugin/test_metrics.rb
+- test/plugin/test_metrics_local.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
@@ -865,6 +874,7 @@ files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_http_server_helper.rb
 - test/plugin_helper/test_inject.rb
+- test/plugin_helper/test_metrics.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb
@@ -899,6 +909,7 @@ files:
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_msgpack_factory.rb
+- test/test_oj_options.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb
@@ -918,7 +929,7 @@ homepage: https://www.fluentd.org/
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -929,12 +940,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.2.5
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Fluentd event collector
 test_files:
@@ -1029,6 +1040,8 @@ test_files:
 - test/plugin/test_in_unix.rb
 - test/plugin/test_input.rb
 - test/plugin/test_metadata.rb
+- test/plugin/test_metrics.rb
+- test/plugin/test_metrics_local.rb
 - test/plugin/test_multi_output.rb
 - test/plugin/test_out_copy.rb
 - test/plugin/test_out_exec.rb
@@ -1104,6 +1117,7 @@ test_files:
 - test/plugin_helper/test_formatter.rb
 - test/plugin_helper/test_http_server_helper.rb
 - test/plugin_helper/test_inject.rb
+- test/plugin_helper/test_metrics.rb
 - test/plugin_helper/test_parser.rb
 - test/plugin_helper/test_record_accessor.rb
 - test/plugin_helper/test_retry_state.rb
@@ -1138,6 +1152,7 @@ test_files:
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_msgpack_factory.rb
+- test/test_oj_options.rb
 - test/test_output.rb
 - test/test_plugin.rb
 - test/test_plugin_classes.rb