fluentd 1.16.5-x86-mingw32 → 1.17.0-x86-mingw32

data/test/plugin/test_out_http.rb

@@ -7,6 +7,7 @@ require 'webrick/https'
 require 'net/http'
 require 'uri'
 require 'json'
+require 'aws-sdk-core'
 
 # WEBrick's ProcHandler doesn't handle PUT by default
 module WEBrick::HTTPServlet
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
     end
   end
 
+
+  sub_test_case 'aws sigv4 auth' do
+    setup do
+      @@fake_aws_credentials = Aws::Credentials.new(
+        'fakeaccess',
+        'fakesecret',
+        'fake session token'
+      )
+    end
+
+    def server_port
+      19883
+    end
+
+    def test_aws_sigv4_sts_role_arn
+      stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
+        stub(credentials_provider).credentials {
+          @@fake_aws_credentials
+        }
+        credentials_provider
+      end
+
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+            aws_role_arn arn:aws:iam::123456789012:role/MyRole
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+
+    def test_aws_sigv4_no_role
+      stub(Aws::CredentialProviderChain).new do |provider_chain|
+        stub(provider_chain).resolve {
+          @@fake_aws_credentials
+        }
+        provider_chain
+      end
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+  end
+
   sub_test_case 'HTTPS' do
     def server_port
       19882
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
       assert_not_empty result.headers
     end
   end
+
+  sub_test_case 'connection_reuse' do
+    def server_port
+      19883
+    end
+
+    def test_connection_recreation
+      d = create_driver(%[
+        endpoint http://127.0.0.1:#{server_port}/test
+        reuse_connections true
+      ])
+
+      d.run(default_tag: 'test.http', shutdown: false) do
+        d.feed(test_events[0])
+      end
+
+      data = @@result.data
+
+      # Restart server to simulate connection loss
+      @@http_server_thread.kill
+      @@http_server_thread.join
+      @@http_server_thread = Thread.new do
+        run_http_server
+      end
+
+      d.run(default_tag: 'test.http') do
+        d.feed(test_events[1])
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, data.concat(result.data)
+      assert_not_empty result.headers
+    end
+  end
 end

data/test/plugin/test_owned_by.rb

@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
 
       assert_equal parent.object_id, child.owner.object_id
 
-      assert child.instance_eval{ @_plugin_id_configured }
       assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
 
       assert_equal Fluent::Log::LEVEL_TRACE, child.log.level

data/test/plugin/test_parser_json.rb

@@ -135,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case "various record pattern" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
+    data("Only number", { record: "0", expected: [nil] }, keep: true)
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": 2}]',
+        expected: [{"k1" => 1}, {"k2" => 2}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of both Hash and invalid",
+      {
+        record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
+        expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
+      },
+      keep: true,
+    )
+    data(
+      "Array of all invalid",
+      {
+        record: '["string", 0, [{"k": 0}]]',
+        expected: [nil, nil, nil]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data(
+      "Hash",
+      {
+        record: '{"k1": 1, "k2": ""}',
+        expected: [{"k1" => 1, "k2" => nil}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": ""}]',
+        expected: [{"k1" => 1}, {"k2" => nil}]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
 end

data/test/plugin/test_parser_msgpack.rb

@@ -0,0 +1,127 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_msgpack'
+
+class MessagePackParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)
+  end
+
+  sub_test_case "simple setting" do
+    data(
+      "Normal Hash",
+      {
+        input: "\x82\xA7message\xADHello msgpack\xA3numd",
+        expected: [{"message" => "Hello msgpack", "num" => 100}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        input: "\x92\x81\xA7message\xA3foo\x81\xA7message\xA3bar",
+        expected: [{"message"=>"foo"}, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String",
+      {
+        # ["foo", "bar"].to_msgpack
+        input: "\x92\xA3foo\xA3bar",
+        expected: [nil, nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String and Hash",
+      {
+        # ["foo", {message: "bar"}].to_msgpack
+        input: "\x92\xA3foo\x81\xA7message\xA3bar",
+        expected: [nil, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data(
+      "Normal hash",
+      {
+        # {message: "foo", empty: ""}.to_msgpack
+        input: "\x82\xA7message\xA3foo\xA5empty\xA0",
+        expected: [{"message" => "foo", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        # [{message: "foo", empty: ""}, {message: "bar", empty: ""}].to_msgpack
+        input: "\x92\x82\xA7message\xA3foo\xA5empty\xA0\x82\xA7message\xA3bar\xA5empty\xA0",
+        expected: [{"message"=>"foo", "empty" => nil}, {"message"=>"bar", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("null_empty_string").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("null_empty_string").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+end

data/test/plugin/test_storage.rb

@@ -68,7 +68,6 @@ class StorageTest < Test::Unit::TestCase
 
       assert_equal 'mytest', s.owner.system_config.process_name
       assert_equal '1', s.instance_eval{ @_plugin_id }
-      assert_equal true, s.instance_eval{ @_plugin_id_configured }
     end
 
     test 'does NOT have features for high-performance/high-consistent storages' do

data/test/plugin_helper/test_child_process.rb

@@ -569,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can specify subprocess name' do
       io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
-      process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
+      process_naming_enabled = (IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
       Process.kill(:TERM, io.pid) rescue nil
       io.close rescue nil
 
@@ -586,7 +586,7 @@ class ChildProcessTest < Test::Unit::TestCase
           m.lock
           ran = true
           pids << @d.child_process_id
-          proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
+          proc_lines += IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }
           m.unlock
           readio.read
         end
@@ -645,8 +645,8 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can change working directory' do
       # check my real /tmp directory (for mac)
-      cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
-      mytmpdir = open(cmd){|io| io.read.chomp }
+      cmd = ['ruby', '-e', 'Dir.chdir("/tmp"); puts Dir.pwd']
+      mytmpdir = IO.popen(cmd){|io| io.read.chomp }
 
       m = Mutex.new
       str = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.5
+  version: 1.17.0
 platform: x86-mingw32
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2024-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +200,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: drb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: win32-service
   requirement: !ruby/object:Gem::Requirement
@@ -416,6 +458,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.50.0
+- !ruby/object:Gem::Dependency
+  name: aws-sigv4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.191'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.191'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Fluentd is an open source data collector designed to scale and simplify
   log management. It can collect, process and ship many kinds of data in near real-time.
 email:
@@ -435,12 +519,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".deepsource.toml"
+- ".github/DISCUSSION_TEMPLATE/q-a-japanese.yml"
+- ".github/DISCUSSION_TEMPLATE/q-a.yml"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.yml"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
 - ".github/workflows/stale-actions.yml"
+- ".github/workflows/test-ruby-head.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ADOPTERS.md
@@ -812,6 +899,10 @@ files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/data/log_numeric/01.log
+- test/plugin/data/log_numeric/02.log
+- test/plugin/data/log_numeric/12.log
+- test/plugin/data/log_numeric/14.log
 - test/plugin/data/sd_file/config
 - test/plugin/data/sd_file/config.json
 - test/plugin/data/sd_file/config.yaml
@@ -896,6 +987,7 @@ files:
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
+- test/plugin/test_parser_msgpack.rb
 - test/plugin/test_parser_multiline.rb
 - test/plugin/test_parser_nginx.rb
 - test/plugin/test_parser_none.rb
@@ -1005,14 +1097,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.13
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector
@@ -1055,6 +1147,10 @@ test_files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/data/log_numeric/01.log
+- test/plugin/data/log_numeric/02.log
+- test/plugin/data/log_numeric/12.log
+- test/plugin/data/log_numeric/14.log
 - test/plugin/data/sd_file/config
 - test/plugin/data/sd_file/config.json
 - test/plugin/data/sd_file/config.yaml
@@ -1139,6 +1235,7 @@ test_files:
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
+- test/plugin/test_parser_msgpack.rb
 - test/plugin/test_parser_multiline.rb
 - test/plugin/test_parser_nginx.rb
 - test/plugin/test_parser_none.rb