fluentd 1.16.3 → 1.17.0

data/test/plugin/test_parser_json.rb

@@ -135,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case "various record pattern" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
+    data("Only number", { record: "0", expected: [nil] }, keep: true)
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": 2}]',
+        expected: [{"k1" => 1}, {"k2" => 2}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of both Hash and invalid",
+      {
+        record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
+        expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
+      },
+      keep: true,
+    )
+    data(
+      "Array of all invalid",
+      {
+        record: '["string", 0, [{"k": 0}]]',
+        expected: [nil, nil, nil]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data(
+      "Hash",
+      {
+        record: '{"k1": 1, "k2": ""}',
+        expected: [{"k1" => 1, "k2" => nil}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": ""}]',
+        expected: [{"k1" => 1}, {"k2" => nil}]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
 end

data/test/plugin/test_parser_msgpack.rb

@@ -0,0 +1,127 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_msgpack'
+
+class MessagePackParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)
+  end
+
+  sub_test_case "simple setting" do
+    data(
+      "Normal Hash",
+      {
+        input: "\x82\xA7message\xADHello msgpack\xA3numd",
+        expected: [{"message" => "Hello msgpack", "num" => 100}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        input: "\x92\x81\xA7message\xA3foo\x81\xA7message\xA3bar",
+        expected: [{"message"=>"foo"}, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String",
+      {
+        # ["foo", "bar"].to_msgpack
+        input: "\x92\xA3foo\xA3bar",
+        expected: [nil, nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String and Hash",
+      {
+        # ["foo", {message: "bar"}].to_msgpack
+        input: "\x92\xA3foo\x81\xA7message\xA3bar",
+        expected: [nil, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data(
+      "Normal hash",
+      {
+        # {message: "foo", empty: ""}.to_msgpack
+        input: "\x82\xA7message\xA3foo\xA5empty\xA0",
+        expected: [{"message" => "foo", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        # [{message: "foo", empty: ""}, {message: "bar", empty: ""}].to_msgpack
+        input: "\x92\x82\xA7message\xA3foo\xA5empty\xA0\x82\xA7message\xA3bar\xA5empty\xA0",
+        expected: [{"message"=>"foo", "empty" => nil}, {"message"=>"bar", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("null_empty_string").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("null_empty_string").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+end

data/test/plugin/test_storage.rb

@@ -68,7 +68,6 @@ class StorageTest < Test::Unit::TestCase
 
       assert_equal 'mytest', s.owner.system_config.process_name
       assert_equal '1', s.instance_eval{ @_plugin_id }
-      assert_equal true, s.instance_eval{ @_plugin_id_configured }
     end
 
     test 'does NOT have features for high-performance/high-consistent storages' do

data/test/plugin_helper/test_child_process.rb

@@ -515,6 +515,9 @@ class ChildProcessTest < Test::Unit::TestCase
   end
 
   test 'can scrub characters without exceptions' do
+    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
+      pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
+    end
     m = Mutex.new
     str = nil
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
@@ -529,19 +532,25 @@ class ChildProcessTest < Test::Unit::TestCase
       sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
       m.lock
       assert_equal Encoding.find('utf-8'), str.encoding
-      expected = "\xEF\xBF\xBD\xEF\xBF\xBD\x00\xEF\xBF\xBD\xEF\xBF\xBD".force_encoding("utf-8")
+      replacement = "\uFFFD" # U+FFFD (REPLACEMENT CHARACTER)
+      nul = "\x00" # U+0000 (NUL)
+      expected = replacement * 2 + nul + replacement * 2
       assert_equal expected, str
       @d.stop; @d.shutdown; @d.close; @d.terminate
     end
   end
 
   test 'can scrub characters without exceptions and replace specified chars' do
+    if Gem::Version.create(RUBY_VERSION) >= Gem::Version.create('3.3.0')
+      pend "Behaviour of IO#set_encoding is changed as of Ruby 3.3 (#4058)"
+    end
     m = Mutex.new
     str = nil
+    replacement = "?"
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       args = ['-e', 'STDOUT.set_encoding("ascii-8bit"); STDOUT.write "\xFF\xFF\x00\xF0\xF0"']
-      @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: '?') do |io|
+      @d.child_process_execute(:t13b, "ruby", arguments: args, mode: [:read], scrub: true, replace_string: replacement) do |io|
         m.lock
         ran = true
         str = io.read
@@ -550,7 +559,8 @@ class ChildProcessTest < Test::Unit::TestCase
       sleep TEST_WAIT_INTERVAL_FOR_BLOCK_RUNNING until m.locked? || ran
       m.lock
       assert_equal Encoding.find('utf-8'), str.encoding
-      expected = "??\x00??".force_encoding("utf-8")
+      nul = "\x00" # U+0000 (NUL)
+      expected = replacement * 2 + nul + replacement * 2
       assert_equal expected, str
       @d.stop; @d.shutdown; @d.close; @d.terminate
     end
@@ -559,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can specify subprocess name' do
       io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
-      process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
+      process_naming_enabled = (IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
       Process.kill(:TERM, io.pid) rescue nil
       io.close rescue nil
 
@@ -576,7 +586,7 @@ class ChildProcessTest < Test::Unit::TestCase
           m.lock
           ran = true
           pids << @d.child_process_id
-          proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
+          proc_lines += IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }
           m.unlock
           readio.read
         end
@@ -635,8 +645,8 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can change working directory' do
       # check my real /tmp directory (for mac)
-      cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
-      mytmpdir = open(cmd){|io| io.read.chomp }
+      cmd = ['ruby', '-e', 'Dir.chdir("/tmp"); puts Dir.pwd']
+      mytmpdir = IO.popen(cmd){|io| io.read.chomp }
 
       m = Mutex.new
       str = nil

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.16.3
+  version: 1.17.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-11-14 00:00:00.000000000 Z
+date: 2024-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -200,6 +200,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: base64
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: drb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -360,6 +402,48 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.50.0
+- !ruby/object:Gem::Dependency
+  name: aws-sigv4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.191'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.191'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: Fluentd is an open source data collector designed to scale and simplify
   log management. It can collect, process and ship many kinds of data in near real-time.
 email:
@@ -379,15 +463,16 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".deepsource.toml"
+- ".github/DISCUSSION_TEMPLATE/q-a-japanese.yml"
+- ".github/DISCUSSION_TEMPLATE/q-a.yml"
 - ".github/ISSUE_TEMPLATE.md"
-- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/bug_report.yml"
 - ".github/ISSUE_TEMPLATE/config.yml"
-- ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/ISSUE_TEMPLATE/feature_request.yml"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/workflows/linux-test.yaml"
-- ".github/workflows/macos-test.yaml"
 - ".github/workflows/stale-actions.yml"
-- ".github/workflows/windows-test.yaml"
+- ".github/workflows/test-ruby-head.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ADOPTERS.md
 - AUTHORS
@@ -758,6 +843,10 @@ files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/data/log_numeric/01.log
+- test/plugin/data/log_numeric/02.log
+- test/plugin/data/log_numeric/12.log
+- test/plugin/data/log_numeric/14.log
 - test/plugin/data/sd_file/config
 - test/plugin/data/sd_file/config.json
 - test/plugin/data/sd_file/config.yaml
@@ -842,6 +931,7 @@ files:
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
+- test/plugin/test_parser_msgpack.rb
 - test/plugin/test_parser_multiline.rb
 - test/plugin/test_parser_nginx.rb
 - test/plugin/test_parser_none.rb
@@ -951,14 +1041,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.4.13
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector
@@ -1001,6 +1091,10 @@ test_files:
 - test/plugin/data/log/foo/bar.log
 - test/plugin/data/log/foo/bar2
 - test/plugin/data/log/test.log
+- test/plugin/data/log_numeric/01.log
+- test/plugin/data/log_numeric/02.log
+- test/plugin/data/log_numeric/12.log
+- test/plugin/data/log_numeric/14.log
 - test/plugin/data/sd_file/config
 - test/plugin/data/sd_file/config.json
 - test/plugin/data/sd_file/config.yaml
@@ -1085,6 +1179,7 @@ test_files:
 - test/plugin/test_parser_csv.rb
 - test/plugin/test_parser_json.rb
 - test/plugin/test_parser_labeled_tsv.rb
+- test/plugin/test_parser_msgpack.rb
 - test/plugin/test_parser_multiline.rb
 - test/plugin/test_parser_nginx.rb
 - test/plugin/test_parser_none.rb

data/.github/workflows/macos-test.yaml

@@ -1,34 +0,0 @@
-name: Testing on macOS
-
-on:
-  push:
-    branches: [master, v1.16]
-  pull_request:
-    branches: [master, v1.16]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['3.2', '3.1', '3.0', '2.7']
-        os: [macos-latest]
-        experimental: [true]
-        include:
-          - ruby-version: head
-            os: macos-latest
-            experimental: true
-
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Install dependencies
-        run: bundle install
-      - name: Run tests
-        run: bundle exec rake test

data/.github/workflows/windows-test.yaml

@@ -1,49 +0,0 @@
-name: Testing on Windows
-
-on:
-  push:
-    branches: [master, v1.16]
-  pull_request:
-    branches: [master, v1.16]
-
-jobs:
-  test:
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
-    strategy:
-      fail-fast: false
-      matrix:
-        ruby-version: ['3.2', '3.1', '2.7']
-        os:
-          - windows-latest
-        experimental: [false]
-        include:
-          - ruby-version: head
-            os: windows-latest
-            experimental: true
-          - ruby-version: '3.0.3'
-            os: windows-latest
-            experimental: false
-            # On Ruby 3.0, we need to use fiddle 1.0.8 or later to retrieve correct
-            # error code. In addition, we have to specify the path of fiddle by RUBYLIB
-            # because RubyInstaller loads Ruby's bundled fiddle before initializing gem.
-            # See also:
-            # * https://github.com/ruby/fiddle/issues/72
-            # * https://bugs.ruby-lang.org/issues/17813
-            # * https://github.com/oneclick/rubyinstaller2/blob/8225034c22152d8195bc0aabc42a956c79d6c712/lib/ruby_installer/build/dll_directory.rb
-            ruby-lib-opt: RUBYLIB=%RUNNER_TOOL_CACHE%/Ruby/3.0.3/x64/lib/ruby/gems/3.0.0/gems/fiddle-1.1.0/lib
-
-    name: Unit testing with Ruby ${{ matrix.ruby-version }} on ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Add Fiddle 1.1.0
-        if: ${{ matrix.ruby-version == '3.0.3' }}
-        run: gem install fiddle --version 1.1.0
-      - name: Install dependencies
-        run: ridk exec bundle install
-      - name: Run tests
-        run: bundle exec rake test TESTOPTS=-v ${{ matrix.ruby-lib-opt }}