fluentd 1.12.1 → 1.12.4

data/test/plugin/test_output.rb

@@ -390,6 +390,18 @@ class OutputTest < Test::Unit::TestCase
       assert { logs.none? { |log| log.include?("${chunk_id}") } }
     end
 
+    test '#extract_placeholders does not log for ${chunk_id} placeholder (with @chunk_keys)' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1')]))
+      tmpl = "/mypath/${chunk_id}/${key1}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+      @i.log.out.logs.clear
+      @i.extract_placeholders(tmpl, c)
+      logs = @i.log.out.logs
+      assert { logs.none? { |log| log.include?("${chunk_id}") } }
+    end
+
     test '#extract_placeholders logs warn message with not replaced key' do
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
       tmpl = "/mypath/${key1}/test"
@@ -868,7 +880,7 @@ class OutputTest < Test::Unit::TestCase
       test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
         o = create_output(:buffered)
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput")
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
         assert_not_nil o.instance_variable_get(:@secondary)
@@ -877,7 +889,7 @@ class OutputTest < Test::Unit::TestCase
       test "don't warn if primary type is the same as secondary type" do
         o = Fluent::Plugin::TestOutput.new
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" ).never
 
         o.configure(config_element('ROOT','',{'name' => "cool2"},
                                    [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
@@ -889,7 +901,7 @@ class OutputTest < Test::Unit::TestCase
       test "don't warn if primary type is different from secondary type and both don't have custom_format" do
         o = create_output(:standard)
         mock(o.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+                         primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput").never
 
         o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
         assert_not_nil o.instance_variable_get(:@secondary)

data/test/plugin/test_output_as_buffered_backup.rb

@@ -173,6 +173,8 @@ class BufferedOutputBackupTest < Test::Unit::TestCase
       waiting(5) {
         target_dir = File.join(File.dirname(target_file), "*")
         while Dir.glob(target_dir).size.zero?
+          # Avoid to lose globbed entries on Windows in busy loop
+          sleep 0.1 if Fluent.windows?
         end
       }
     end

data/test/plugin/test_parser_csv.rb

@@ -183,4 +183,18 @@ class CSVParserTest < ::Test::Unit::TestCase
       # And more...
     end
   end
+
+  # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
+  # https://github.com/fluent/fluentd/issues/3296
+  data('normal' => :normal, 'fast' => :fast)
+  def test_parser_type_method(engine)
+    d = create_driver('keys' => '["time"]','time_key' => 'time', 'parser_type' => engine.to_s)
+    assert_equal(:text_per_line, d.instance.parser_type)
+  end
+
+  data('normal' => :normal, 'fast' => :fast)
+  def test_parser_engine(engine)
+    d = create_driver('keys' => '["time"]', 'time_key' => 'time', 'parser_engine' => engine.to_s)
+    assert_equal(engine, d.instance.parser_engine)
+  end
 end

data/test/plugin/test_parser_syslog.rb

@@ -633,4 +633,18 @@ class SyslogParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  # "parser_type" config shouldn't hide Fluent::Plugin::Parser#plugin_type
+  # https://github.com/fluent/fluentd/issues/3296
+  data('regexp' => :regexp, 'fast' => :string)
+  def test_parser_type_method(engine)
+    @parser.configure({'parser_type' => engine.to_s})
+    assert_equal(:text_per_line, @parser.instance.parser_type)
+  end
+
+  data('regexp' => :regexp, 'string' => :string)
+  def test_parser_engine(engine)
+    d = @parser.configure({'parser_engine' => engine.to_s})
+    assert_equal(engine, @parser.instance.parser_engine)
+  end
 end

data/test/plugin_helper/service_discovery/test_manager.rb

@@ -71,7 +71,7 @@ class TestServiceDiscoveryManager < ::Test::Unit::TestCase
     test 'if new service added and deleted' do
       sdm = Fluent::PluginHelper::ServiceDiscovery::Manager.new(log: $log)
       t = TestSdPlugin.new
-      mock(Fluent::Plugin).new_sd(:sd_test, anything) { t }
+      mock(Fluent::Plugin).new_sd(:sd_test, parent: anything) { t }
       sdm.configure([{ type: :sd_test, conf: config_element('service_discovery', '', {})}])
       sdm.start
 

data/test/plugin_helper/test_child_process.rb

@@ -1,3 +1,4 @@
+# coding: utf-8
 require_relative '../helper'
 require 'fluent/plugin_helper/child_process'
 require 'fluent/plugin/base'
@@ -269,10 +270,12 @@ class ChildProcessTest < Test::Unit::TestCase
     Timeout.timeout(TEST_DEADLOCK_TIMEOUT) do
       ran = false
       @d.child_process_execute(:t4, "ruby -e 'Signal.trap(:TERM, nil); while sleep 0.1; puts 1; STDOUT.flush rescue nil; end'", mode: [:read]) do |io|
-        m.lock
-        ran = true
         begin
           while line = io.readline
+            unless ran
+              m.lock
+              ran = true
+            end
             ary << line
           end
         rescue

data/test/plugin_helper/test_http_server_helper.rb

@@ -354,7 +354,7 @@ class HttpHelperTest < Test::Unit::TestCase
           watcher.should_receive(:stop).once
         end
 
-        stub(Fluent::PluginHelper::HttpServer::Server).new(anything) { server }
+        stub(Fluent::PluginHelper::HttpServer::Server).new(addr: anything, port: anything, logger: anything, default_app: anything) { server }
         driver.http_server_create_http_server(:http_server_helper_test, addr: '127.0.0.1', port: PORT, logger: NULL_LOGGER) do
           # nothing
         end

data/test/plugin_helper/test_server.rb

@@ -234,7 +234,13 @@ class ServerPluginHelperTest < Test::Unit::TestCase
       assert_raise(ArgumentError.new("BUG: backlog is available for tcp/tls")) do
         @d.__send__(m, :myserver, PORT, proto: proto, backlog: 500){|x| x }
       end
-      assert_raise(ArgumentError.new("BUG: send_keepalive_packet is available for tcp")) do
+    end
+
+    data(
+      'server_create udp' => [:server_create, :udp],
+    )
+    test 'raise error if tcp/tls send_keepalive_packet option is specified for udp' do |(m, proto)|
+      assert_raise(ArgumentError.new("BUG: send_keepalive_packet is available for tcp/tls")) do
         @d.__send__(m, :myserver, PORT, proto: proto, send_keepalive_packet: true){|x| x }
       end
     end
@@ -1300,7 +1306,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
     test 'can accept all keyword arguments valid for tcp/tls server' do
       assert_nothing_raised do
-        @d.server_create_tls(:s, PORT, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options) do |data, conn|
+        @d.server_create_tls(:s, PORT, bind: '127.0.0.1', shared: false, resolve_name: true, linger_timeout: 10, backlog: 500, tls_options: @tls_options, send_keepalive_packet: true) do |data, conn|
           # ...
         end
       end

data/test/test_event.rb

@@ -401,6 +401,22 @@ module EventTest
         i += 1
       }
     end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'size_after_any' do
+      @es.any?
+
+      assert_equal 2, @es.size
+    end
+
+    # `any?` represents an Enumerable method which calls `each` internally
+    test 'each_after_any' do
+      @es.any?
+
+      count = 0
+      @es.each { |time, record| count += 1 }
+      assert_equal 2, count
+    end
   end
 
   class CompressedMessagePackEventStreamTest < ::Test::Unit::TestCase

data/test/test_formatter.rb

@@ -184,6 +184,36 @@ module FormatterTest
 
       assert_equal("message=awesome,greeting=hello#{@newline}", formatted)
     end
+
+    def record_with_tab
+      {'message' => "awe\tsome", 'greeting' => "hello\t"}
+    end
+
+    def test_format_suppresses_tab
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:awe some\tgreeting:hello #{@newline}", formatted)
+    end
+
+    def test_format_suppresses_tab_custom_replacement
+      @formatter.configure(
+        'replacement'      => 'X',
+      )
+      formatted = @formatter.format(tag, @time, record_with_tab)
+
+      assert_equal("message:aweXsome\tgreeting:helloX#{@newline}", formatted)
+    end
+
+    def test_format_suppresses_custom_delimiter
+      @formatter.configure(
+        'delimiter'       => 'w',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=a esomewgreeting=hello#{@newline}", formatted)
+    end
   end
 
   class CsvFormatterTest < ::Test::Unit::TestCase

data/test/test_output.rb

@@ -117,7 +117,7 @@ module FluentOutputTest
       end
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" })
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output")
       d.configure(CONFIG + %[
         <secondary>
           type test2
@@ -133,7 +133,7 @@ module FluentOutputTest
       d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::ObjectBufferedOutput)
 
       mock(d.instance.log).warn("Use different plugin for secondary. Check the plugin works with primary like secondary_file",
-                                { primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output" }).never
+                                primary: d.instance.class.to_s, secondary: "Fluent::Plugin::Test2Output").never
       d.configure(CONFIG + %[
         <secondary>
           type test2

data/test/test_time_parser.rb

@@ -228,4 +228,113 @@ class TimeParserTest < ::Test::Unit::TestCase
       assert_equal_event_time(time, parser.parse("#{time.sec}.#{time.nsec}"))
     end
   end
+
+  sub_test_case 'MixedTimeParser fallback' do
+    class DummyForTimeParser
+      include Fluent::Configurable
+      include Fluent::TimeMixin::Parser
+    end
+
+    test 'no time_format_fallbacks failure' do
+      i = DummyForTimeParser.new
+      assert_raise(Fluent::ConfigError.new("time_type is :mixed but time_format and time_format_fallbacks is empty.")) do
+        i.configure(config_element('parse', '', {'time_type' => 'mixed'}))
+      end
+    end
+
+    test 'fallback time format failure' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      assert_raise(Fluent::TimeParser::TimeParseError.new("invalid time format: value = INVALID, even though fallbacks: Fluent::TimeParser")) do
+        parser.parse("INVALID")
+      end
+    end
+
+    test 'primary format is unixtime, secondary %iso8601 is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime',
+                                  'time_format_fallbacks' => ['%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+
+    test 'primary format is %iso8601, secondary unixtime is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601',
+                                  'time_format_fallbacks' => ['unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("2021-01-01T12:00:00+0900"))
+    end
+
+    test 'primary format is unixtime, no secondary is used' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse("#{time.sec}"))
+    end
+
+    test 'primary format is %iso8601, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => '%iso8601'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time.sec}, even though fallbacks: Fluent::TimeParser") do
+        parser.parse("#{time.sec}")
+      end
+    end
+
+    test 'primary format is unixtime, raise error because of no appropriate secondary' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '',
+                                 {'time_type' => 'mixed',
+                                  'time_format' => 'unixtime'}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_raise("Fluent::TimeParser::TimeParseError: invalid time format: value = #{time}, even though fallbacks: Fluent::NumericTimeParser") do
+        parser.parse("2021-01-01T12:00:00+0900")
+      end
+    end
+
+    test 'fallback to unixtime' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['%iso8601', 'unixtime']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(Fluent::EventTime.new(time.to_i), parser.parse("#{time.sec}"))
+    end
+
+    test 'fallback to %iso8601' do
+      i = DummyForTimeParser.new
+      i.configure(config_element('parse', '', {'time_type' => 'mixed',
+                                               'time_format_fallbacks' => ['unixtime', '%iso8601']}))
+      parser = i.time_parser_create
+      time = event_time('2021-01-01T12:00:00+0900')
+      assert_equal_event_time(time, parser.parse('2021-01-01T12:00:00+0900'))
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 1.12.1
+  version: 1.12.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-18 00:00:00.000000000 Z
+date: 2021-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -186,6 +186,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -248,14 +268,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -350,18 +370,20 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".deepsource.toml"
 - ".drone.yml"
 - ".github/ISSUE_TEMPLATE.md"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/PULL_REQUEST_TEMPLATE.md"
-- ".github/workflows/build.yaml"
 - ".github/workflows/issue-auto-closer.yml"
+- ".github/workflows/linux-test.yaml"
+- ".github/workflows/macos-test.yaml"
 - ".github/workflows/stale-actions.yml"
+- ".github/workflows/windows-test.yaml"
 - ".gitignore"
 - ".gitlab-ci.yml"
-- ".travis.yml"
 - ADOPTERS.md
 - AUTHORS
 - CHANGELOG.md
@@ -373,7 +395,6 @@ files:
 - MAINTAINERS.md
 - README.md
 - Rakefile
-- appveyor.yml
 - bin/fluent-binlog-reader
 - bin/fluent-ca-generate
 - bin/fluent-cap-ctl
@@ -739,6 +760,7 @@ files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
+- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb
@@ -910,8 +932,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: Fluentd event collector
@@ -977,6 +998,7 @@ test_files:
 - test/plugin/test_buffer_memory_chunk.rb
 - test/plugin/test_compressable.rb
 - test/plugin/test_file_util.rb
+- test/plugin/test_file_wrapper.rb
 - test/plugin/test_filter.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_parser.rb

data/.travis.yml

@@ -1,77 +0,0 @@
-language: ruby
-cache: bundler
-
-# script: bundle exec rake test TESTOPTS=-v
-
-# http://rubies.travis-ci.org/
-# See here for osx_image -> OSX versions: https://docs.travis-ci.com/user/languages/objective-c
-matrix:
-  include:
-    - rvm: 2.4.9
-      os: linux
-      env: USE_CAPNG=false
-    - rvm: 2.4.9
-      os: linux-ppc64le
-      env: USE_CAPNG=false
-    - rvm: 2.5.7
-      os: linux
-      env: USE_CAPNG=false
-    - rvm: 2.5.7
-      os: linux
-      arch: s390x
-      dist: xenial
-      env: USE_CAPNG=false
-    - rvm: 2.6.5
-      os: linux
-      env: USE_CAPNG=false
-    - rvm: 2.6.6
-      os: linux
-      env: USE_CAPNG=true
-    - rvm: 2.7.0
-      os: linux
-      env: USE_CAPNG=false
-    - rvm: ruby-head
-      os: linux
-      env: USE_CAPNG=false
-    - rvm: ruby-head
-      os: linux-ppc64le
-      env: USE_CAPNG=false
-    - rvm: 2.4.6
-      os: osx
-      osx_image: xcode8.3 # OSX 10.12
-      env: USE_CAPNG=false
-    - rvm: ruby-head
-      os: osx
-      osx_image: xcode8.3 # OSX 10.12
-      env: USE_CAPNG=false
-  allow_failures:
-    - rvm: 2.4.6
-      os: osx
-      osx_image: xcode8.3
-      env: USE_CAPNG=false
-    - rvm: 2.5.7
-      os: linux
-      arch: s390x
-      dist: xenial
-      env: USE_CAPNG=false
-    - rvm: ruby-head
-      env: USE_CAPNG=false
-
-branches:
-  only:
-    - master
-
-before_install: |
-  gem update --system=3.1.2
-  if [[ x"${USE_CAPNG}" == "xtrue" ]]; then
-    echo 'gem "capng_c"' >> Gemfile.local
-  fi
-
-sudo: false
-dist: trusty # for TLSv1.2 support
-
-addons:
-  apt:
-    packages:
-      - libgmp3-dev
-      - libcap-ng-dev

data/appveyor.yml

@@ -1,31 +0,0 @@
-version: '{build}'
-
-# init:
-#   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
-
-install:
-  - SET PATH=C:\Ruby%ruby_version%\bin;%PATH%
-  - ruby --version
-  - gem --version
-  # stay 0.14.0 for Windows CI until https://github.com/socketry/protocol-http2/issues/6 will be fixed
-  - ps: Write-Output "gem 'protocol-http2', ['<= 0.14.0']" | Out-File -FilePath Gemfile.local -Encoding default
-  - type Gemfile.local
-  - ridk.cmd exec bundle install
-build: off
-test_script:
-  - bundle exec rake test
-#  - bundle exec rake test TESTOPTS=-v
-
-branches:
-  only:
-    - master
-
-# https://www.appveyor.com/docs/installed-software/#ruby
-environment:
-  matrix:
-    - ruby_version: "26-x64"
-    - ruby_version: "26"
-    - ruby_version: "25-x64"
-    - ruby_version: "25"
-    - ruby_version: "24-x64"
-    - ruby_version: "24"