fluentd 0.14.23.rc1 → 0.14.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e963518d8ad90373d793c9af9275cc45720d45e
-  data.tar.gz: 43875ee26f0702c7c23514d5b38ef8b163e0e6d8
+  metadata.gz: 295a3c8dbe521b90ed22fcbc07ed1934d77325ad
+  data.tar.gz: 57bcb6c43a76065fc9b3fd681a83f3a4aa2c0eae
 SHA512:
-  metadata.gz: ec41c99be159e167051b78446a57e89d7627b7f940d10f4c5bc012224c5e7756fd4cbeccd4e4e59fc2d96204e57851f85c5e089896a4fd0acbe914bfd82e3158
-  data.tar.gz: 8b2f5f28e11ed7342f816f1e35e5fa9c4de984aad1cbf24c36c273ddc90f4982b06386bdebd31a8259b2dd924d965b84bc194e28cbc6bb6436a71d62109c0cb1
+  metadata.gz: ff0029c699dbe57532351805e5d2cd4bc0d65d1635219b18eeddb019b1ff8845b71ac1029367024765b48f2971fa4f999fd7fa361329ecc4394b806651b539f4
+  data.tar.gz: d7cb461f8f6cf8d65e088e1ea2db7aed60387280ac7b95c15b0220ee5e527f947cc0cecd9541162a81960f9dbce2393f26eddf4239745270763a59ad7303dbae

data/.travis.yml

@@ -13,7 +13,7 @@ matrix:
       os: linux
     - rvm: 2.3.3
       os: linux
-    - rvm: 2.4.1
+    - rvm: 2.4.2
       os: linux
     - rvm: ruby-head
       os: linux
@@ -26,7 +26,7 @@ matrix:
     # - rvm: 2.3.3
     #   os: osx
     #   osx_image: xcode8.2 # OSX 10.12
-    - rvm: 2.4.0
+    - rvm: 2.4.1
       os: osx
       osx_image: xcode8.3 # OSX 10.12
     - rvm: ruby-head

data/CHANGELOG.md

@@ -1,5 +1,29 @@
 # v0.14
 
+## Release v0.14.23 - 2017/11/15
+
+### New features / Enhancements
+
+* in_udp: Add remove_newline parameter
+  https://github.com/fluent/fluentd/pull/1747
+
+### Bug fixes
+
+* buffer: Lock buffers in order of metadata
+  https://github.com/fluent/fluentd/pull/1722
+* in_tcp: Fix log corruption under load.
+  https://github.com/fluent/fluentd/pull/1729
+* out_forward: Fix elapsed time miscalculation in tcp heartbeat
+  https://github.com/fluent/fluentd/pull/1738
+* supervisor: Fix worker pid handling during worker restart
+  https://github.com/fluent/fluentd/pull/1739
+* in_tail: Skip setup failed watcher to avoid resource leak and log bloat
+  https://github.com/fluent/fluentd/pull/1742
+* agent: Add error location to emit error logs
+  https://github.com/fluent/fluentd/pull/1746
+* command: Consider hyphen and underscore in fluent-plugin-generate arguments
+  https://github.com/fluent/fluentd/pull/1751
+
 ## Release v0.14.22 - 2017/11/01
 
 ### New features / Enhancements

data/GOVERNANCE.md

@@ -0,0 +1,55 @@
+# Fluentd Governance
+
+## Principles
+
+The Fluentd community adheres to the following principles:
+
+- Open: Fluentd is open source. See repository guidelines and CLA, below.
+- Welcoming and respectful: See Code of Conduct, below.
+- Transparent and accessible: Work and collaboration are done in public.
+- Merit: Ideas and contributions are accepted according to their technical merit and alignment with project objectives, scope, and design principles.
+
+## Voting
+
+The Fluentd project employs "organization voting" to ensure no single organization can dominate the project.
+
+Individuals not associated with or employed by a company or organization are allowed one organization vote. Each company or organization (regardless of the number of maintainers associated with or employed by that company/organization) receives one organization vote.
+
+In other words, if two maintainers are employed by Company X, two by Company Y, two by Company Z, and one maintainer is an un-affiliated individual, a total of four "organization votes" are possible; one for X, one for Y, one for Z, and one for the un-affiliated individual.
+
+Any maintainer from an organization may cast the vote for that organization.
+
+For formal votes, a specific statement of what is being voted on should be added to the relevant github issue or PR, and a link to that issue or PR added to the maintainers meeting agenda document. Maintainers should indicate their yes/no vote on that issue or PR, and after a suitable period of time, the votes will be tallied and the outcome noted.
+
+## Changes in Maintainership
+
+New maintainers are proposed by an existing maintainer and are elected by a 2/3 majority organization vote.
+
+Maintainers can be removed by a 2/3 majority organization vote.
+
+## Github Project Administration
+
+Maintainers will be added to the __fluent__ GitHub organization and added to the GitHub cni-maintainers team, and made a GitHub maintainer of that team.
+
+After 6 months a maintainer will be made an "owner" of the GitHub organization.
+
+## Projects
+
+The fluent organization is open to receive new projects under it umbrella. To apply a project as part of the __fluent__ organization, it have to met the following criteria:
+
+- Licensed under the terms of the Apache License v2.0
+- Project have been active for at least one year since it inception
+- More than 2 contributors
+- Related to one or more scopes of Fluentd ecosystem:
+  - Data collection
+  - Log management
+  - Metering
+- Be supported by 2/3 majority of organization
+
+The submission process starts as a Pull Request on Fluentd repository with the required information mentioned above.
+
+## Code of Conduct
+
+Fluentd follows the CNCF Code of Conduct:
+
+https://github.com/cncf/foundation/blob/master/code-of-conduct.md

data/lib/fluent/command/plugin_generator.rb

@@ -137,32 +137,36 @@ BANNER
   end
 
   def gem_name
-    "fluent-plugin-#{name}"
+    "fluent-plugin-#{dash_name}"
+  end
+
+  def plugin_name
+    underscore_name
   end
 
   def class_name
-    "#{name.capitalize}#{type.capitalize}"
+    "#{capitalized_name}#{type.capitalize}"
   end
 
   def plugin_filename
     case type
     when "input"
-      "in_#{name}.rb"
+      "in_#{underscore_name}.rb"
     when "output"
-      "out_#{name}.rb"
+      "out_#{underscore_name}.rb"
     else
-      "#{type}_#{name}.rb"
+      "#{type}_#{underscore_name}.rb"
     end
   end
 
   def test_filename
     case type
     when "input"
-      "test_in_#{name}.rb"
+      "test_in_#{underscore_name}.rb"
     when "output"
-      "test_out_#{name}.rb"
+      "test_out_#{underscore_name}.rb"
     else
-      "test_#{type}_#{name}.rb"
+      "test_#{type}_#{underscore_name}.rb"
     end
   end
 
@@ -179,6 +183,18 @@ BANNER
     end
   end
 
+  def capitalized_name
+    @capitalized_name ||= name.split(/[-_]/).map(&:capitalize).join
+  end
+
+  def underscore_name
+    @underscore_name ||= name.tr("-", "_")
+  end
+
+  def dash_name
+    @dash_name ||= name.tr("_", "-")
+  end
+
   def preamble
     @license.preamble(user_name)
   end

data/lib/fluent/plugin/in_tail.rb

@@ -34,6 +34,16 @@ module Fluent::Plugin
 
     helpers :timer, :event_loop, :parser, :compat_parameters
 
+    class WatcherSetupError < StandardError
+      def initialize(msg)
+        @message = msg
+      end
+
+      def to_s
+        @message
+      end
+    end
+
     FILE_PERMISSION = 0644
 
     def initialize
@@ -250,6 +260,12 @@ module Fluent::Plugin
         event_loop_attach(watcher.stat_trigger)
       end
       tw
+    rescue => e
+      if tw
+        tw.detach
+        tw.close
+      end
+      raise e
     end
 
     def start_watchers(paths)
@@ -266,7 +282,13 @@ module Fluent::Plugin
           end
         end
 
-        @tails[path] = setup_watcher(path, pe)
+        begin
+          tw = setup_watcher(path, pe)
+        rescue WatcherSetupError => e
+          log.warn "Skip #{path} because unexpected setup error happens: #{e}"
+          next
+        end
+        @tails[path] = tw
       }
     end
 
@@ -480,8 +502,8 @@ module Fluent::Plugin
       end
 
       def detach
-        @timer_trigger.detach if @enable_watch_timer && @timer_trigger.attached?
-        @stat_trigger.detach if @stat_trigger.attached?
+        @timer_trigger.detach if @enable_watch_timer && @timer_trigger && @timer_trigger.attached?
+        @stat_trigger.detach if @stat_trigger && @stat_trigger.attached?
         @io_handler.on_notify if @io_handler
       end
 
@@ -698,6 +720,9 @@ module Fluent::Plugin
           io = Fluent::FileWrapper.open(@watcher.path)
           io.seek(@watcher.pe.read_pos + @fifo.bytesize)
           io
+        rescue RangeError
+          io.close if io
+          raise WatcherSetupError, "seek error with #{@watcher.path}: file position = #{@watcher.pe.read_pos.to_s(16)}, reading bytesize = #{@fifo.bytesize.to_s(16)}"
         rescue Errno::ENOENT
           nil
         end
@@ -715,6 +740,9 @@ module Fluent::Plugin
               @io ||= open
               yield @io
             end
+          rescue WatcherSetupError => e
+            close
+            raise e
           rescue
             @watcher.log.error $!.to_s
             @watcher.log.error_backtrace

data/lib/fluent/plugin/in_udp.rb

@@ -38,6 +38,8 @@ module Fluent::Plugin
     config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
     desc "The max bytes of message"
     config_param :message_length_limit, :size, default: 4096
+    desc "Remove newline from the end of incoming payload"
+    config_param :remove_newline, :bool, default: true
 
     config_param :blocking_timeout, :time, default: 0.5
 
@@ -60,7 +62,7 @@ module Fluent::Plugin
 
       log.info "listening udp socket", bind: @bind, port: @port
       server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
-        data.chomp!
+        data.chomp! if @remove_newline
         begin
           @parser.parse(data) do |time, record|
             unless time && record

data/lib/fluent/plugin/out_forward.rb

@@ -395,9 +395,6 @@ module Fluent::Plugin
 
     def on_timer
       @nodes.each {|n|
-        if n.tick
-          rebuild_weight_array
-        end
         begin
           log.trace "sending heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type
           n.usock = @usock if @usock
@@ -407,6 +404,9 @@ module Fluent::Plugin
         rescue => e
           log.debug "unexpected error happen during heartbeat", host: n.host, port: n.port, heartbeat_type: @heartbeat_type, error: e
         end
+        if n.tick
+          rebuild_weight_array
+        end
       }
     end
 

data/lib/fluent/root_agent.rb

@@ -292,7 +292,7 @@ module Fluent
     end
 
     def emit_error_event(tag, time, record, error)
-      error_info = {error: error, tag: tag, time: time}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time}
       if @error_collector
         # A record is not included in the logs because <@ERROR> handles it. This warn is for the notification
         log.warn "send an error event to @ERROR:", error_info
@@ -304,7 +304,7 @@ module Fluent
     end
 
     def handle_emits_error(tag, es, error)
-      error_info = {error: error, tag: tag}
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
       if @error_collector
         log.warn "send an error event stream to @ERROR:", error_info
         @error_collector.emit_stream(tag, es)
@@ -333,7 +333,7 @@ module Fluent
       end
 
       def emit_error_event(tag, time, record, error)
-        error_info = {error: error, tag: tag, time: time, record: record}
+        error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
         log.warn "dump an error event in @ERROR:", error_info
       end
 

data/lib/fluent/supervisor.rb

@@ -164,7 +164,7 @@ module Fluent
       end
 
       if config[:worker_pid]
-        config[:worker_pid].each do |pid|
+        config[:worker_pid].each_value do |pid|
           Process.kill(:USR1, pid)
           # don't rescue Erro::ESRSH here (invalid status)
         end
@@ -175,7 +175,7 @@ module Fluent
       if config[:worker_pid]
         pids = config[:worker_pid].clone
         config[:worker_pid].clear
-        pids.each do |pid|
+        pids.each_value do |pid|
           if Fluent.windows?
             Process.kill :KILL, pid
           else
@@ -204,7 +204,7 @@ module Fluent
     end
 
     def after_start
-      (config[:worker_pid] ||= []).push(@pm.pid)
+      (config[:worker_pid] ||= {})[@worker_id] = @pm.pid
     end
   end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.23.rc1'
+  VERSION = '0.14.23'
 
 end

data/templates/new_gem/lib/fluent/plugin/filter.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/filter"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Filter
-      Fluent::Plugin.register_filter("<%= name %>", self)
+      Fluent::Plugin.register_filter("<%= plugin_name %>", self)
 
       def filter(tag, time, record)
       end

data/templates/new_gem/lib/fluent/plugin/input.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/input"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Input
-      Fluent::Plugin.register_input("<%= name %>", self)
+      Fluent::Plugin.register_input("<%= plugin_name %>", self)
     end
   end
 end

data/templates/new_gem/lib/fluent/plugin/output.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/output"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Output
-      Fluent::Plugin.register_output("<%= name %>", self)
+      Fluent::Plugin.register_output("<%= plugin_name %>", self)
     end
   end
 end

data/templates/new_gem/lib/fluent/plugin/parser.rb.erb

@@ -5,7 +5,7 @@ require "fluent/plugin/parser"
 module Fluent
   module Plugin
     class <%= class_name %> < Fluent::Plugin::Parser
-      Fluent::Plugin.register_parser("<%= name %>", self)
+      Fluent::Plugin.register_parser("<%= plugin_name %>", self)
     end
 
     def parse(text)

data/test/command/test_plugin_generator.rb

@@ -16,51 +16,77 @@ class TestFluentPluginGenerator < Test::Unit::TestCase
     FileUtils.rm_rf(TEMP_DIR)
   end
 
-  data(input: ["input", "in"],
-       output: ["output", "out"],
-       filter: ["filter", "filter"],
-       parser: ["parser", "parser"],
-       formatter: ["formatter", "formatter"])
-  test "generate plugin" do |(type, part)|
-    capture_stdout do
-      FluentPluginGenerator.new([type, "fake"]).call
+  sub_test_case "generate plugin" do
+    data(input: ["input", "in"],
+         output: ["output", "out"],
+         filter: ["filter", "filter"],
+         parser: ["parser", "parser"],
+         formatter: ["formatter", "formatter"])
+    test "generate plugin" do |(type, part)|
+      capture_stdout do
+        FluentPluginGenerator.new([type, "fake"]).call
+      end
+      plugin_base_dir = Pathname("fluent-plugin-fake")
+      assert { plugin_base_dir.directory? }
+      expected = [
+        "fluent-plugin-fake",
+        "fluent-plugin-fake/Gemfile",
+        "fluent-plugin-fake/LICENSE",
+        "fluent-plugin-fake/README.md",
+        "fluent-plugin-fake/Rakefile",
+        "fluent-plugin-fake/fluent-plugin-fake.gemspec",
+        "fluent-plugin-fake/lib",
+        "fluent-plugin-fake/lib/fluent",
+        "fluent-plugin-fake/lib/fluent/plugin",
+        "fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb",
+        "fluent-plugin-fake/test",
+        "fluent-plugin-fake/test/helper.rb",
+        "fluent-plugin-fake/test/plugin",
+        "fluent-plugin-fake/test/plugin/test_#{part}_fake.rb",
+      ]
+      actual = plugin_base_dir.find.reject {|f| f.fnmatch("*/.git*") }.map(&:to_s).sort
+      assert_equal(expected, actual)
     end
-    plugin_base_dir = Pathname("fluent-plugin-fake")
-    assert { plugin_base_dir.directory? }
-    expected = [
-      "fluent-plugin-fake",
-      "fluent-plugin-fake/Gemfile",
-      "fluent-plugin-fake/LICENSE",
-      "fluent-plugin-fake/README.md",
-      "fluent-plugin-fake/Rakefile",
-      "fluent-plugin-fake/fluent-plugin-fake.gemspec",
-      "fluent-plugin-fake/lib",
-      "fluent-plugin-fake/lib/fluent",
-      "fluent-plugin-fake/lib/fluent/plugin",
-      "fluent-plugin-fake/lib/fluent/plugin/#{part}_fake.rb",
-      "fluent-plugin-fake/test",
-      "fluent-plugin-fake/test/helper.rb",
-      "fluent-plugin-fake/test/plugin",
-      "fluent-plugin-fake/test/plugin/test_#{part}_fake.rb",
-    ]
-    actual = plugin_base_dir.find.reject {|f| f.fnmatch("*/.git*") }.map(&:to_s).sort
-    assert_equal(expected, actual)
-  end
 
-  test "no license" do
-    capture_stdout do
-      FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call
+    test "no license" do
+      capture_stdout do
+        FluentPluginGenerator.new(["--no-license", "filter", "fake"]).call
+      end
+      assert { !Pathname("fluent-plugin-fake/LICENSE").exist? }
+      assert { Pathname("fluent-plugin-fake/Gemfile").exist? }
+    end
+
+    test "unknown license" do
+      out = capture_stdout do
+        assert_raise(SystemExit) do
+          FluentPluginGenerator.new(["--license=unknown", "filter", "fake"]).call
+        end
+      end
+      assert { out.lines.include?("License: unknown\n") }
     end
-    assert { !Pathname("fluent-plugin-fake/LICENSE").exist? }
-    assert { Pathname("fluent-plugin-fake/Gemfile").exist? }
   end
 
-  test "unknown license" do
-    out = capture_stdout do
-      assert_raise(SystemExit) do
-        FluentPluginGenerator.new(["--license=unknown", "filter", "fake"]).call
+  sub_test_case "unify plugin name" do
+    data("word" => ["fake", "fake"],
+         "underscore" => ["rewrite_tag_filter", "rewrite_tag_filter"],
+         "dash" => ["rewrite-tag-filter", "rewrite_tag_filter"])
+    test "plugin_name" do |(name, plugin_name)|
+      generator = FluentPluginGenerator.new(["filter", name])
+      capture_stdout do
+        generator.call
+      end
+      assert_equal(plugin_name, generator.__send__(:plugin_name))
+    end
+
+    data("word" => ["fake", "fluent-plugin-fake"],
+         "underscore" => ["rewrite_tag_filter", "fluent-plugin-rewrite-tag-filter"],
+         "dash" => ["rewrite-tag-filter", "fluent-plugin-rewrite-tag-filter"])
+    test "gem_name" do |(name, gem_name)|
+      generator = FluentPluginGenerator.new(["output", name])
+      capture_stdout do
+        generator.call
       end
+      assert_equal(gem_name, generator.__send__(:gem_name))
     end
-    assert { out.lines.include?("License: unknown\n") }
   end
 end

data/test/plugin/test_in_udp.rb

@@ -149,4 +149,27 @@ class UdpInputTest < Test::Unit::TestCase
       assert_equal expected_record, d.events[i][2]
     end
   end
+
+  test 'remove_newline' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      remove_newline false
+    !)
+    payloads = ["test1\n", "test2\n"]
+    d.run(expect_records: 2) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        payloads.each do |payload|
+          u.send(payload, 0)
+        end
+      end
+    end
+
+    expecteds = payloads.map { |payload| {'message' => payload} }
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "udp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.14.23.rc1
+  version: 0.14.23
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2017-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -324,6 +324,7 @@ files:
 - CONTRIBUTING.md
 - COPYING
 - ENTERPRISE_PROVIDERS.md
+- GOVERNANCE.md
 - Gemfile
 - MAINTAINERS.md
 - README.md
@@ -736,9 +737,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13