fluentd 0.14.12 → 0.14.13

data/lib/fluent/plugin/parser_nginx.rb

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
 
 module Fluent
   module Plugin

data/lib/fluent/plugin/parser_regexp.rb

@@ -1,3 +1,21 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
 module Fluent
   module Plugin
     class RegexpParser < Parser

data/lib/fluent/plugin_helper.rb

@@ -37,7 +37,15 @@ module Fluent
       end
     end
 
-    def helpers(*snake_case_symbols)
+    def self.extended(mod)
+      def mod.inherited(subclass)
+        subclass.module_eval do
+          @_plugin_helpers_list = []
+        end
+      end
+    end
+
+    def helpers_internal(*snake_case_symbols)
       helper_modules = []
       snake_case_symbols.each do |name|
         begin
@@ -48,5 +56,15 @@ module Fluent
       end
       include(*helper_modules)
     end
+
+    def helpers(*snake_case_symbols)
+      @_plugin_helpers_list ||= []
+      @_plugin_helpers_list.concat(snake_case_symbols)
+      helpers_internal(*snake_case_symbols)
+    end
+
+    def plugin_helpers
+      @_plugin_helpers_list || []
+    end
   end
 end

data/lib/fluent/plugin_helper/retry_state.rb

@@ -64,7 +64,12 @@ module Fluent
           @secondary_threshold = secondary_threshold
           if @secondary
             raise "BUG: secondary_transition_threshold MUST be between 0 and 1" if @secondary_threshold <= 0 || @secondary_threshold >= 1
-            @secondary_transition_at = @start + timeout * @secondary_threshold
+            max_retry_timeout = timeout
+            if max_steps
+              timeout_by_max_steps = calc_max_retry_timeout(max_steps)
+              max_retry_timeout = timeout_by_max_steps if timeout_by_max_steps < max_retry_timeout
+            end
+            @secondary_transition_at = @start + max_retry_timeout * @secondary_threshold
             @secondary_transition_steps = nil
           end
         end
@@ -136,41 +141,60 @@ module Fluent
       end
 
       class ExponentialBackOffRetry < RetryStateMachine
-        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threathold)
-          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, backoff_base, max_interval, secondary, secondary_threshold)
           @constant_factor = wait
           @backoff_base = backoff_base
           @max_interval = max_interval
 
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
           @next_time = @start + @constant_factor
         end
 
         def naive_next_time(retry_next_times)
-          # make it infinite if calculated "interval" is too big
-          interval = @constant_factor.to_f * ( @backoff_base ** ( retry_next_times - 1 ) )
-          intr = if interval.finite?
-                   if @max_interval && interval > @max_interval
-                     @max_interval
-                   else
-                     interval
-                   end
-                 else
-                   interval
-                 end
+          intr = calc_interval(retry_next_times)
           current_time + randomize(intr)
         end
+
+        def calc_max_retry_timeout(max_steps)
+          result = 0
+          max_steps.times { |i|
+            result += calc_interval(i)
+          }
+          result
+        end
+
+        def calc_interval(num)
+          # make it infinite if calculated "interval" is too big
+          interval = @constant_factor.to_f * (@backoff_base ** (num - 1))
+          if interval.finite?
+            if @max_interval && interval > @max_interval
+              @max_interval
+            else
+              interval
+            end
+          else
+            interval
+          end
+        end
       end
 
       class PeriodicRetry < RetryStateMachine
-        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
-          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threathold)
+        def initialize(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
           @retry_wait = wait
+
+          super(title, wait, timeout, forever, max_steps, randomize, randomize_width, secondary, secondary_threshold)
+
           @next_time = @start + @retry_wait
         end
 
         def naive_next_time(retry_next_times)
           current_time + randomize(@retry_wait)
         end
+
+        def calc_max_retry_timeout(max_steps)
+          @retry_wait * max_steps
+        end
       end
     end
   end

data/lib/fluent/process.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/detach_process_mixin'
+
+module Fluent
+  DetachProcessMixin = Fluent::Compat::DetachProcessMixin
+  DetachMultiProcessMixin = Fluent::Compat::DetachMultiProcessMixin
+end

data/lib/fluent/supervisor.rb

@@ -517,28 +517,9 @@ module Fluent
     end
 
     def show_plugin_config
-      $log.info "Show config for #{@show_plugin_config}"
-      @system_config = SystemConfig.new
-      init_engine
-      name, type = @show_plugin_config.split(":")
-      plugin = Plugin.__send__("new_#{name}", type)
-      dumped_config = "\n"
-      level = 0
-      plugin.class.ancestors.reverse_each do |plugin_class|
-        if plugin_class.respond_to?(:dump)
-          $log.on_debug do
-            dumped_config << plugin_class.name
-            dumped_config << "\n"
-            level = 1
-          end
-          dumped_config << plugin_class.dump(level)
-        end
-      end
-      $log.info dumped_config
+      name, type = @show_plugin_config.split(":") # input:tail
+      $log.info "Use fluent-plugin-config-format --format=txt #{name} #{type}"
       exit 0
-    rescue => e
-      $log.error "show config failed: #{e}"
-      exit 1
     end
 
     def supervise

data/lib/fluent/test/helpers.rb

@@ -107,6 +107,11 @@ EOT
         end
       end
 
+      #
+      # Use this method with v0.12 compatibility layer.
+      #
+      # For v0.14 API, use `driver.logs` instead.
+      #
       def capture_log(driver)
         tmp = driver.instance.log.out
         driver.instance.log.out = StringIO.new
@@ -115,6 +120,15 @@ EOT
       ensure
         driver.instance.log.out = tmp
       end
+
+      def capture_stdout
+        out = StringIO.new
+        $stdout = out
+        yield
+        out.string.force_encoding('utf-8')
+      ensure
+        $stdout = STDOUT
+      end
     end
   end
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.12'
+  VERSION = '0.14.13'
 
 end

data/templates/new_gem/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/templates/new_gem/README.md.erb

@@ -0,0 +1,43 @@
+# <%= gem_name %>
+
+[Fluentd](http://fluentd.org/) <%= type %> plugin to do something.
+
+TODO: write description for you plugin.
+
+## Installation
+
+### RubyGems
+
+```
+$ gem install <%= gem_name %>
+```
+
+### Bundler
+
+Add following line to your Gemfile:
+
+```ruby
+gem "<%= gem_name %>"
+```
+
+And then execute:
+
+```
+$ bundle
+```
+
+## Configuration
+
+You can generate configuration template:
+
+```
+$ fluent-plugin-format-config <%= type %> <%= name %>
+```
+
+You can copy and paste generated documents here.
+
+## Copyright
+
+* Copyright(c) <%= Date.today.year %>- <%= user_name %>
+* License
+  * <%= @license.full_name %>

data/templates/new_gem/Rakefile

@@ -0,0 +1,13 @@
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/templates/new_gem/fluent-plugin.gemspec.erb

@@ -0,0 +1,27 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "<%= gem_name %>"
+  spec.version = "0.1.0"
+  spec.authors = ["<%= user_name %>"]
+  spec.email   = ["<%= user_email %>"]
+
+  spec.summary       = %q{TODO: Write a short summary, because Rubygems requires one.}
+  spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "TODO: Put your gem's website or public repo URL here."
+  spec.license       = "<%= @license.name %>"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/templates/new_gem/lib/fluent/plugin/filter.rb.erb

@@ -0,0 +1,14 @@
+<%= preamble %>
+
+require "fluent/plugin/filter"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter("<%= name %>", self)
+
+      def filter(tag, time, record)
+      end
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/formatter.rb.erb

@@ -0,0 +1,14 @@
+<%= preamble %>
+
+require "fluent/plugin/formatter"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Formatter
+      Fluent::Plugin.register_formatter("<%= name %>", self)
+
+      def format(tag, time, record)
+      end
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/input.rb.erb

@@ -0,0 +1,11 @@
+<%= preamble %>
+
+require "fluent/plugin/input"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("<%= name %>", self)
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/output.rb.erb

@@ -0,0 +1,11 @@
+<%= preamble %>
+
+require "fluent/plugin/output"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Output
+      Fluent::Plugin.register_output("<%= name %>", self)
+    end
+  end
+end

data/templates/new_gem/lib/fluent/plugin/parser.rb.erb

@@ -0,0 +1,15 @@
+<%= preamble %>
+
+require "fluent/plugin/parser"
+
+module Fluent
+  module Plugin
+    class <%= class_name %> < Fluent::Plugin::Parser
+      Fluent::Plugin.register_parser("<%= name %>", self)
+    end
+
+    def parse(text)
+      yield time, record
+    end
+  end
+end

data/templates/new_gem/test/helper.rb.erb

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/<%= type %>"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/templates/new_gem/test/plugin/test_filter.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end

data/templates/new_gem/test/plugin/test_formatter.rb.erb

@@ -0,0 +1,18 @@
+require "helper"
+require "fluent/plugin/<%= plugin_filename %>"
+
+class <%= class_name %>Test < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::<%= class_name %>).configure(conf)
+  end
+end