fluentd 0.14.12 → 0.14.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c60b140cfb115311fce72ec8be60de12fac24d5
-  data.tar.gz: d7d3eb5f2f0edcdbf3af33dc738f504bb7e9c05b
+  metadata.gz: 158d62963384d98d44438e5f3a8e31c97f3124e4
+  data.tar.gz: 59a98cfbf508e67e462849f8ada07a509255f8bf
 SHA512:
-  metadata.gz: 4fc6d3c53b3890e58f8193f2e92098ddd2a27e91720f28df6cca793011b792bc91c078b295afa2f73ad36572d7aa6255a2b811002b7070686f32f2837e8445a6
-  data.tar.gz: 373cef8be79781fc5189bf3497e794ea46d4d8218628816539935cb32f4ea44ffa4a3c70cda9533254dff5e0171900cb7d9ebb444e6bdb3fb2d0a8ba42d6a729
+  metadata.gz: bf8ab978620b5472988d8483828a9b6e5004e5028b86165b31628810c93d15f591b07fd24e0f6f9929b9390bd8de7ccb1035eeb6d389c8b862638a1325f8fc6a
+  data.tar.gz: 52257e481b34bddce80b2c7f7442328f704f0b11ee1c3495fa895dea4902040ff3eec6ad65c62a261dcb659c5f48bd3cd0455abe045c860259dad420d1b2f9e5

data/CONTRIBUTING.md

@@ -16,6 +16,8 @@ Don't use Github issue for asking questions. Here are examples:
 We may close such questions to keep clear repository for developers and users.
 Github issue is mainly for submitting a bug report or feature request. See below.
 
+If you can't judge your case is a bug or not, use mailing list or slack first.
+
 ## Found a bug?
 
 If you find a bug of Fluentd or a mistake in the documentation, you can help us by

data/ChangeLog

@@ -1,5 +1,38 @@
 # v0.14
 
+## Release v0.14.13 - 2017/02/17
+
+### New features / Enhancements
+
+* in_tail: Add 'limit_recently_modified' to limit watch files.
+  https://github.com/fluent/fluentd/pull/1474
+* configuration: Improve 'flush_interval' handling for better message and backward compatibility
+  https://github.com/fluent/fluentd/pull/1442
+* command: Add 'fluent-plugin-generate' command
+  https://github.com/fluent/fluentd/pull/1427
+* output: Skip record when 'Output#format' returns nil
+  https://github.com/fluent/fluentd/pull/1469
+
+### Bug fixes
+
+* output: Secondary calculation should consider 'retry_max_times'
+  https://github.com/fluent/fluentd/pull/1452
+* Fix regression of deprecatd 'process' module
+  https://github.com/fluent/fluentd/pull/1443
+* Fix missing parser_regex require
+  https://github.com/fluent/fluentd/issues/1458
+  https://github.com/fluent/fluentd/pull/1453
+* Keep 'Fluent::BufferQueueLimitError' for exsting plugins
+  https://github.com/fluent/fluentd/pull/1456
+* in_tail: Untracked files should be removed from watching list to avoid memory bloat
+  https://github.com/fluent/fluentd/pull/1467
+* in_tail: directories should be skipped when the ** pattern is used
+  https://github.com/fluent/fluentd/pull/1464
+* record_transformer: Revert "Use BasicObject for cleanroom" for `enable_ruby` regression.
+  https://github.com/fluent/fluentd/pull/1461
+* buf_file: handle "Too many open files" error to keep buffer and metadata pair
+  https://github.com/fluent/fluentd/pull/1468
+
 ## Release v0.14.12 - 2017/01/30
 
 ### New features / Enhancements

data/bin/fluent-plugin-config-format

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+require 'fluent/command/plugin_config_formatter'
+
+FluentPluginConfigFormatter.new.call

data/bin/fluent-plugin-generate

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH << File.expand_path(File.join(__dir__, '..', 'lib'))
+require 'fluent/command/plugin_generator'
+
+FluentPluginGenerator.new.call

data/lib/fluent/command/plugin_config_formatter.rb

@@ -0,0 +1,258 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+
+class FluentPluginConfigFormatter
+
+  AVAILABLE_FORMATS = [:markdown, :txt, :json]
+  SUPPORTED_TYPES = [
+    "input", "output", "filter",
+    "buffer", "parser", "formatter", "storage"
+  ]
+
+  def initialize(argv = ARGV)
+    @argv = argv
+
+    @compact = false
+    @format = :markdown
+    @verbose = false
+    @libs = []
+    @plugin_dirs = []
+    @options = {}
+
+    prepare_option_parser
+  end
+
+  def call
+    parse_options!
+    init_libraries
+    @plugin = Fluent::Plugin.__send__("new_#{@plugin_type}", @plugin_name)
+    dumped_config = {}
+    if @plugin.class.respond_to?(:plugin_helpers)
+      dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers
+    end
+    @plugin.class.ancestors.reverse_each do |plugin_class|
+      next unless plugin_class.respond_to?(:dump_config_definition)
+      unless @verbose
+        next if plugin_class.name =~ /::PluginHelper::/
+      end
+      dumped_config_definition = plugin_class.dump_config_definition
+      dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
+    end
+    case @format
+    when :txt
+      puts dump_txt(dumped_config)
+    when :markdown
+      puts dump_markdown(dumped_config)
+    when :json
+      puts dump_json(dumped_config)
+    end
+  end
+
+  private
+
+  def dump_txt(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped << "helpers: #{plugin_helpers.join(',')}\n"
+    end
+    if @verbose
+      dumped_config.each do |name, config|
+        dumped << "#{name}\n"
+        dumped << dump_section_txt(config)
+      end
+    else
+      configs = dumped_config.values
+      root_section = configs.shift
+      configs.each do |config|
+        root_section.update(config)
+      end
+      dumped << dump_section_txt(root_section)
+    end
+    dumped
+  end
+
+  def dump_section_txt(base_section, level = 0)
+    dumped = ""
+    indent = " " * level
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      dumped << "#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})"
+      dumped << " # #{config[:description]}" if config.key?(:description)
+      dumped << "\n"
+    end
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      required_label = required ? "required" : "optional"
+      multi_label = multi ? "multiple" : "single"
+      alias_label = "alias: #{alias_name}"
+      dumped << "#{indent}<#{section_name}>: #{required_label}, #{multi_label}"
+      if alias_name
+        dumped << ", #{alias_label}\n"
+      else
+        dumped << "\n"
+      end
+      sub_section.delete(:section)
+      dumped << dump_section_txt(sub_section, level + 1)
+    end
+    dumped
+  end
+
+  def dump_markdown(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped = "## Plugin helpers\n\n"
+      plugin_helpers.each do |plugin_helper|
+        dumped << "* #{plugin_helper}\n"
+      end
+      dumped << "\n"
+    end
+    dumped_config.each do |name, config|
+      if name == @plugin.class.name
+        dumped << "## #{name}\n\n"
+        dumped << dump_section_markdown(config)
+      else
+        dumped << "* See also: #{name}\n\n"
+      end
+    end
+    dumped
+  end
+
+  def dump_section_markdown(base_section, level = 0)
+    dumped = ""
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      template_name = @compact ? "param.md-compact.erb" : "param.md.erb"
+      dumped << ERB.new(template_path(template_name).read, nil, "-").result(binding)
+    end
+    dumped << "\n"
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      $log.trace(name: section_name, required: required, multi: multi, alias_name: alias_name)
+      sub_section.delete(:section)
+      dumped << ERB.new(template_path("section.md.erb").read, nil, "-").result(binding)
+    end
+    dumped
+  end
+
+  def dump_json(dumped_config)
+    if @compact
+      JSON.generate(dumped_config)
+    else
+      JSON.pretty_generate(dumped_config)
+    end
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.banner = <<BANNER
+Usage: #{$0} [options] <type> <name>
+
+Output plugin config definitions
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: registered plugin name
+
+Options:
+BANNER
+    @parser.on("--verbose", "Be verbose") do
+      @verbose = true
+    end
+    @parser.on("-c", "--compact", "Compact output") do
+      @compact = true
+    end
+    @parser.on("-f", "--format=FORMAT", "Specify format. (#{AVAILABLE_FORMATS.join(',')})") do |s|
+      format = s.to_sym
+      usage("Unsupported format: #{s}") unless AVAILABLE_FORMATS.include?(format)
+      @format = format
+    end
+    @parser.on("-I PATH", "Add PATH to $LOAD_PATH") do |s|
+      $LOAD_PATH.unshift(s)
+    end
+    @parser.on("-r NAME", "Load library") do |s|
+      @libs << s
+    end
+    @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
+      @plugin_dirs << s
+    end
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify plugin type and name" unless @argv.size == 2
+
+    @plugin_type, @plugin_name = @argv
+    @options = {
+      compact: @compact,
+      format: @format,
+      verbose: @verbose,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def init_libraries
+    @libs.each do |lib|
+      require lib
+    end
+
+    @plugin_dirs.each do |dir|
+      if Dir.exist?(dir)
+        dir = File.expand_path(dir)
+        Fluent::Plugin.add_plugin_dir(dir)
+      end
+    end
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/plugin_config_formatter/#{name}").realpath
+  end
+end

data/lib/fluent/command/plugin_generator.rb

@@ -0,0 +1,301 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "optparse"
+require "pathname"
+require "fileutils"
+require "erb"
+require "open-uri"
+
+require "fluent/registry"
+
+class FluentPluginGenerator
+  attr_reader :type, :name
+  attr_reader :license_name
+
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+
+  def initialize(argv = ARGV)
+    @argv = argv
+    @parser = prepare_parser
+
+    @license_name = "Apache-2.0"
+    @overwrite_all = false
+  end
+
+  def call
+    parse_options!
+    FileUtils.mkdir_p(gem_name)
+    Dir.chdir(gem_name) do
+      copy_license
+      template_directory.find do |path|
+        next if path.directory?
+        dest_dir = path.dirname.sub(/\A#{Regexp.quote(template_directory.to_s)}\/?/, "")
+        dest_file = dest_filename(path)
+        if path.extname == ".erb"
+          if path.fnmatch?("*/plugin/*")
+            next unless path.basename.fnmatch?("*#{type}*")
+          end
+          template(path, dest_dir + dest_file)
+        else
+          file(path, dest_dir + dest_file)
+        end
+      end
+      pid = spawn("git", "init", ".")
+      Process.wait(pid)
+    end
+  end
+
+  private
+
+  def template_directory
+    (Pathname(__dir__) + "../../../templates/new_gem").realpath
+  end
+
+  def template_file(filename)
+    template_directory + filename
+  end
+
+  def template(source, dest)
+    dest.dirname.mkpath
+    contents = ERB.new(source.read, nil, "-").result(binding)
+    label = create_label(dest, contents)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    File.write(dest, contents)
+  end
+
+  def file(source, dest)
+    label = create_label(dest, source.read)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    FileUtils.cp(source, dest)
+  end
+
+  def prepare_parser
+    @parser = OptionParser.new
+    @parser.banner = <<BANNER
+Usage: fluent-plugin-generate [options] <type> <name>
+
+Generate a project skeleton for creating a Fluentd plugin
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: Your plugin name
+
+Options:
+BANNER
+
+    @parser.on("--[no-]license=NAME", "Specify license name (default: Apache-2.0)") do |v|
+      @license_name = v || "no-license"
+    end
+    @parser
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+    unless @argv.size == 2
+      raise ArgumentError, "Missing arguments"
+    end
+    @type, @name = @argv
+  rescue => e
+    usage("#{e.class}:#{e.message}")
+  end
+
+  def usage(message = "")
+    puts message
+    puts
+    puts @parser.help
+    exit(false)
+  end
+
+  def user_name
+    v = `git config --get user.name`.chomp
+    v.empty? ? "TODO: Write your name" : v
+  end
+
+  def user_email
+    v = `git config --get user.email`.chomp
+    v.empty? ? "TODO: Write your email" : v
+  end
+
+  def gem_name
+    "fluent-plugin-#{name}"
+  end
+
+  def class_name
+    "#{name.capitalize}#{type.capitalize}"
+  end
+
+  def plugin_filename
+    case type
+    when "input"
+      "in_#{name}.rb"
+    when "output"
+      "out_#{name}.rb"
+    else
+      "#{type}_#{name}.rb"
+    end
+  end
+
+  def test_filename
+    case type
+    when "input"
+      "test_in_#{name}.rb"
+    when "output"
+      "test_out_#{name}.rb"
+    else
+      "test_#{type}_#{name}.rb"
+    end
+  end
+
+  def dest_filename(path)
+    case path.to_s
+    when %r!\.gemspec!
+      "#{gem_name}.gemspec"
+    when %r!lib/fluent/plugin!
+      plugin_filename
+    when %r!test/plugin!
+      test_filename
+    else
+      path.basename.sub_ext("")
+    end
+  end
+
+  def preamble
+    @license.preamble(user_name)
+  end
+
+  def copy_license
+    # in gem_name directory
+    return unless license_name
+    puts "License: #{license_name}"
+    license_class = self.class.lookup_license(license_name)
+    @license = license_class.new
+    Pathname("LICENSE").write(@license.text) unless @license.text.empty?
+  rescue Fluent::ConfigError
+    usage("Unknown license: #{license_name}")
+  rescue => ex
+    usage("#{ex.class}: #{ex.message}")
+  end
+
+  def create_label(dest, contents)
+    if dest.exist?
+      if dest.read == contents
+        "identical"
+      else
+        "conflict"
+      end
+    else
+      "create"
+    end
+  end
+
+  def overwrite?(dest)
+    return true if @overwrite_all
+    loop do
+      print "Overwrite #{dest}? (enter \"h\" for help) [Ynaqh]"
+      answer = $stdin.gets.chomp
+      return true if /\Ay\z/i =~ answer || answer.empty?
+      case answer
+      when "n"
+        return false
+      when "a"
+        @overwrite_all = true
+        return true
+      when "q"
+        exit
+      when "h"
+        puts <<HELP
+\tY - yes, overwrite
+\tn - no, do not overwrite
+\ta - all, overwrite this and all others
+\tq - quite, abort
+\th - help, show this help
+HELP
+      end
+      puts "Retrying..."
+    end
+  end
+
+  class NoLicense
+    attr_reader :name, :full_name, :text
+
+    def initialize
+      @name = ""
+      @full_name = ""
+      @text = ""
+    end
+
+    def preamble(usename)
+      ""
+    end
+  end
+
+  class ApacheLicense
+    LICENSE_URL = "http://www.apache.org/licenses/LICENSE-2.0.txt"
+
+    attr_reader :text
+
+    def initialize
+      @text = ""
+      @preamble_source = ""
+      @preamble = nil
+      open(LICENSE_URL) do |io|
+        @text = io.read
+      end
+      @preamble_source = @text[/^(\s*Copyright.+)/m, 1]
+    end
+
+    def name
+      "Apache-2.0"
+    end
+
+    def full_name
+      "Apache License, Version 2.0"
+    end
+
+    def preamble(user_name)
+      @preamble ||= @preamble_source.dup.tap do |source|
+        source.gsub!(/\[yyyy\]/, "#{Date.today.year}-")
+        source.gsub!(/\[name of copyright owner\]/, user_name)
+        source.gsub!(/^ {2}|^$/, "#")
+        source.chomp!
+      end
+    end
+  end
+
+  LICENSE_REGISTRY = Fluent::Registry.new(:license, "")
+
+  def self.register_license(license, klass)
+    LICENSE_REGISTRY.register(license, klass)
+  end
+
+  def self.lookup_license(license)
+    LICENSE_REGISTRY.lookup(license)
+  end
+
+  {
+    "no-license" => NoLicense,
+    "Apache-2.0" => ApacheLicense
+  }.each do |license, klass|
+    register_license(license, klass)
+  end
+end