fluentd 0.12.40 → 1.6.2

data/lib/fluent/command/plugin_generator.rb

@@ -0,0 +1,324 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "optparse"
+require "pathname"
+require "fileutils"
+require "erb"
+require "open-uri"
+
+require "fluent/registry"
+require 'fluent/version'
+
+class FluentPluginGenerator
+  attr_reader :type, :name
+  attr_reader :license_name
+
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter"]
+
+  def initialize(argv = ARGV)
+    @argv = argv
+    @parser = prepare_parser
+
+    @license_name = "Apache-2.0"
+    @overwrite_all = false
+  end
+
+  def call
+    parse_options!
+    FileUtils.mkdir_p(gem_name)
+    Dir.chdir(gem_name) do
+      copy_license
+      template_directory.find do |path|
+        next if path.directory?
+        dest_dir = path.dirname.sub(/\A#{Regexp.quote(template_directory.to_s)}\/?/, "")
+        dest_file = dest_filename(path)
+        if path.extname == ".erb"
+          if path.fnmatch?("*/plugin/*")
+            next unless path.basename.fnmatch?("*#{type}*")
+          end
+          template(path, dest_dir + dest_file)
+        else
+          file(path, dest_dir + dest_file)
+        end
+      end
+      pid = spawn("git", "init", ".")
+      Process.wait(pid)
+    end
+  end
+
+  private
+
+  def template_directory
+    (Pathname(__dir__) + "../../../templates/new_gem").realpath
+  end
+
+  def template_file(filename)
+    template_directory + filename
+  end
+
+  def template(source, dest)
+    dest.dirname.mkpath
+    contents =
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(source.read, trim_mode: "-")
+      else
+        ERB.new(source.read, nil, "-")
+      end.result(binding)
+    label = create_label(dest, contents)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    File.write(dest, contents)
+  end
+
+  def file(source, dest)
+    label = create_label(dest, source.read)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    FileUtils.cp(source, dest)
+  end
+
+  def prepare_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: fluent-plugin-generate [options] <type> <name>
+
+Generate a project skeleton for creating a Fluentd plugin
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: Your plugin name
+
+Options:
+BANNER
+
+    @parser.on("--[no-]license=NAME", "Specify license name (default: Apache-2.0)") do |v|
+      @license_name = v || "no-license"
+    end
+    @parser
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+    unless @argv.size == 2
+      raise ArgumentError, "Missing arguments"
+    end
+    @type, @name = @argv
+  rescue => e
+    usage("#{e.class}:#{e.message}")
+  end
+
+  def usage(message = "")
+    puts message
+    puts
+    puts @parser.help
+    exit(false)
+  end
+
+  def user_name
+    v = `git config --get user.name`.chomp
+    v.empty? ? "TODO: Write your name" : v
+  end
+
+  def user_email
+    v = `git config --get user.email`.chomp
+    v.empty? ? "TODO: Write your email" : v
+  end
+
+  def gem_name
+    "fluent-plugin-#{dash_name}"
+  end
+
+  def plugin_name
+    underscore_name
+  end
+
+  def class_name
+    "#{capitalized_name}#{type.capitalize}"
+  end
+
+  def plugin_filename
+    case type
+    when "input"
+      "in_#{underscore_name}.rb"
+    when "output"
+      "out_#{underscore_name}.rb"
+    else
+      "#{type}_#{underscore_name}.rb"
+    end
+  end
+
+  def test_filename
+    case type
+    when "input"
+      "test_in_#{underscore_name}.rb"
+    when "output"
+      "test_out_#{underscore_name}.rb"
+    else
+      "test_#{type}_#{underscore_name}.rb"
+    end
+  end
+
+  def dest_filename(path)
+    case path.to_s
+    when %r!\.gemspec!
+      "#{gem_name}.gemspec"
+    when %r!lib/fluent/plugin!
+      plugin_filename
+    when %r!test/plugin!
+      test_filename
+    else
+      path.basename.sub_ext("")
+    end
+  end
+
+  def capitalized_name
+    @capitalized_name ||= name.split(/[-_]/).map(&:capitalize).join
+  end
+
+  def underscore_name
+    @underscore_name ||= name.tr("-", "_")
+  end
+
+  def dash_name
+    @dash_name ||= name.tr("_", "-")
+  end
+
+  def preamble
+    @license.preamble(user_name)
+  end
+
+  def copy_license
+    # in gem_name directory
+    return unless license_name
+    puts "License: #{license_name}"
+    license_class = self.class.lookup_license(license_name)
+    @license = license_class.new
+    Pathname("LICENSE").write(@license.text) unless @license.text.empty?
+  rescue Fluent::ConfigError
+    usage("Unknown license: #{license_name}")
+  rescue => ex
+    usage("#{ex.class}: #{ex.message}")
+  end
+
+  def create_label(dest, contents)
+    if dest.exist?
+      if dest.read == contents
+        "identical"
+      else
+        "conflict"
+      end
+    else
+      "create"
+    end
+  end
+
+  def overwrite?(dest)
+    return true if @overwrite_all
+    loop do
+      print "Overwrite #{dest}? (enter \"h\" for help) [Ynaqh]"
+      answer = $stdin.gets.chomp
+      return true if /\Ay\z/i =~ answer || answer.empty?
+      case answer
+      when "n"
+        return false
+      when "a"
+        @overwrite_all = true
+        return true
+      when "q"
+        exit
+      when "h"
+        puts <<HELP
+\tY - yes, overwrite
+\tn - no, do not overwrite
+\ta - all, overwrite this and all others
+\tq - quit, abort
+\th - help, show this help
+HELP
+      end
+      puts "Retrying..."
+    end
+  end
+
+  class NoLicense
+    attr_reader :name, :full_name, :text
+
+    def initialize
+      @name = ""
+      @full_name = ""
+      @text = ""
+    end
+
+    def preamble(usename)
+      ""
+    end
+  end
+
+  class ApacheLicense
+    LICENSE_URL = "http://www.apache.org/licenses/LICENSE-2.0.txt"
+
+    attr_reader :text
+
+    def initialize
+      @text = ""
+      @preamble_source = ""
+      @preamble = nil
+      open(LICENSE_URL) do |io|
+        @text = io.read
+      end
+      @preamble_source = @text[/^(\s*Copyright.+)/m, 1]
+    end
+
+    def name
+      "Apache-2.0"
+    end
+
+    def full_name
+      "Apache License, Version 2.0"
+    end
+
+    def preamble(user_name)
+      @preamble ||= @preamble_source.dup.tap do |source|
+        source.gsub!(/\[yyyy\]/, "#{Date.today.year}-")
+        source.gsub!(/\[name of copyright owner\]/, user_name)
+        source.gsub!(/^ {2}|^$/, "#")
+        source.chomp!
+      end
+    end
+  end
+
+  LICENSE_REGISTRY = Fluent::Registry.new(:license, "")
+
+  def self.register_license(license, klass)
+    LICENSE_REGISTRY.register(license, klass)
+  end
+
+  def self.lookup_license(license)
+    LICENSE_REGISTRY.lookup(license)
+  end
+
+  {
+    "no-license" => NoLicense,
+    "Apache-2.0" => ApacheLicense
+  }.each do |license, klass|
+    register_license(license, klass)
+  end
+end

data/lib/fluent/compat/call_super_mixin.rb

@@ -0,0 +1,67 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module CallSuperMixin
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # In past, there were not strong rule to call super in #start, #before_shutdown and #shutdown.
+      # But v0.14 API requires to call super in these methods to setup/teardown plugin helpers and others.
+      # This mixin prepends method calls to call super forcedly if checker returns false (it shows Fluent::Plugin::Base#methods wasn't called)
+
+      def self.prepended(klass)
+        @@_super_start ||= {}
+        @@_super_before_shutdown ||= {}
+        @@_super_shutdown ||= {}
+
+        # ancestors[0]: this module
+        # ancestors[1]: prepended class (plugin itself)
+        method_search = ->(ancestors, method){
+          closest = ancestors[2, ancestors.size - 2].index{|m| m.method_defined?(method) }
+          ancestors[2 + closest].instance_method(method)
+        }
+        @@_super_start[klass]           = method_search.call(klass.ancestors, :start) # this returns Fluent::Compat::*#start (or helpers on it)
+        @@_super_before_shutdown[klass] = method_search.call(klass.ancestors, :before_shutdown)
+        @@_super_shutdown[klass]        = method_search.call(klass.ancestors, :shutdown)
+      end
+
+      def start
+        super
+        unless self.started?
+          @@_super_start[self.class].bind(self).call
+          # #super will reset logdev (especially in test), so this warn should be after calling it
+          log.warn "super was not called in #start: called it forcedly", plugin: self.class
+        end
+      end
+
+      def before_shutdown
+        super
+        unless self.before_shutdown?
+          log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
+          @@_super_before_shutdown[self.class].bind(self).call
+        end
+      end
+
+      def shutdown
+        super
+        unless self.shutdown?
+          log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
+          @@_super_shutdown[self.class].bind(self).call
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/detach_process_mixin.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module DetachProcessMixin
+      def detach_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
+    end
+
+    module DetachMultiProcessMixin
+      def detach_multi_process
+        log.warn "#{__method__} is not supported in this version. ignored."
+        yield
+      end
+    end
+  end
+end

data/lib/fluent/compat/exec_util.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
+module Fluent
+  module Compat
+    module ExecUtil
+      SUPPORTED_FORMAT = {
+        'tsv' => :tsv,
+        'json' => :json,
+        'msgpack' => :msgpack,
+      }
+
+      class Parser
+        def initialize(on_message)
+          @on_message = on_message
+        end
+      end
+
+      class TextParserWrapperParser < Parser
+        def initialize(conf, on_message)
+          @parser = Plugin.new_parser(conf['format'])
+          @parser.configure(conf)
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          @parser.parse(line) { |time, record|
+            @on_message.call(record, time)
+          }
+        end
+      end
+
+      class TSVParser < Parser
+        def initialize(keys, on_message)
+          @keys = keys
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          vals = line.split("\t")
+
+          record = Hash[@keys.zip(vals)]
+
+          @on_message.call(record)
+        end
+      end
+
+      class JSONParser < Parser
+        def call(io)
+          y = Yajl::Parser.new
+          y.on_parse_complete = @on_message
+          y.parse(io)
+        end
+      end
+
+      class MessagePackParser < Parser
+        def call(io)
+          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          begin
+            @u.each(&@on_message)
+          rescue EOFError
+          end
+        end
+      end
+
+      class Formatter
+      end
+
+      class TSVFormatter < Formatter
+        def initialize(in_keys)
+          @in_keys = in_keys
+          super()
+        end
+
+        def call(record, out)
+          last = @in_keys.length-1
+          for i in 0..last
+            key = @in_keys[i]
+            out << record[key].to_s
+            out << "\t" if i != last
+          end
+          out << "\n"
+        end
+      end
+
+      class JSONFormatter < Formatter
+        def call(record, out)
+          out << Yajl.dump(record) << "\n"
+        end
+      end
+
+      class MessagePackFormatter < Formatter
+        def call(record, out)
+          record.to_msgpack(out)
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/file_util.rb

@@ -0,0 +1,54 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module FileUtil
+      # Check file is writable if file exists
+      # Check directory is writable if file does not exist
+      #
+      # @param [String] path File path
+      # @return [Boolean] file is writable or not
+      def writable?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable?
+
+      # Check file is writable in conjunction with mkdir_p(dirname(path))
+      #
+      # @param [String] path File path
+      # @return [Boolean] file writable or not
+      def writable_p?(path)
+        return false if File.directory?(path)
+        return File.writable?(path) if File.exist?(path)
+
+        dirname = File.dirname(path)
+        until File.exist?(dirname)
+          dirname = File.dirname(dirname)
+        end
+
+        return false if !File.directory?(dirname)
+        File.writable?(dirname)
+      end
+      module_function :writable_p?
+    end
+  end
+end

data/lib/fluent/compat/filter.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin'
+require 'fluent/plugin/filter'
+require 'fluent/compat/call_super_mixin'
+require 'fluent/compat/formatter_utils'
+require 'fluent/compat/parser_utils'
+
+module Fluent
+  module Compat
+    class Filter < Fluent::Plugin::Filter
+      # TODO: warn when deprecated
+
+      helpers_internal :inject
+
+      def initialize
+        super
+        unless self.class.ancestors.include?(Fluent::Compat::CallSuperMixin)
+          self.class.prepend Fluent::Compat::CallSuperMixin
+        end
+      end
+
+      def configure(conf)
+        ParserUtils.convert_parser_conf(conf)
+        FormatterUtils.convert_formatter_conf(conf)
+
+        super
+      end
+
+      # These definitions are to get instance methods of superclass of 3rd party plugins
+      # to make it sure to call super
+      def start
+        super
+
+        if instance_variable_defined?(:@formatter) && @inject_config
+          unless @formatter.class.ancestors.include?(Fluent::Compat::HandleTagAndTimeMixin)
+            if @formatter.respond_to?(:owner) && !@formatter.owner
+              @formatter.owner = self
+              @formatter.singleton_class.prepend FormatterUtils::InjectMixin
+            end
+          end
+        end
+      end
+
+      def before_shutdown
+        super
+      end
+
+      def shutdown
+        super
+      end
+    end
+  end
+end