dtomasgu-fluentd 1.14.7.pre.dev

data/lib/fluent/command/plugin_config_formatter.rb

@@ -0,0 +1,308 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "erb"
+require "optparse"
+require "pathname"
+require "fluent/plugin"
+require "fluent/env"
+require "fluent/engine"
+require "fluent/system_config"
+require "fluent/config/element"
+require 'fluent/version'
+
+class FluentPluginConfigFormatter
+
+  AVAILABLE_FORMATS = [:markdown, :txt, :json]
+  SUPPORTED_TYPES = [
+    "input", "output", "filter",
+    "buffer", "parser", "formatter", "storage",
+    "service_discovery"
+  ]
+
+  DOCS_BASE_URL = "https://docs.fluentd.org/v/1.0"
+  DOCS_PLUGIN_HELPER_BASE_URL = "#{DOCS_BASE_URL}/plugin-helper-overview/"
+
+  def initialize(argv = ARGV)
+    @argv = argv
+
+    @compact = false
+    @format = :markdown
+    @verbose = false
+    @libs = []
+    @plugin_dirs = []
+    @table = false
+    @options = {}
+
+    prepare_option_parser
+  end
+
+  def call
+    parse_options!
+    init_libraries
+    @plugin = Fluent::Plugin.__send__("new_#{@plugin_type}", @plugin_name)
+    dumped_config = {}
+    if @plugin.class.respond_to?(:plugin_helpers)
+      dumped_config[:plugin_helpers] = @plugin.class.plugin_helpers
+    end
+    @plugin.class.ancestors.reverse_each do |plugin_class|
+      next unless plugin_class.respond_to?(:dump_config_definition)
+      unless @verbose
+        next if plugin_class.name =~ /::PluginHelper::/
+      end
+      dumped_config_definition = plugin_class.dump_config_definition
+      dumped_config[plugin_class.name] = dumped_config_definition unless dumped_config_definition.empty?
+    end
+    case @format
+    when :txt
+      puts dump_txt(dumped_config)
+    when :markdown
+      puts dump_markdown(dumped_config)
+    when :json
+      puts dump_json(dumped_config)
+    end
+  end
+
+  private
+
+  def dump_txt(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped << "helpers: #{plugin_helpers.join(',')}\n"
+    end
+    if @verbose
+      dumped_config.each do |name, config|
+        dumped << "#{name}\n"
+        dumped << dump_section_txt(config)
+      end
+    else
+      configs = dumped_config.values
+      root_section = configs.shift
+      configs.each do |config|
+        root_section.update(config)
+      end
+      dumped << dump_section_txt(root_section)
+    end
+    dumped
+  end
+
+  def dump_section_txt(base_section, level = 0)
+    dumped = ""
+    indent = " " * level
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    params.each do |name, config|
+      next if name == :section
+      dumped << "#{indent}#{name}: #{config[:type]}: (#{config[:default].inspect})"
+      dumped << " # #{config[:description]}" if config.key?(:description)
+      dumped << "\n"
+    end
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      required_label = required ? "required" : "optional"
+      multi_label = multi ? "multiple" : "single"
+      alias_label = "alias: #{alias_name}"
+      dumped << "#{indent}<#{section_name}>: #{required_label}, #{multi_label}"
+      if alias_name
+        dumped << ", #{alias_label}\n"
+      else
+        dumped << "\n"
+      end
+      sub_section.delete(:section)
+      dumped << dump_section_txt(sub_section, level + 1)
+    end
+    dumped
+  end
+
+  def dump_markdown(dumped_config)
+    dumped = ""
+    plugin_helpers = dumped_config.delete(:plugin_helpers)
+    if plugin_helpers && !plugin_helpers.empty?
+      dumped = "## Plugin helpers\n\n"
+      plugin_helpers.each do |plugin_helper|
+        dumped << "* #{plugin_helper_markdown_link(plugin_helper)}\n"
+      end
+      dumped << "\n"
+    end
+    dumped_config.each do |name, config|
+      if name == @plugin.class.name
+        dumped << "## #{name}\n\n"
+        dumped << dump_section_markdown(config)
+      else
+        dumped << "* See also: #{plugin_overview_markdown_link(name)}\n\n"
+      end
+    end
+    dumped
+  end
+
+  def dump_section_markdown(base_section, level = 0)
+    dumped = ""
+    if base_section[:section]
+      sections = []
+      params = base_section
+    else
+      sections, params = base_section.partition {|_name, value| value[:section] }
+    end
+    if @table && (not params.empty?)
+      dumped << "### Configuration\n\n"
+      dumped << "|parameter|type|description|default|\n"
+      dumped << "|---|---|---|---|\n"
+    end
+    params.each do |name, config|
+      next if name == :section
+      template_name = if @compact
+                        "param.md-compact.erb"
+                      elsif @table
+                        "param.md-table.erb"
+                      else
+                        "param.md.erb"
+                      end
+      template = template_path(template_name).read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
+    end
+    dumped << "\n"
+    sections.each do |section_name, sub_section|
+      required = sub_section.delete(:required)
+      multi = sub_section.delete(:multi)
+      alias_name = sub_section.delete(:alias)
+      sub_section.delete(:section)
+      template = template_path("section.md.erb").read
+      dumped <<
+        if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+          ERB.new(template, trim_mode: "-")
+        else
+          ERB.new(template, nil, "-")
+        end.result(binding)
+    end
+    dumped
+  end
+
+  def dump_json(dumped_config)
+    if @compact
+      JSON.generate(dumped_config)
+    else
+      JSON.pretty_generate(dumped_config)
+    end
+  end
+
+  def plugin_helper_url(plugin_helper)
+    "#{DOCS_PLUGIN_HELPER_BASE_URL}api-plugin-helper-#{plugin_helper}"
+  end
+
+  def plugin_helper_markdown_link(plugin_helper)
+    "[#{plugin_helper}](#{plugin_helper_url(plugin_helper)})"
+  end
+
+  def plugin_overview_url(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1).downcase
+    "#{DOCS_BASE_URL}/#{plugin_type}#overview"
+  end
+
+  def plugin_overview_markdown_link(class_name)
+    plugin_type = class_name.slice(/::(\w+)\z/, 1)
+    "[#{plugin_type} Plugin Overview](#{plugin_overview_url(class_name)})"
+  end
+
+  def usage(message = nil)
+    puts @parser.to_s
+    puts
+    puts "Error: #{message}" if message
+    exit(false)
+  end
+
+  def prepare_option_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: #{$0} [options] <type> <name>
+
+Output plugin config definitions
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: registered plugin name
+
+Options:
+BANNER
+    @parser.on("--verbose", "Be verbose") do
+      @verbose = true
+    end
+    @parser.on("-c", "--compact", "Compact output") do
+      @compact = true
+    end
+    @parser.on("-f", "--format=FORMAT", "Specify format. (#{AVAILABLE_FORMATS.join(',')})") do |s|
+      format = s.to_sym
+      usage("Unsupported format: #{s}") unless AVAILABLE_FORMATS.include?(format)
+      @format = format
+    end
+    @parser.on("-I PATH", "Add PATH to $LOAD_PATH") do |s|
+      $LOAD_PATH.unshift(s)
+    end
+    @parser.on("-r NAME", "Load library") do |s|
+      @libs << s
+    end
+    @parser.on("-p", "--plugin=DIR", "Add plugin directory") do |s|
+      @plugin_dirs << s
+    end
+    @parser.on("-t", "--table", "Use table syntax to dump parameters") do
+      @table = true
+    end
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+
+    raise "Must specify plugin type and name" unless @argv.size == 2
+
+    @plugin_type, @plugin_name = @argv
+    @options = {
+      compact: @compact,
+      format: @format,
+      verbose: @verbose,
+    }
+  rescue => e
+    usage(e)
+  end
+
+  def init_libraries
+    @libs.each do |lib|
+      require lib
+    end
+
+    @plugin_dirs.each do |dir|
+      if Dir.exist?(dir)
+        dir = File.expand_path(dir)
+        Fluent::Plugin.add_plugin_dir(dir)
+      end
+    end
+  end
+
+  def template_path(name)
+    (Pathname(__dir__) + "../../../templates/plugin_config_formatter/#{name}").realpath
+  end
+end

data/lib/fluent/command/plugin_generator.rb

@@ -0,0 +1,365 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require "optparse"
+require "pathname"
+require "fileutils"
+require "erb"
+require "open-uri"
+
+require "fluent/registry"
+require 'fluent/version'
+
+class FluentPluginGenerator
+  attr_reader :type, :name
+  attr_reader :license_name
+
+  SUPPORTED_TYPES = ["input", "output", "filter", "parser", "formatter", "storage"]
+
+  def initialize(argv = ARGV)
+    @argv = argv
+    @parser = prepare_parser
+
+    @license_name = "Apache-2.0"
+    @overwrite_all = false
+  end
+
+  def call
+    parse_options!
+    FileUtils.mkdir_p(gem_name)
+    Dir.chdir(gem_name) do
+      copy_license
+      template_directory.find do |path|
+        next if path.directory?
+        dest_dir = path.dirname.sub(/\A#{Regexp.quote(template_directory.to_s)}\/?/, "")
+        dest_file = dest_filename(path)
+        if path.extname == ".erb"
+          if path.fnmatch?("*/plugin/*")
+            next unless path.basename.fnmatch?("*#{type}*")
+          end
+          template(path, dest_dir + dest_file)
+        else
+          file(path, dest_dir + dest_file)
+        end
+      end
+      pid = spawn("git", "init", ".")
+      Process.wait(pid)
+    end
+  end
+
+  private
+
+  def template_directory
+    (Pathname(__dir__) + "../../../templates/new_gem").realpath
+  end
+
+  def template_file(filename)
+    template_directory + filename
+  end
+
+  def template(source, dest)
+    dest.dirname.mkpath
+    contents =
+      if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+
+        ERB.new(source.read, trim_mode: "-")
+      else
+        ERB.new(source.read, nil, "-")
+      end.result(binding)
+    label = create_label(dest, contents)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    File.write(dest, contents)
+  end
+
+  def file(source, dest)
+    label = create_label(dest, source.read)
+    puts "\t#{label} #{dest}"
+    if label == "conflict"
+      return unless overwrite?(dest)
+    end
+    FileUtils.cp(source, dest)
+  end
+
+  def prepare_parser
+    @parser = OptionParser.new
+    @parser.version = Fluent::VERSION
+    @parser.banner = <<BANNER
+Usage: fluent-plugin-generate [options] <type> <name>
+
+Generate a project skeleton for creating a Fluentd plugin
+
+Arguments:
+\ttype: #{SUPPORTED_TYPES.join(",")}
+\tname: Your plugin name (fluent-plugin- prefix will be added to <name>)
+
+Options:
+BANNER
+
+    @parser.on("--[no-]license=NAME", "Specify license name (default: Apache-2.0)") do |v|
+      @license_name = v || "no-license"
+    end
+    @parser
+  end
+
+  def parse_options!
+    @parser.parse!(@argv)
+    unless @argv.size == 2
+      raise ArgumentError, "Missing arguments"
+    end
+    @type, @name = @argv
+  rescue => e
+    usage("#{e.class}:#{e.message}")
+  end
+
+  def usage(message = "")
+    puts message
+    puts
+    puts @parser.help
+    exit(false)
+  end
+
+  def user_name
+    v = `git config --get user.name`.chomp
+    v.empty? ? "TODO: Write your name" : v
+  end
+
+  def user_email
+    v = `git config --get user.email`.chomp
+    v.empty? ? "TODO: Write your email" : v
+  end
+
+  def gem_name
+    "fluent-plugin-#{dash_name}"
+  end
+
+  def plugin_name
+    underscore_name
+  end
+
+  def gem_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile"))
+  end
+
+  def lock_file_path
+    File.expand_path(File.join(File.dirname(__FILE__),
+                               "../../../",
+                               "Gemfile.lock"))
+  end
+
+  def locked_gem_version(gem_name)
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.dependencies[gem_name].requirement.requirements.first.last.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name(gem_name).version.version
+    end
+  end
+
+  def rake_version
+    locked_gem_version("rake")
+  end
+
+  def test_unit_version
+    locked_gem_version("test-unit")
+  end
+
+  def bundler_version
+    if File.exist?(lock_file_path)
+      d = Bundler::Definition.build(gem_file_path, lock_file_path, false)
+      d.locked_gems.bundler_version.version
+    else
+      # fallback even though Fluentd is installed without bundler
+      Gem::Specification.find_by_name("bundler").version.version
+    end
+  end
+
+  def class_name
+    "#{capitalized_name}#{type.capitalize}"
+  end
+
+  def plugin_filename
+    case type
+    when "input"
+      "in_#{underscore_name}.rb"
+    when "output"
+      "out_#{underscore_name}.rb"
+    else
+      "#{type}_#{underscore_name}.rb"
+    end
+  end
+
+  def test_filename
+    case type
+    when "input"
+      "test_in_#{underscore_name}.rb"
+    when "output"
+      "test_out_#{underscore_name}.rb"
+    else
+      "test_#{type}_#{underscore_name}.rb"
+    end
+  end
+
+  def dest_filename(path)
+    case path.to_s
+    when %r!\.gemspec!
+      "#{gem_name}.gemspec"
+    when %r!lib/fluent/plugin!
+      plugin_filename
+    when %r!test/plugin!
+      test_filename
+    else
+      path.basename.sub_ext("")
+    end
+  end
+
+  def capitalized_name
+    @capitalized_name ||= name.split(/[-_]/).map(&:capitalize).join
+  end
+
+  def underscore_name
+    @underscore_name ||= name.tr("-", "_")
+  end
+
+  def dash_name
+    @dash_name ||= name.tr("_", "-")
+  end
+
+  def preamble
+    @license.preamble(user_name)
+  end
+
+  def copy_license
+    # in gem_name directory
+    return unless license_name
+    puts "License: #{license_name}"
+    license_class = self.class.lookup_license(license_name)
+    @license = license_class.new
+    Pathname("LICENSE").write(@license.text) unless @license.text.empty?
+  rescue Fluent::ConfigError
+    usage("Unknown license: #{license_name}")
+  rescue => ex
+    usage("#{ex.class}: #{ex.message}")
+  end
+
+  def create_label(dest, contents)
+    if dest.exist?
+      if dest.read == contents
+        "identical"
+      else
+        "conflict"
+      end
+    else
+      "create"
+    end
+  end
+
+  def overwrite?(dest)
+    return true if @overwrite_all
+    loop do
+      print "Overwrite #{dest}? (enter \"h\" for help) [Ynaqh]"
+      answer = $stdin.gets.chomp
+      return true if /\Ay\z/i =~ answer || answer.empty?
+      case answer
+      when "n"
+        return false
+      when "a"
+        @overwrite_all = true
+        return true
+      when "q"
+        exit
+      when "h"
+        puts <<HELP
+\tY - yes, overwrite
+\tn - no, do not overwrite
+\ta - all, overwrite this and all others
+\tq - quit, abort
+\th - help, show this help
+HELP
+      end
+      puts "Retrying..."
+    end
+  end
+
+  class NoLicense
+    attr_reader :name, :full_name, :text
+
+    def initialize
+      @name = ""
+      @full_name = ""
+      @text = ""
+    end
+
+    def preamble(usename)
+      ""
+    end
+  end
+
+  class ApacheLicense
+    LICENSE_URL = "http://www.apache.org/licenses/LICENSE-2.0.txt"
+
+    attr_reader :text
+
+    def initialize
+      @text = ""
+      @preamble_source = ""
+      @preamble = nil
+      uri = URI.parse(LICENSE_URL)
+      uri.open do |io|
+        @text = io.read
+      end
+      @preamble_source = @text[/^(\s*Copyright.+)/m, 1]
+    end
+
+    def name
+      "Apache-2.0"
+    end
+
+    def full_name
+      "Apache License, Version 2.0"
+    end
+
+    def preamble(user_name)
+      @preamble ||= @preamble_source.dup.tap do |source|
+        source.gsub!(/\[yyyy\]/, "#{Date.today.year}-")
+        source.gsub!(/\[name of copyright owner\]/, user_name)
+        source.gsub!(/^ {2}|^$/, "#")
+        source.chomp!
+      end
+    end
+  end
+
+  LICENSE_REGISTRY = Fluent::Registry.new(:license, "")
+
+  def self.register_license(license, klass)
+    LICENSE_REGISTRY.register(license, klass)
+  end
+
+  def self.lookup_license(license)
+    LICENSE_REGISTRY.lookup(license)
+  end
+
+  {
+    "no-license" => NoLicense,
+    "Apache-2.0" => ApacheLicense
+  }.each do |license, klass|
+    register_license(license, klass)
+  end
+end