fluentd 0.14.4-x64-mingw32

data/lib/fluent/command/fluentd.rb

@@ -0,0 +1,273 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+
+require 'fluent/supervisor'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/version'
+
+$fluentdargv = Marshal.load(Marshal.dump(ARGV))
+
+op = OptionParser.new
+op.version = Fluent::VERSION
+
+opts = Fluent::Supervisor.default_options
+
+op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
+  opts[:setup_path] = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
+}
+
+op.on('-c', '--config PATH', "config file path (default: #{Fluent::DEFAULT_CONFIG_PATH})") {|s|
+  opts[:config_path] = s
+}
+
+op.on('--dry-run', "Check fluentd setup is correct or not", TrueClass) {|b|
+  opts[:dry_run] = b
+}
+
+op.on('--show-plugin-config=PLUGIN', "Show PLUGIN configuration and exit(ex: input:dummy)") {|plugin|
+  opts[:show_plugin_config] = plugin
+}
+
+op.on('-p', '--plugin DIR', "add plugin directory") {|s|
+  opts[:plugin_dirs] << s
+}
+
+op.on('-I PATH', "add library path") {|s|
+  $LOAD_PATH << s
+}
+
+op.on('-r NAME', "load library") {|s|
+  opts[:libs] << s
+}
+
+op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
+  opts[:daemonize] = s
+}
+
+op.on('--under-supervisor', "run fluent worker under supervisor (this option is NOT for users)") {
+  opts[:supervise] = false
+}
+
+op.on('--no-supervisor', "run fluent worker without supervisor") {
+  opts[:supervise] = false
+  opts[:standalone_worker] = true
+}
+
+op.on('--user USER', "change user") {|s|
+  opts[:chuser] = s
+}
+
+op.on('--group GROUP', "change group") {|s|
+  opts[:chgroup] = s
+}
+
+op.on('-o', '--log PATH', "log file path") {|s|
+  opts[:log_path] = s
+}
+
+op.on('-i', '--inline-config CONFIG_STRING', "inline config which is appended to the config file on-the-fly") {|s|
+  opts[:inline_config] = s
+}
+
+op.on('--emit-error-log-interval SECONDS', "suppress interval seconds of emit error logs") {|s|
+  opts[:suppress_interval] = s.to_i
+}
+
+op.on('--suppress-repeated-stacktrace [VALUE]', "suppress repeated stacktrace", TrueClass) {|b|
+  b = true if b.nil?
+  opts[:suppress_repeated_stacktrace] = b
+}
+
+op.on('--without-source', "invoke a fluentd without input plugins", TrueClass) {|b|
+  opts[:without_source] = b
+}
+
+op.on('--use-v1-config', "Use v1 configuration format (default)", TrueClass) {|b|
+  opts[:use_v1_config] = b
+}
+
+op.on('--use-v0-config', "Use v0 configuration format", TrueClass) {|b|
+  opts[:use_v1_config] = !b
+}
+
+op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max
+  end
+}
+
+op.on('-q', '--quiet', "decrease verbose level (-q: warn, -qq: error)", TrueClass) {|b|
+  if b
+    opts[:log_level] = [opts[:log_level] + 1, Fluent::Log::LEVEL_ERROR].min
+  end
+}
+
+op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", TrueClass) {|b|
+  opts[:suppress_config_dump] = b
+}
+
+op.on('-g', '--gemfile GEMFILE', "Gemfile path") {|s|
+  opts[:gemfile] = s
+}
+
+op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)") {|s|
+  opts[:gem_install_path] = s
+}
+
+if Fluent.windows?
+  require 'windows/library'
+  include Windows::Library
+
+  op.on('-x', '--signame INTSIGNAME', "an object name which is used for Windows Service signal (Windows only)") {|s|
+    opts[:signame] = s
+  }
+
+  op.on('--reg-winsvc MODE', "install/uninstall as Windows Service. (i: install, u: uninstall) (Windows only)") {|s|
+    opts[:regwinsvc] = s
+  }
+
+  op.on('--[no-]reg-winsvc-auto-start', "Automatically start the Windows Service at boot. (only effective with '--reg-winsvc i') (Windows only)") {|s|
+    opts[:regwinsvcautostart] = s
+  }
+
+  op.on('--reg-winsvc-fluentdopt OPTION', "specify fluentd option paramters for Windows Service. (Windows only)") {|s|
+    opts[:fluentdopt] = s
+  }
+end
+
+
+(class << self; self; end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  rest = op.parse(ARGV)
+
+  if rest.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+
+##
+## Bundler injection
+#
+if ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] != '1' && gemfile = opts[:gemfile]
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  if path = opts[:gem_install_path]
+    ENV['BUNDLE_PATH'] = path
+  else
+    ENV['BUNDLE_PATH'] = File.expand_path(File.join(File.dirname(gemfile), 'vendor/bundle'))
+  end
+  ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] = '1'
+  load File.expand_path(File.join(File.dirname(__FILE__), 'bundler_injection.rb'))
+end
+
+if setup_path = opts[:setup_path]
+  require 'fileutils'
+  FileUtils.mkdir_p File.join(setup_path, "plugin")
+  confpath = File.join(setup_path, "fluent.conf")
+  if File.exist?(confpath)
+    puts "#{confpath} already exists."
+  else
+    File.open(confpath, "w") {|f|
+      conf = File.read File.join(File.dirname(__FILE__), "..", "..", "..", "fluent.conf")
+      f.write conf
+    }
+    puts "Installed #{confpath}."
+  end
+  exit 0
+end
+
+early_exit = false
+start_service = false
+if winsvcinstmode = opts[:regwinsvc]
+  FLUENTD_WINSVC_NAME="fluentdwinsvc"
+  FLUENTD_WINSVC_DISPLAYNAME="Fluentd Windows Service"
+  FLUENTD_WINSVC_DESC="Fluentd is an event collector system."
+  require 'fileutils'
+  require "win32/service"
+  require "win32/registry"
+  include Win32
+
+  case winsvcinstmode
+  when 'i'
+    binary_path = File.join(File.dirname(__FILE__), "..")
+    ruby_path = "\0" * 256
+    GetModuleFileName.call(0,ruby_path,256)
+    ruby_path = ruby_path.rstrip.gsub(/\\/, '/')
+    start_type = Service::DEMAND_START
+    if opts[:regwinsvcautostart]
+      start_type = Service::AUTO_START
+      start_service = true
+    end
+
+    Service.create(
+      service_name: FLUENTD_WINSVC_NAME,
+      host: nil,
+      service_type: Service::WIN32_OWN_PROCESS,
+      description: FLUENTD_WINSVC_DESC,
+      start_type: start_type,
+      error_control: Service::ERROR_NORMAL,
+      binary_path_name: ruby_path+" -C "+binary_path+" winsvc.rb",
+      load_order_group: "",
+      dependencies: [""],
+      display_name: FLUENTD_WINSVC_DISPLAYNAME
+    )
+  when 'u'
+    if Service.status(FLUENTD_WINSVC_NAME).current_state != 'stopped'
+      begin
+        Service.stop(FLUENTD_WINSVC_NAME)
+      rescue => ex
+        puts "Warning: Failed to stop service: ", ex
+      end
+    end
+    Service.delete(FLUENTD_WINSVC_NAME)
+  else
+    # none
+  end
+  early_exit = true
+end
+
+if fluentdopt = opts[:fluentdopt]
+  Win32::Registry::HKEY_LOCAL_MACHINE.open("SYSTEM\\CurrentControlSet\\Services\\fluentdwinsvc", Win32::Registry::KEY_ALL_ACCESS) do |reg|
+    reg['fluentdopt', Win32::Registry::REG_SZ] = fluentdopt
+  end
+  early_exit = true
+end
+
+if start_service
+  Service.start(FLUENTD_WINSVC_NAME)
+end
+
+exit 0 if early_exit
+
+require 'fluent/supervisor'
+if opts[:supervise]
+  Fluent::Supervisor.new(opts).run_supervisor
+else
+  Fluent::Supervisor.new(opts).run_worker
+end
+

data/lib/fluent/compat/call_super_mixin.rb

@@ -0,0 +1,67 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Compat
+    module CallSuperMixin
+      # This mixin is to prepend to 3rd party plugins of v0.12 APIs.
+      # In past, there were not strong rule to call super in #start, #before_shutdown and #shutdown.
+      # But v0.14 API requires to call super in these methods to setup/teardown plugin helpers and others.
+      # This mixin prepends method calls to call super forcedly if checker returns false (it shows Fluent::Plugin::Base#methods wasn't called)
+
+      def self.prepended(klass)
+        @@_super_start ||= {}
+        @@_super_before_shutdown ||= {}
+        @@_super_shutdown ||= {}
+
+        # ancestors[0]: this module
+        # ancestors[1]: prepended class (plugin itself)
+        method_search = ->(ancestors, method){
+          closest = ancestors[2, ancestors.size - 2].index{|m| m.method_defined?(method) }
+          ancestors[2 + closest].instance_method(method)
+        }
+        @@_super_start[klass]           = method_search.call(klass.ancestors, :start) # this returns Fluent::Compat::*#start (or helpers on it)
+        @@_super_before_shutdown[klass] = method_search.call(klass.ancestors, :before_shutdown)
+        @@_super_shutdown[klass]        = method_search.call(klass.ancestors, :shutdown)
+      end
+
+      def start
+        super
+        unless self.started?
+          @@_super_start[self.class].bind(self).call
+          # #super will reset logdev (especially in test), so this warn should be after calling it
+          log.warn "super was not called in #start: called it forcedly", plugin: self.class
+        end
+      end
+
+      def before_shutdown
+        super
+        unless self.before_shutdown?
+          log.warn "super was not called in #before_shutdown: calling it forcedly", plugin: self.class
+          @@_super_before_shutdown[self.class].bind(self).call
+        end
+      end
+
+      def shutdown
+        super
+        unless self.shutdown?
+          log.warn "super was not called in #shutdown: calling it forcedly", plugin: self.class
+          @@_super_shutdown[self.class].bind(self).call
+        end
+      end
+    end
+  end
+end

data/lib/fluent/compat/exec_util.rb

@@ -0,0 +1,129 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
+module Fluent
+  module Compat
+    module ExecUtil
+      SUPPORTED_FORMAT = {
+        'tsv' => :tsv,
+        'json' => :json,
+        'msgpack' => :msgpack,
+      }
+
+      class Parser
+        def initialize(on_message)
+          @on_message = on_message
+        end
+      end
+
+      class TextParserWrapperParser < Parser
+        def initialize(conf, on_message)
+          @parser = Plugin.new_parser(conf['format'])
+          @parser.configure(conf)
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          @parser.parse(line) { |time, record|
+            @on_message.call(record, time)
+          }
+        end
+      end
+
+      class TSVParser < Parser
+        def initialize(keys, on_message)
+          @keys = keys
+          super(on_message)
+        end
+
+        def call(io)
+          io.each_line(&method(:each_line))
+        end
+
+        def each_line(line)
+          line.chomp!
+          vals = line.split("\t")
+
+          record = Hash[@keys.zip(vals)]
+
+          @on_message.call(record)
+        end
+      end
+
+      class JSONParser < Parser
+        def call(io)
+          y = Yajl::Parser.new
+          y.on_parse_complete = @on_message
+          y.parse(io)
+        end
+      end
+
+      class MessagePackParser < Parser
+        def call(io)
+          @u = Fluent::Engine.msgpack_factory.unpacker(io)
+          begin
+            @u.each(&@on_message)
+          rescue EOFError
+          end
+        end
+      end
+
+      class Formatter
+      end
+
+      class TSVFormatter < Formatter
+        def initialize(in_keys)
+          @in_keys = in_keys
+          super()
+        end
+
+        def call(record, out)
+          last = @in_keys.length-1
+          for i in 0..last
+            key = @in_keys[i]
+            out << record[key].to_s
+            out << "\t" if i != last
+          end
+          out << "\n"
+        end
+      end
+
+      class JSONFormatter < Formatter
+        def call(record, out)
+          out << Yajl.dump(record) << "\n"
+        end
+      end
+
+      class MessagePackFormatter < Formatter
+        def call(record, out)
+          record.to_msgpack(out)
+        end
+      end
+    end
+  end
+end