fluentd 0.10.0

data/lib/fluent/command/fluentd.rb

@@ -0,0 +1,245 @@
+#
+# Fluentd
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/log'
+require 'fluent/env'
+require 'fluent/version'
+
+op = OptionParser.new
+op.version = Fluent::VERSION
+
+# default values
+config_path = Fluent::DEFAULT_CONFIG_PATH
+plugin_dirs = [Fluent::DEFAULT_PLUGIN_DIR]
+log_level = Fluent::Log::LEVEL_INFO
+log_file = nil
+daemonize = false
+libs = []
+setup_path = nil
+chuser = nil
+chgroup = nil
+
+op.on('-s', "--setup [DIR=#{File.dirname(Fluent::DEFAULT_CONFIG_PATH)}]", "install sample configuration file to the directory") {|s|
+  setup_path = s || File.dirname(Fluent::DEFAULT_CONFIG_PATH)
+}
+
+op.on('-c', '--config PATH', "config flie path (default: #{config_path})") {|s|
+  config_path = s
+}
+
+op.on('-p', '--plugin DIR', "add plugin directory") {|s|
+  plugin_dirs << s
+}
+
+op.on('-I PATH', "add library path") {|s|
+  $LOAD_PATH << s
+}
+
+op.on('-r NAME', "load library") {|s|
+  libs << s
+}
+
+op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
+  daemonize = s
+}
+
+op.on('--user USER', "change user") {|s|
+  chuser = s
+}
+
+op.on('--group GROUP', "change group") {|s|
+  chgroup = s
+}
+
+op.on('-o', '--log PATH', "log file path") {|s|
+  log_file = s
+}
+
+op.on('-v', '--verbose', "increment verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
+  if b
+    case log_level
+    when Fluent::Log::LEVEL_INFO
+      log_level = Fluent::Log::LEVEL_DEBUG
+    when Fluent::Log::LEVEL_DEBUG
+      log_level = Fluent::Log::LEVEL_TRACE
+    end
+  end
+}
+
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+
+if setup_path
+  require 'fileutils'
+  FileUtils.mkdir_p File.join(setup_path, "plugin")
+  confpath = File.join(setup_path, "fluent.conf")
+  if File.exist?(confpath)
+    puts "#{confpath} already exists."
+  else
+    File.open(confpath, "w") {|f|
+      conf = File.read File.join(File.dirname(__FILE__), "..", "..", "..", "fluent.conf")
+      f.write conf
+    }
+    puts "Installed #{confpath}."
+  end
+  exit 0
+end
+
+
+if log_file && log_file != "-"
+  log_out = File.open(log_file, "a")
+else
+  log_out = STDOUT
+end
+
+$log = Fluent::Log.new(log_out, log_level)
+
+$log.enable_color(false) if log_file
+$log.enable_debug if log_level <= Fluent::Log::LEVEL_DEBUG
+
+
+require 'fluent/load'
+
+begin
+  #
+  # initialize
+  #
+  Fluent::Engine.init
+
+  libs.each {|lib|
+    require lib
+  }
+
+  plugin_dirs.each {|dir|
+    if Dir.exist?(dir)
+      dir = File.expand_path(dir)
+      Fluent::Engine.load_plugin_dir(dir)
+    end
+  }
+
+  Fluent::Engine.read_config(config_path)
+
+
+  #
+  # daemonize
+  #
+  if chgroup
+    chgid = chgroup.to_i
+    if chgid.to_s != chgroup
+      chgid = `id -u #{chgroup}`.to_i
+      if $?.to_i != 0
+        exit 1
+      end
+    end
+    Process::GID.change_privilege(chgid)
+  end
+
+  if chuser
+    chuid = chuser.to_i
+    if chuid.to_s != chuser
+      chuid = `id -u #{chuser}`.to_i
+      if $?.to_i != 0
+        exit 1
+      end
+    end
+    Process::UID.change_privilege(chuid)
+  end
+
+  trap :INT do
+    Fluent::Engine.stop
+  end
+
+  trap :TERM do
+    Fluent::Engine.stop
+  end
+
+  trap :HUP do
+    if log_file
+      $log.reopen(log_file, "a")
+    end
+  end
+
+  trap :USR1 do
+    $log.info "force flushing buffered events"
+    Fluent::Engine.flush!
+  end
+
+  if daemonize
+    exit!(0) if fork
+    Process.setsid
+    exit!(0) if fork
+    File.umask(0)
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen("/dev/null", "w")
+    STDERR.reopen("/dev/null", "w")
+    File.open(daemonize, "w") {|f|
+      f.write Process.pid.to_s
+    }
+  end
+
+
+  #
+  # run
+  #
+  $log.info "running fluent-#{Fluent::VERSION}"
+  Fluent::Engine.run
+
+rescue Fluent::ConfigError
+  $log.error "config error", :file=>config_path, :error=>$!.to_s
+  $log.debug_backtrace
+
+  # also STDOUT
+  if log_out != STDOUT
+    console = Fluent::Log.new(STDOUT, log_level).enable_debug
+    console.error "config error", :file=>config_path, :error=>$!.to_s
+    console.debug_backtrace
+  end
+
+  exit 1
+
+rescue
+  $log.error "unexpected error", :error=>$!.to_s
+  $log.error_backtrace
+
+  # also STDOUT
+  if log_out != STDOUT
+    console = Fluent::Log.new(STDOUT, log_level).enable_debug
+    console.error "unexpected error", :error=>$!.to_s
+    console.error_backtrace
+  end
+
+  exit 1
+end
+

data/lib/fluent/config.rb

@@ -0,0 +1,304 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class ConfigError < StandardError
+end
+
+class ConfigParseError < ConfigError
+end
+
+
+module Config
+  class Element < Hash
+    def initialize(name, arg, attrs, elements, used=[])
+      @name = name
+      @arg = arg
+      @elements = elements
+      super()
+      attrs.each {|k,v|
+        self[k] = v
+      }
+      @used = used
+    end
+
+    attr_accessor :name, :arg, :elements, :used
+
+    def add_element(name, arg='')
+      e = Element.new(name, arg, {}, [])
+      @elements << e
+      e
+    end
+
+    def +(o)
+      Element.new(@name.dup, @arg.dup, o.merge(self), @elements+o.elements, @used+o.used)
+    end
+
+    def has_key?(key)
+      @used << key
+      super
+    end
+
+    def [](key)
+      @used << key
+      super
+    end
+
+    def check_not_fetched(&block)
+      each_key {|key|
+        unless @used.include?(key)
+          block.call(key, self)
+        end
+      }
+      @elements.each {|e|
+        e.check_not_fetched(&block)
+      }
+    end
+
+    def to_s(nest = 0)
+      indent = "  "*nest
+      nindent = "  "*(nest+1)
+      out = ""
+      if @arg.empty?
+        out << "#{indent}<#{@name}>\n"
+      else
+        out << "#{indent}<#{@name} #{@name}>\n"
+      end
+      each_pair {|k,v|
+        out << "#{nindent}#{k} #{v}\n"
+      }
+      @elements.each {|e|
+        out << e.to_s(nest+1)
+      }
+      out << "#{indent}</#{@name}>\n"
+      out
+    end
+  end
+
+  def self.read(path)
+    parse(File.read(path), File.basename(path))
+  end
+
+  def self.parse(str, fname)
+    lines = str.split("\n")
+    i, attrs, elems = parse_element('end', lines, 0, fname)
+    Element.new('ROOT', '', attrs, elems)
+  end
+
+  def self.new(name='')
+    Element.new('', '', {}, [])
+  end
+
+  def self.size_value(str)
+    case str.to_s
+    when /([0-9]+)k/i
+      $~[1].to_i * 1024
+    when /([0-9]+)m/i
+      $~[1].to_i * (1024**2)
+    when /([0-9]+)g/i
+      $~[1].to_i * (1024**3)
+    when /([0-9]+)t/i
+      $~[1].to_i * (1024**4)
+    else
+      str.to_i
+    end
+  end
+
+  def self.time_value(str)
+    case str.to_s
+    when /([0-9]+)s/
+      $~[1].to_i
+    when /([0-9]+)m/
+      $~[1].to_i * 60
+    when /([0-9]+)h/
+      $~[1].to_i * 60*60
+    when /([0-9]+)d/
+      $~[1].to_i * 24*60*60
+    else
+      str.to_f
+    end
+  end
+
+  def self.bool_value(str)
+    case str.to_s
+    when 'true', 'yes'
+      true
+    when 'false', 'no'
+      false
+    else
+      nil
+    end
+  end
+
+  private
+  def self.parse_element(name, lines, i, fname)
+    attrs = {}
+    elems = []
+    while i < lines.length
+      line = lines[i]
+      line.lstrip!
+      line.gsub!(/\s*(?:\#.*)?$/,'')
+        if line.empty?
+          i += 1
+          next
+        elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
+          e_name = m[1]
+          e_arg = m[2] || ""
+          i, e_attrs, e_elems = parse_element(e_name, lines, i+1, fname)
+          elems << Element.new(e_name, e_arg, e_attrs, e_elems)
+        elsif line == "</#{name}>"
+          i += 1
+          break
+        elsif m = /^([a-zA-Z0-9_]+)\s*(.+)?$/.match(line)
+          attrs[m[1]] = m[2] || ""
+          i += 1
+          next
+        else
+          raise ConfigParseError, "parse error at #{fname}:#{i}"
+        end
+    end
+    return i, attrs, elems
+  end
+end
+
+
+module Configurable
+  def self.included(mod)
+    mod.extend(ClassMethods)
+  end
+
+  def initialize
+    self.class.config_defaults.each_pair {|name,defval|
+      varname = :"@#{name}"
+      instance_variable_set(varname, defval)
+    }
+  end
+
+  def configure(conf)
+    self.class.config_params.each_pair {|name,(block,opts)|
+      varname = :"@#{name}"
+      if val = conf[name.to_s]
+        val = self.instance_exec(val, opts, name, &block)
+        instance_variable_set(varname, val)
+      end
+      unless instance_variable_defined?(varname)
+        $log.error "config error in:\n#{conf}"
+        raise ConfigError, "'#{name}' parameter is required"
+      end
+    }
+  end
+
+  module ClassMethods
+    def config_param(name, *args, &block)
+      name = name.to_sym
+
+      opts = {}
+      args.each {|a|
+        if a.is_a?(Symbol)
+          opts[:type] = a
+        elsif a.is_a?(Hash)
+          opts.merge!(a)
+        else
+          raise ArgumentError, "wrong number of arguments (#{1+args.length} for #{block ? 2 : 3})"
+        end
+      }
+
+      type = opts[:type]
+      if block && type
+        raise ArgumentError, "wrong number of arguments (#{1+args.length} for #{block ? 2 : 3})"
+      end
+
+      block ||= case type
+          when :string, nil
+            Proc.new {|val| val }
+          when :integer
+            Proc.new {|val| val.to_i }
+          when :size
+            Proc.new {|val| Config.size_value(val) }
+          when :bool
+            Proc.new {|val| Config.bool_value(val) }
+          when :time
+            Proc.new {|val| Config.time_value(val) }
+          else
+            raise ArgumentError, "unknown config_param type `#{type}'"
+          end
+
+      params = config_params_set
+      params.delete(name)
+      params[name] = [block, opts]
+
+      if opts.has_key?(:default)
+        config_set_default(name, opts[:default])
+      end
+
+      attr_accessor name
+    end
+
+    def config_set_default(name, defval)
+      name = name.to_sym
+
+      defaults = config_defaults_set
+      defaults.delete(name)
+      defaults[name] = defval
+
+      nil
+    end
+
+    def config_params
+      singleton_value(:_config_params)
+    end
+
+    def config_defaults
+      singleton_value(:_config_defaults)
+    end
+
+    private
+    def config_params_set
+      singleton_value_set(:_config_params)
+    end
+
+    def config_defaults_set
+      singleton_value_set(:_config_defaults)
+    end
+
+    def singleton_value_set(name)
+      if methods(false).include?(name)
+        __send__(name)
+      else
+        val = {}
+        define_singleton_method(name) { val }
+        val
+      end
+    end
+
+    def singleton_value(name)
+      val = {}
+      ancestors.reverse_each {|c|
+        if c.methods(false).include?(name)
+          val.merge!(c.__send__(name))
+        end
+      }
+      val
+    end
+  end
+end
+
+
+end
+