fluentd 0.10.0

data/lib/fluent/parser.rb

@@ -0,0 +1,115 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class TextParser
+  TEMPLATES = {
+    'apache' => [/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, "%d/%b/%Y:%H:%M:%S %z"],
+    'syslog' => [/^(?<time>[^ ]* [^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, "%b %d %H:%M:%S"],
+  }
+
+  def self.register_template(name, regexp, time_format=nil)
+    TEMPLATES[name] = [regexp, time_format]
+  end
+
+  def self.get_template(name)
+    return *TEMPLATES[name]
+  end
+
+  def initialize
+    require 'time'  # Time.strptime, Time.parse
+    @regexp = nil
+    @time_format = nil
+  end
+
+  attr_accessor :regexp, :time_format
+
+  def use_template(name)
+    @regexp, @time_format = TextParser.get_template(name)
+    unless @regexp
+      raise ConfigError, "Unknown format template '#{name}'"
+    end
+  end
+
+  def configure(conf, required=true)
+    if format = conf['format']
+      if format[0] == ?/ && format[format.length-1] == ?/
+        # regexp
+        begin
+          @regexp = Regexp.new(format[1..-2])
+          if @regexp.named_captures.empty?
+            raise "No named captures"
+          end
+        rescue
+          raise ConfigError, "Invalid regexp '#{format[1..-2]}': #{$!}"
+        end
+
+      else
+        # template
+        use_template(format)
+      end
+    else
+      return nil if !required
+      raise ConfigError, "'format' parameter is required"
+    end
+
+    if time_format = conf['time_format']
+      unless @regexp.names.include?('time')
+        raise ConfigError, "'time_format' parameter is invalid when format doesn't have 'time' capture"
+      end
+      @time_format = time_format
+    end
+
+    return true
+  end
+
+  def parse(text)
+    m = @regexp.match(text)
+    unless m
+      $log.debug "pattern not match: #{text}"
+      # TODO?
+      return nil, nil
+    end
+
+    time = nil
+    record = {}
+
+    m.names.each {|name|
+      if value = m[name]
+        case name
+        when "time"
+          if @time_format
+            time = Time.strptime(value, @time_format).to_i
+          else
+            time = Time.parse(value).to_i
+          end
+        else
+          record[name] = value
+        end
+      end
+    }
+
+    time ||= Engine.now
+
+    return time, record
+  end
+end
+
+
+end

data/lib/fluent/plugin.rb

@@ -0,0 +1,145 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class PluginClass
+  def initialize
+    @input = {}
+    @output = {}
+    @buffer = {}
+  end
+
+  def register_input(type, klass)
+    register_impl('input', @input, type, klass)
+  end
+
+  def register_output(type, klass)
+    register_impl('output', @output, type, klass)
+  end
+
+  def register_buffer(type, klass)
+    register_impl('buffer', @buffer, type, klass)
+  end
+
+  def new_input(type)
+    new_impl('input', @input, type)
+  end
+
+  def new_output(type)
+    new_impl('output', @output, type)
+  end
+
+  def new_buffer(type)
+    new_impl('buffer', @buffer, type)
+  end
+
+  def load_plugins
+    dir = File.join(File.dirname(__FILE__), "plugin")
+    load_plugin_dir(dir)
+    load_gem_plugins
+  end
+
+  def load_plugin_dir(dir)
+    dir = File.expand_path(dir)
+    Dir.entries(dir).sort.each {|fname|
+      if fname =~ /\.rb$/
+        require File.join(dir, fname)
+      end
+    }
+    nil
+  end
+
+  private
+  def load_gem_plugins
+    return unless defined? Gem
+    plugins = Gem.find_files('fluent_plugin')
+
+    plugins.each {|plugin|
+      begin
+        load plugin
+      rescue ::Exception => e
+        msg = "#{plugin.inspect}: #{e.message} (#{e.class})"
+        $log.warn "Error loading Fluent plugin #{msg}"
+      end
+    }
+  end
+
+  def register_impl(name, map, type, klass)
+    map[type] = klass
+    $log.trace { "registered #{name} plugin '#{type}'" }
+    nil
+  end
+
+  def new_impl(name, map, type)
+    if klass = map[type]
+      return klass.new
+    end
+    try_load_plugin(name, type)
+    if klass = map[type]
+      return klass.new
+    end
+    raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search fluent-plugin' to find plugins"
+  end
+
+  def try_load_plugin(name, type)
+    case name
+    when 'input'
+      path = "fluent/plugin/in_#{type}"
+    when 'output'
+      path = "fluent/plugin/out_#{type}"
+    when 'buffer'
+      path = "fluent/plugin/buf_#{type}"
+    else
+      return
+    end
+
+    # prefer LOAD_PATH
+    files = $LOAD_PATH.map {|lp|
+      lpath = File.join(lp, "#{path}.rb")
+      File.exist?(lpath) ? lpath : nil
+    }.compact
+    unless files.empty?
+      # prefer newer version
+      require files.sort.last
+      return
+    end
+
+    # search gems
+    if defined?(::Gem) && ::Gem.respond_to?(:searcher)
+      #files = Gem.find_files(path).sort
+      specs = Gem.searcher.find_all(path)
+      specs = specs.sort_by {|spec| spec.version }
+
+      # prefer newer version
+      specs.reverse_each {|spec|
+        files = Gem.searcher.matching_files(spec, path)
+        unless files.empty?
+          require files.first
+          return
+        end
+      }
+    end
+  end
+end
+
+Plugin = PluginClass.new
+
+
+end
+

data/lib/fluent/plugin/buf_file.rb

@@ -0,0 +1,181 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class FileBufferChunk < BufferChunk
+  def initialize(key, path, mode="a+")
+    super(key)
+    @path = path
+    @file = File.open(@path, mode)
+    @size = @file.stat.size
+  end
+
+  def <<(data)
+    while true
+      n = @file.syswrite(data)
+      @size += n
+      if data.bytesize <= n
+        break
+      else
+        data = data[n..-1]
+        #data.slice!(0, n-1)
+      end
+    end
+    #@file.write(data)
+    #@size += data.bytesize
+    #@file.flush
+  end
+
+  def size
+    @size
+  end
+
+  def empty?
+    @size == 0
+  end
+
+  def close
+    @file.close
+    if @file.stat.size == 0
+      File.unlink(@path)
+    end
+  end
+
+  def purge
+    @file.close
+    File.unlink(@path) rescue nil  # TODO rescue?
+  end
+
+  def read
+    @file.pos = 0
+    @file.read
+  end
+
+  def open(&block)
+    @file.pos = 0
+    yield @file
+  end
+
+  attr_reader :path
+
+  def mv(path)
+    File.rename(@path, path)
+    @path = path
+  end
+end
+
+
+class FileBuffer < BasicBuffer
+  Plugin.register_buffer('file', self)
+
+  def initialize
+    require 'uri'
+    super
+  end
+
+  config_param :buffer_path, :string
+
+  def configure(conf)
+    super
+
+    if pos = @buffer_path.index('*')
+      @buffer_path_prefix = @buffer_path[0,pos]
+      @buffer_path_suffix = @buffer_path[pos+1..-1]
+    else
+      @buffer_path_prefix = @buffer_path+"."
+      @buffer_path_suffix = ".log"
+    end
+  end
+
+  def start
+    FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
+    super
+  end
+
+  PATH_MATCH = /^(.*)[\._](b|q)([0-9a-fA-F]{1,16})$/
+
+  def new_chunk(key)
+    tsuffix = Engine.now.to_i.to_s(16)
+    encoded_key = encode_key(key)
+    path = "#{@buffer_path_prefix}#{encoded_key}.b#{tsuffix}#{@buffer_path_suffix}"
+    FileBufferChunk.new(key, path)
+  end
+
+  def resume
+    maps = []
+    queues = []
+
+    Dir.glob("#{@buffer_path_prefix}*#{@buffer_path_suffix}") {|path|
+      match = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
+      if m = PATH_MATCH.match(match)
+        key = decode_key(m[1])
+        bq = m[2]
+        tsuffix = m[3].to_i(16)
+
+        if bq == 'b'
+          chunk = FileBufferChunk.new(key, path, "a+")
+          maps << [tsuffix, chunk]
+        elsif bq == 'q'
+          chunk = FileBufferChunk.new(key, path, "r")
+          queues << [tsuffix, chunk]
+        end
+      end
+    }
+
+    map = {}
+    maps.sort_by {|(tsuffix,chunk)|
+      tsuffix
+    }.each {|(tsuffix,chunk)|
+      map[chunk.key] = chunk
+    }
+
+    queue = queues.sort_by {|(tsuffix,chunk)|
+      tsuffix
+    }.map {|(tsuffix,chunk)|
+      chunk
+    }
+
+    return queue, map
+  end
+
+  def enqueue(chunk)
+    path = chunk.path
+    mp = path[@buffer_path_prefix.length..-(@buffer_path_suffix.length+1)]
+
+    m = PATH_MATCH.match(mp)
+    encoded_key = m ? m[1] : ""
+    tsuffix = Engine.now.to_i.to_s(16)
+
+    path = "#{@buffer_path_prefix}#{encoded_key}.q#{tsuffix}#{@buffer_path_suffix}"
+    chunk.mv(path)
+  end
+
+  protected
+  def encode_key(key)
+		URI.encode(key, /[^-_.a-zA-Z0-9]/n)
+  end
+
+  def decode_key(encoded_key)
+		URI.decode(encoded_key, /[^-_.a-zA-Z0-9]/n)
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/buf_memory.rb

@@ -0,0 +1,97 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class MemoryBufferChunk < BufferChunk
+  def initialize(key, data='')
+    @data = data
+    super(key)
+  end
+
+  def <<(data)
+    @data << data
+  end
+
+  def size
+    @data.bytesize
+  end
+
+  def close
+  end
+
+  def purge
+  end
+
+  def read
+    @data
+  end
+
+  def open(&block)
+    StringIO.open(@data, &block)
+  end
+
+  # optimize
+  def write_to(io)
+    io.write @data
+  end
+
+  # optimize
+  def msgpack_each(&block)
+    u = MessagePack::Unpacker.new(io)
+    u.feed_each(@data, &block)
+  end
+end
+
+
+class MemoryBuffer < BasicBuffer
+  Plugin.register_buffer('memory', self)
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def before_shutdown(out)
+    synchronize do
+      @map.each_key {|key|
+        push(key)
+      }
+      while pop(out)
+      end
+    end
+  end
+
+  def new_chunk(key)
+    MemoryBufferChunk.new(key)
+  end
+
+  def resume
+    return [], {}
+  end
+
+  def enqueue(chunk)
+  end
+end
+
+
+end
+