fluentd 0.10.0

data/lib/fluent/engine.rb

@@ -0,0 +1,224 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class EngineClass
+  def initialize
+    @matches = []
+    @sources = []
+    @match_cache = {}
+  end
+
+  def init
+    BasicSocket.do_not_reverse_lookup = true
+    Plugin.load_plugins
+    if defined?(Encoding)
+      Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
+      Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
+    end
+    self
+  end
+
+  def read_config(path)
+    $log.info "reading config file", :path=>path
+    conf = Config.read(path)
+    configure(conf)
+    conf.check_not_fetched {|key,e|
+      $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
+    }
+  end
+
+  def configure(conf)
+    conf.elements.select {|e|
+      e.name == 'source'
+    }.each {|e|
+      type = e['type']
+      unless type
+        raise ConfigError, "Missing 'type' parameter on <source> directive"
+      end
+      $log.info "adding source type=#{type.dump}"
+
+      input = Plugin.new_input(type)
+      input.configure(e)
+
+      @sources << input
+    }
+
+    conf.elements.select {|e|
+      e.name == 'match'
+    }.each {|e|
+      type = e['type']
+      pattern = e.arg
+      unless type
+        raise ConfigError, "Missing 'type' parameter on <match #{e.arg}> directive"
+      end
+      $log.info "adding match", :pattern=>pattern, :type=>type
+
+      output = Plugin.new_output(type)
+      output.configure(e)
+
+      match = Match.new(pattern, output)
+      @matches << match
+    }
+  end
+
+  def load_plugin_dir(dir)
+    Plugin.load_plugin_dir(dir)
+  end
+
+  def emit(tag, time, record)
+    emit_stream tag, OneEventStream.new(time, record)
+  end
+
+  def emit_array(tag, array)
+    emit_stream tag, ArrayEventStream.new(array)
+  end
+
+  def emit_stream(tag, es)
+    match = @match_cache[tag]
+    unless match
+      match = @matches.find {|m| m.match(tag) } || NoMatchMatch.new
+      if @match_cache.size < 1024  # TODO size limit
+        @match_cache[tag] = match
+      end
+    end
+    match.emit(tag, es)
+  rescue
+    $log.warn "emit transaction faild ", :error=>$!.to_s
+    $log.warn_backtrace
+    raise
+  end
+
+  def match?(tag)
+    @matches.find {|m| m.match(tag) }
+  end
+
+  def flush!
+    flush_recursive(@matches)
+  end
+
+  def now
+    # TODO thread update
+    Time.now.to_i
+  end
+
+  def run
+    start
+
+    if match?($log.tag)
+      $log.enable_event
+    end
+
+    # for empty loop
+    Coolio::Loop.default.attach Coolio::TimerWatcher.new(1, true)
+    # TODO attach async watch for thread pool
+    Coolio::Loop.default.run
+
+    shutdown
+  end
+
+  def stop
+    $log.info "shutting down fluentd"
+    Coolio::Loop.default.stop
+    nil
+  end
+
+  private
+  def start
+    @matches.each {|m|
+      m.start
+    }
+    @sources.each {|s|
+      s.start
+    }
+  end
+
+  def shutdown
+    @matches.each {|m|
+      m.shutdown rescue nil
+    }
+    @sources.each {|s|
+      s.shutdown rescue nil
+    }
+  end
+
+  def flush_recursive(array)
+    array.each {|m|
+      begin
+        if m.is_a?(Match)
+          m = m.output
+        end
+        if m.is_a?(BufferedOutput)
+          m.try_flush
+        elsif m.is_a?(MultiOutput)
+          flush_recursive(m.outputs)
+        end
+      rescue
+        $log.debug "error while force flushing", :error=>$!.to_s
+        $log.debug_backtrace
+      end
+    }
+  end
+
+  class NoMatchMatch
+    def emit(tag, es)
+      $log.on_trace { $log.trace "no pattern matched", :tag=>tag }
+    end
+
+    def start
+    end
+
+    def shutdown
+    end
+
+    def match(tag)
+      false
+    end
+  end
+end
+
+Engine = EngineClass.new
+
+
+module Test
+  @@test = false
+
+  def test?
+    @@test
+  end
+
+  def self.setup
+    @@test = true
+
+    Fluent.__send__(:remove_const, :Engine)
+    engine = Fluent.const_set(:Engine, EngineClass.new).init
+
+    engine.define_singleton_method(:now=) {|n|
+      @now = n.to_i
+    }
+    engine.define_singleton_method(:now) {
+      @now || super()
+    }
+
+    nil
+  end
+end
+
+end
+

data/lib/fluent/env.rb

@@ -0,0 +1,6 @@
+module Fluent
+DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
+DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
+DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+DEFAULT_LISTEN_PORT = 24224
+end

data/lib/fluent/event.rb

@@ -0,0 +1,159 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class EventStream
+  include Enumerable
+
+  def repeatable?
+    false
+  end
+
+  #def each(&block)
+  #end
+
+  def to_msgpack_stream
+    out = ''
+    each {|record|
+      record.to_msgpack(out)
+    }
+    out
+  end
+end
+
+
+class OneEventStream < EventStream
+  def initialize(time, record)
+    @time = time
+    @record = record
+  end
+
+  def repeatable?
+    true
+  end
+
+  def each(&block)
+    block.call(@time, @record)
+    nil
+  end
+end
+
+
+class ArrayEventStream < EventStream
+  def initialize(entries)
+    @entries = entries
+  end
+
+  def repeatable?
+    true
+  end
+
+  def empty?
+    @time_array.empty?
+  end
+
+  def each(&block)
+    @entries.each(&block)
+    nil
+  end
+
+  #attr_reader :entries
+  #
+  #def to_a
+  #  @entries
+  #end
+end
+
+
+class MultiEventStream < EventStream
+  def initialize
+    @time_array = []
+    @record_array = []
+  end
+
+  def add(time, record)
+    @time_array << time
+    @record_array << record
+  end
+
+  def repeatable?
+    true
+  end
+
+  def empty?
+    @time_array.empty?
+  end
+
+  def each(&block)
+    time_array = @time_array
+    record_array = @record_array
+    for i in 0..time_array.length-1
+      block.call(time_array[i], record_array[i])
+    end
+    nil
+  end
+end
+
+
+class MessagePackEventStream < EventStream
+  def initialize(data, cached_unpacker=nil)
+    @data = data
+    @unpacker = cached_unpacker || MessagePack::Unpacker.new
+  end
+
+  def repeatable?
+    true
+  end
+
+  def each(&block)
+    @unpacker.reset
+    # TODO format check
+    @unpacker.feed_each(@data, &block)
+    nil
+  end
+
+  def to_msgpack_stream
+    @data
+  end
+end
+
+
+#class IoEventStream < EventStream
+#  def initialize(io, num)
+#    @io = io
+#    @num = num
+#    @u = MessagePack::Unpacker.new(@io)
+#  end
+#
+#  def repeatable?
+#    false
+#  end
+#
+#  def each(&block)
+#    return nil if @num == 0
+#    @u.each {|obj|
+#      block.call(obj[0], obj[1])
+#      break if @array.size >= @num
+#    }
+#  end
+#end
+
+
+end
+

data/lib/fluent/input.rb

@@ -0,0 +1,41 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class Input
+  include Configurable
+
+	def initialize
+    super
+	end
+
+	def configure(conf)
+    super
+	end
+
+	def start
+	end
+
+	def shutdown
+	end
+end
+
+
+end
+

data/lib/fluent/load.rb

@@ -0,0 +1,23 @@
+require 'thread'
+require 'socket'
+require 'time'
+require 'monitor'
+require 'stringio'
+require 'fileutils'
+require 'json'
+require 'msgpack'
+require 'cool.io'
+require 'cool.io/eventmachine'
+require 'fluent/env'
+require 'fluent/version'
+require 'fluent/log'
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/mixin'
+require 'fluent/plugin'
+require 'fluent/parser'
+require 'fluent/event'
+require 'fluent/buffer'
+require 'fluent/input'
+require 'fluent/output'
+require 'fluent/match'