fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/event.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class EventStream
     include Enumerable
@@ -28,11 +27,11 @@ module Fluent
     end
 
     def to_msgpack_stream
-      out = $use_msgpack_5 ? MessagePack::Buffer.new : ''.force_encoding('ASCII-8BIT')
+      out = ''
       each {|time,record|
         [time,record].to_msgpack(out)
       }
-      out.to_s
+      out
     end
   end
 
@@ -67,7 +66,7 @@ module Fluent
     end
 
     def dup
-      entries = @entries.map { |entry| entry.dup } # @entries.map(:dup) doesn't work by ArgumentError
+      entries = @entries.map(:dup)
       ArrayEventStream.new(entries)
     end
 
@@ -132,53 +131,26 @@ module Fluent
     end
   end
 
-  if $use_msgpack_5
-
-    class MessagePackEventStream < EventStream
-      def initialize(data, cached_unpacker=nil)
-        @data = data
-      end
-
-      def repeatable?
-        true
-      end
-
-      def each(&block)
-        # TODO format check
-        unpacker = MessagePack::Unpacker.new
-        unpacker.feed_each(@data, &block)
-        nil
-      end
-
-      def to_msgpack_stream
-        @data
-      end
+  class MessagePackEventStream < EventStream
+    # Keep cached_unpacker argument for existence plugins
+    def initialize(data, cached_unpacker = nil)
+      @data = data
     end
 
-  else # for 0.4.x. Will be removed after 0.5.x is stable
-
-    class MessagePackEventStream < EventStream
-      def initialize(data, cached_unpacker=nil)
-        @data = data
-        @unpacker = cached_unpacker || MessagePack::Unpacker.new
-      end
-
-      def repeatable?
-        true
-      end
-
-      def each(&block)
-        @unpacker.reset
-        # TODO format check
-        @unpacker.feed_each(@data, &block)
-        nil
-      end
+    def repeatable?
+      true
+    end
 
-      def to_msgpack_stream
-        @data
-      end
+    def each(&block)
+      # TODO format check
+      unpacker = MessagePack::Unpacker.new
+      unpacker.feed_each(@data, &block)
+      nil
     end
 
+    def to_msgpack_stream
+      @data
+    end
   end
 end
 

data/lib/fluent/event_router.rb

@@ -0,0 +1,187 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  require 'fluent/match'
+
+  #
+  # EventRouter is responsible to route events to a collector.
+  #
+  # It has a list of MatchPattern and Collector pairs:
+  #
+  #  +----------------+     +-----------------+
+  #  |  MatchPattern  |     |    Collector    |
+  #  +----------------+     +-----------------+
+  #  |   access.**  ---------> type forward   |
+  #  |     logs.**  ---------> type copy      |
+  #  |  archive.**  ---------> type s3        |
+  #  +----------------+     +-----------------+
+  #
+  # EventRouter does:
+  #
+  # 1) receive an event at `#emit` methods
+  # 2) match the event's tag with the MatchPatterns
+  # 3) forward the event to the corresponding Collector
+  #
+  # Collector is either of Output, Filter or other EventRouter.
+  #
+  class EventRouter
+    def initialize(emit_error_handler, default_collector)
+      @match_rules = []
+      @match_cache = MatchCache.new
+      @default_collector = default_collector
+      @emit_error_handler = emit_error_handler || RaiseEmitErrorHandler.new
+      @chain = NullOutputChain.instance
+    end
+
+    # Agent implements EmitErrorHandler. See 'fluentd/agent.rb'
+    class RaiseEmitErrorHandler
+      def handle_emits_error(tag, es, error)
+        raise error
+      end
+    end
+
+    attr_accessor :default_collector
+    attr_accessor :emit_error_handler
+
+    class Rule
+      def initialize(pattern, collector)
+        patterns = pattern.split(/\s+/).map { |str| MatchPattern.create(str) }
+        @pattern = if patterns.length == 1
+                     patterns[0]
+                   else
+                     OrMatchPattern.new(patterns)
+                   end
+        @pattern_str = pattern
+        @collector = collector
+      end
+
+      def match?(tag)
+        @pattern.match(tag)
+      end
+
+      attr_reader :collector
+      attr_reader :patatern_str
+    end
+
+    # called by Agent to add new match pattern and collector
+    def add_rule(pattern, collector)
+      @match_rules << Rule.new(pattern, collector)
+    end
+
+    def emit(tag, time, record)
+      unless record.nil?
+        emit_stream(tag, OneEventStream.new(time, record))
+      end
+    end
+
+    def emit_array(tag, array)
+      emit_stream(tag, ArrayEventStream.new(array))
+    end
+
+    def emit_stream(tag, es)
+      match(tag).emit(tag, es, @chain)
+    rescue => e
+      @emit_error_handler.handle_emits_error(tag, es, e)
+    end
+
+    def match?(tag)
+      !!find(tag)
+    end
+
+    def match(tag)
+      collector = @match_cache.get(tag) {
+        c = find(tag) || @default_collector
+      }
+      collector
+    end
+
+    class MatchCache
+      MATCH_CACHE_SIZE = 1024
+
+      def initialize
+        super
+        @map = {}
+        @keys = []
+      end
+
+      def get(key)
+        if collector = @map[key]
+          return collector
+        end
+        collector = @map[key] = yield
+        if @keys.size >= MATCH_CACHE_SIZE
+          # expire the oldest key
+          @map.delete @keys.shift
+        end
+        @keys << key
+        collector
+      end
+    end
+
+    private
+
+    class Pipeline
+      def initialize
+        @filters = []
+        @output = nil
+      end
+
+      def add_filter(filter)
+        @filters << filter
+      end
+
+      def set_output(output)
+        @output = output
+      end
+
+      def emit(tag, es, chain)
+        processed = es
+        @filters.each { |filter|
+          processed = filter.filter_stream(tag, processed)
+        }
+        @output.emit(tag, processed, chain)
+      end
+    end
+
+    def find(tag)
+      pipeline = nil
+      @match_rules.each_with_index { |rule, i|
+        if rule.match?(tag)
+          if rule.collector.is_a?(Filter)
+            pipeline ||= Pipeline.new
+            pipeline.add_filter(rule.collector)
+          else
+            if pipeline
+              pipeline.set_output(rule.collector)
+            else
+              # Use Output directly when filter is not matched
+              pipeline = rule.collector
+            end
+            return pipeline
+          end
+        end
+      }
+
+      if pipeline
+        # filter is matched but no match
+        pipeline.set_output(@default_collector)
+        pipeline
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/fluent/filter.rb

@@ -0,0 +1,32 @@
+module Fluent
+  class Filter
+    include Configurable
+    include PluginId
+    include PluginLoggerMixin
+
+    def initialize
+      super
+    end
+
+    def configure(conf)
+      super
+    end
+
+    def start
+    end
+
+    def shutdown
+    end
+
+    def filter(tag, time, record)
+    end
+
+    def filter_stream(tag, es)
+      new_es = MultiEventStream.new
+      es.each { |time, record|
+        new_es.add(time, filter(tag, time, record))
+      }
+      new_es
+    end
+  end
+end

data/lib/fluent/formatter.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2014 Fluentd project
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,21 +13,10 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   require 'fluent/registry'
 
-  class Formatter
-    include Configurable
-
-    def configure(conf)
-      super
-    end
-
-    def format(tag, time, record)
-      raise NotImplementedError, "Implement this method in child class"
-    end
-  end
-
   module TextFormatter
     module HandleTagAndTimeMixin
       def self.included(klass)
@@ -40,7 +27,6 @@ module Fluent
           config_param :include_tag_key, :bool, :default => false
           config_param :tag_key, :string, :default => 'tag'
           config_param :localtime, :bool, :default => true
-          config_param :timezone, :string, :default => nil
         }
       end
 
@@ -50,7 +36,7 @@ module Fluent
         if conf['utc']
           @localtime = false
         end
-        @timef = TimeFormatter.new(@time_format, @localtime, @timezone)
+        @timef = TimeFormatter.new(@time_format, @localtime)
       end
 
       def filter_record(tag, time, record)
@@ -63,7 +49,8 @@ module Fluent
       end
     end
 
-    class OutFileFormatter < Formatter
+    class OutFileFormatter
+      include Configurable
       include HandleTagAndTimeMixin
 
       config_param :output_time, :bool, :default => true
@@ -76,6 +63,10 @@ module Fluent
         end
       end
 
+      def configure(conf)
+        super
+      end
+
       def format(tag, time, record)
         filter_record(tag, time, record)
         header = ''
@@ -85,12 +76,11 @@ module Fluent
       end
     end
 
-    module StructuredFormatMixin
-      def self.included(klass)
-        klass.instance_eval {
-          config_param :time_as_epoch, :bool, :default => false
-        }
-      end
+    class JSONFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+
+      config_param :time_as_epoch, :bool, :default => false
 
       def configure(conf)
         super
@@ -108,29 +98,12 @@ module Fluent
       def format(tag, time, record)
         filter_record(tag, time, record)
         record[@time_key] = time if @time_as_epoch
-        format_record(record)
-      end
-    end
-
-    class JSONFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
         "#{Yajl.dump(record)}\n"
       end
     end
 
-    class MessagePackFormatter < Formatter
-      include HandleTagAndTimeMixin
-      include StructuredFormatMixin
-
-      def format_record(record)
-        record.to_msgpack
-      end
-    end
-
-    class LabeledTSVFormatter < Formatter
+    class LabeledTSVFormatter
+      include Configurable
       include HandleTagAndTimeMixin
 
       config_param :delimiter, :string, :default => "\t"
@@ -147,37 +120,9 @@ module Fluent
       end
     end
 
-    class CsvFormatter < Formatter
-      include HandleTagAndTimeMixin
-
-      config_param :delimiter, :default => ',' do |val|
-        ['\t', 'TAB'].include?(val) ? "\t" : val
-      end
-      config_param :force_quotes, :bool, :default => true
-      config_param :fields, :default => [] do |val|
-        val.split(',').map do |f|
-          f.strip!
-          f.size > 0 ? f : nil
-        end.compact
-      end
-
-      def initialize
-        super
-        require 'csv'
-      end
-
-      def format(tag, time, record)
-        filter_record(tag, time, record)
-        row = @fields.inject([]) do |memo, key|
-            memo << record[key]
-            memo
-        end
-        CSV.generate_line(row, :col_sep => @delimiter,
-                          :force_quotes => @force_quotes)
-      end
-    end
+    class SingleValueFormatter
+      include Configurable
 
-    class SingleValueFormatter < Formatter
       config_param :message_key, :string, :default => 'message'
       config_param :add_newline, :bool, :default => true
 
@@ -188,58 +133,41 @@ module Fluent
       end
     end
 
-    class ProcWrappedFormatter < Formatter
-      def initialize(proc)
-        @proc = proc
-      end
-
-      def configure(conf)
-      end
-
-      def format(tag, time, record)
-        @proc.call(tag, time, record)
-      end
-    end
-
     TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
     {
       'out_file' => Proc.new { OutFileFormatter.new },
       'json' => Proc.new { JSONFormatter.new },
-      'msgpack' => Proc.new { MessagePackFormatter.new },
       'ltsv' => Proc.new { LabeledTSVFormatter.new },
-      'csv' => Proc.new { CsvFormatter.new },
       'single_value' => Proc.new { SingleValueFormatter.new },
     }.each { |name, factory|
       TEMPLATE_REGISTRY.register(name, factory)
     }
 
     def self.register_template(name, factory_or_proc)
-      factory = if factory_or_proc.is_a?(Class) # XXXFormatter
-                  Proc.new { factory_or_proc.new }
-                elsif factory_or_proc.arity == 3 # Proc.new { |tag, time, record| }
-                  Proc.new { ProcWrappedFormatter.new(factory_or_proc) }
-                else # Proc.new { XXXFormatter.new }
+      factory = if factory_or_proc.arity == 3
+                  Proc.new { factory_or_proc }
+                else
                   factory_or_proc
                 end
 
       TEMPLATE_REGISTRY.register(name, factory)
     end
 
-    def self.lookup(format)
-      TEMPLATE_REGISTRY.lookup(format).call
-    end
-
-    # Keep backward-compatibility
     def self.create(conf)
       format = conf['format']
       if format.nil?
         raise ConfigError, "'format' parameter is required"
       end
 
-      formatter = lookup(format)
-      if formatter.respond_to?(:configure)
-        formatter.configure(conf)
+      # built-in template
+      begin
+        factory = TEMPLATE_REGISTRY.lookup(format)
+      rescue ConfigError => e
+        raise ConfigError, "unknown format: '#{format}'"
       end
+
+      formatter = factory.call
+      formatter.configure(conf)
       formatter
     end
   end