fluentd 0.12.0.pre.2 → 0.12.0.pre.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5728345e3e45be5f8327ea9cfe0268b958c78d92
-  data.tar.gz: 453abbcdf2c21a2ff97fb6da4b9e4468c0519de1
+  metadata.gz: c5a6a2cf3f13289f87a554bfafb531593977ffcd
+  data.tar.gz: bbb612c523e612b177980433bcc86cb7a6571d90
 SHA512:
-  metadata.gz: 0c2878ba5c1d0d45e12d75eea0ed5700ec513b2dde96ecbf8efce4d6e2a0d81eeb9b4a20b7d43b3a053c9c84c24caf3a794fdc1af750b37d7d8e4e3feaf4bdcd
-  data.tar.gz: 68f2edd339b18575244d1643df2144f71b4dc2d48dac9355071fbc4a0ceb141bddcdee67189d84d1bf451b8445b83fe50bc545ecc6c866593012702c57b1bb24
+  metadata.gz: ea0f0eeb1718e00caef76ae713101292daa2019cab3af19a2e26c4f2d2c74920026757e729b5e57b8f60f98f366035211e206a8f9d97849c3f0bfc0fad8ccdc5
+  data.tar.gz: b44314b0bcd226f4666b1079b297ad6f67a646b6be0f22a4e855e5176caac17cdf7b792a2b760bd0ba05184146cacdacc121f643661f41c41e25ee7d9632b5e3

data/example/v0_12_filter.conf

@@ -0,0 +1,78 @@
+# An example config to use filter plugins.
+# THIS FEATURE IS SUPPORTED FOR v0.12 AND ABOVE.
+
+# in_forward to generate events to be tested.
+# You can send an arbitrary event with an arbitrary tag.
+# For example, the following command creates the event
+# {"message":"hello world"} with tag = foo
+#
+# $ echo '{"message":"hello world"}' | fluent-cat foo
+
+<source>
+  type forward
+  port 24224
+</source>
+
+# For all events with the tag "foo", filter it out
+# UNLESS the value of the "message" field matches /keep this/
+#
+# - {"message":"keep this please"} is kept.
+# - {"message":"Do not keep"} is filtered out.
+# - {"messag22":"keep this please"} is filtered out.
+
+<filter foo>
+  type grep
+  regexp1  message keep this
+</filter>
+
+# Matches the events that was kept by the above filter
+<match foo>
+  type stdout
+</match>
+
+# For all events with the tag "bar", add the machine's hostname with
+# the key "hostname" BEFORE forwarding to another instance of Fluentd
+# at 123.4.2.4:24224.
+
+<filter bar>
+  type record_transformer
+  <record>
+    hostname ${hostname}
+  </record>
+</filter>
+
+# By the time it is getting matched here, the event has
+# the "hostname" field.
+<match bar>
+  type forward
+  <server>
+    host 123.4.2.4
+    port 24225
+  </server>
+</match>
+
+# Composing two filters. For all events with the tag foo.bar,
+# 
+# 1. The first filter filters out all events that has the field "hello"
+# 2. Then, for those events, we downcase the value of the "name" field.
+# 
+# - {"name":"SADA", "hello":100} gets filtered out
+# - {"name":"SADA"} becomes {"name":"sada"}
+# - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
+
+<filter foo.bar>
+  type grep
+  exclude1 hello .
+</filter>
+
+<filter foo.bar>
+  type record_transformer
+  enable_ruby true
+  <record>
+    name ${name.downcase}
+  </record>
+</filter>
+
+<match foo.bar>
+  type stdout
+</match>

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.has_rdoc = false
 
-  gem.required_ruby_version = '>= 1.9.2'
+  gem.required_ruby_version = '>= 1.9.3'
 
   gem.add_runtime_dependency("msgpack", [">= 0.5.4", "< 0.6.0"])
   gem.add_runtime_dependency("json", [">= 1.4.3"])
@@ -35,4 +35,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency("rr", [">= 1.0.0"])
   gem.add_development_dependency("timecop", [">= 0.3.0"])
   gem.add_development_dependency("test-unit", ["~> 3.0.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
 end

data/lib/fluent/agent.rb

@@ -37,7 +37,7 @@ module Fluent
       @started_filters = []
 
       @log = Engine.log
-      @event_router = EventRouter.new(self, NoMatchMatch.new(log))
+      @event_router = EventRouter.new(NoMatchMatch.new(log), self)
       @error_collector = nil
     end
 
@@ -137,6 +137,7 @@ module Fluent
       log.info "adding filter#{@context.nil? ? '' : " in #{@context}"}", pattern: pattern, type: type
 
       filter = Plugin.new_filter(type)
+      filter.router = @event_router
       filter.configure(conf)
       @filters << filter
       @event_router.add_rule(pattern, filter)

data/lib/fluent/buffer.rb

@@ -237,11 +237,15 @@ module Fluent
 
     def total_queued_chunk_size
       total = 0
-      @map.each_value {|c|
-        total += c.size
-      }
-      @queue.each {|c|
-        total += c.size
+      synchronize {
+        @map.each_value {|c|
+          total += c.size
+        }
+        @queue.synchronize {
+          @queue.each {|c|
+            total += c.size
+          }
+        }
       }
       total
     end

data/lib/fluent/command/fluentd.rb

@@ -50,6 +50,10 @@ op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
   opts[:daemonize] = s
 }
 
+op.on('--no-supervisor', "run without fluent supervisor") {
+  opts[:supervise] = false
+}
+
 op.on('--user USER', "change user") {|s|
   opts[:chuser] = s
 }

data/lib/fluent/config/basic_parser.rb

@@ -27,6 +27,7 @@ module Fluent
 
       LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
+      ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
 
       module ClassMethods

data/lib/fluent/config/configure_proxy.rb

@@ -84,13 +84,13 @@ module Fluent
           elsif a.is_a?(Hash)
             opts.merge!(a)
           else
-            raise ArgumentError, "wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
+            raise ArgumentError, "#{self.name}: wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
           end
         }
 
         type = opts[:type]
         if block && type
-          raise ArgumentError, "both of block and type cannot be specified"
+          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
         end
 
         begin
@@ -98,7 +98,7 @@ module Fluent
           block ||= Configurable.lookup_type(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
         end
 
         if opts.has_key?(:default)
@@ -110,7 +110,7 @@ module Fluent
 
       def config_argument(name, *args, &block)
         if @argument
-          raise ArgumentError, "config_argument called twice"
+          raise ArgumentError, "#{self.name}: config_argument called twice"
         end
         name, block, opts = parameter_configuration(name, *args, &block)
 
@@ -130,7 +130,7 @@ module Fluent
         name = name.to_sym
 
         if @defaults.has_key?(name)
-          raise ArgumentError, "default value specified twice for #{name}"
+          raise ArgumentError, "#{self.name}: default value specified twice for #{name}"
         end
 
         @defaults[name] = defval
@@ -139,13 +139,13 @@ module Fluent
 
       def config_section(name, *args, &block)
         unless block_given?
-          raise ArgumentError, "config_section requires block parameter"
+          raise ArgumentError, "#{self.name}: config_section requires block parameter"
         end
         name = name.to_sym
 
         opts = {}
         unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
-          raise ArgumentError, "unknown config_section arguments: #{args.inspect}"
+          raise ArgumentError, "#{self.name}: unknown config_section arguments: #{args.inspect}"
         end
 
         sub_proxy = ConfigureProxy.new(name, *args)

data/lib/fluent/config/types.rb

@@ -49,6 +49,7 @@ module Fluent
     end
 
     def self.bool_value(str)
+      return nil if str.nil?
       case str.to_s
       when 'true', 'yes'
         true

data/lib/fluent/config/v1_parser.rb

@@ -79,7 +79,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_nonquoted_string(/(?:#{SPACING}|\>)/)
+            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"

data/lib/fluent/engine.rb

@@ -207,29 +207,4 @@ module Fluent
   end
 
   Engine = EngineClass.new
-
-  module Test
-    @@test = false
-
-    def test?
-      @@test
-    end
-
-    def self.setup
-      @@test = true
-
-      Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init
-
-      engine.define_singleton_method(:now=) {|n|
-        @now = n.to_i
-      }
-      engine.define_singleton_method(:now) {
-        @now || super()
-      }
-
-      nil
-    end
-  end
 end
-

data/lib/fluent/env.rb

@@ -20,4 +20,5 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_LISTEN_PORT = 24224
   DEFAULT_FILE_PERMISSION = 0644
+  DEFAULT_DIR_PERMISSION = 0755
 end

data/lib/fluent/event_router.rb

@@ -38,7 +38,7 @@ module Fluent
   # Collector is either of Output, Filter or other EventRouter.
   #
   class EventRouter
-    def initialize(emit_error_handler, default_collector)
+    def initialize(default_collector, emit_error_handler)
       @match_rules = []
       @match_cache = MatchCache.new
       @default_collector = default_collector
@@ -73,7 +73,7 @@ module Fluent
       end
 
       attr_reader :collector
-      attr_reader :patatern_str
+      attr_reader :pattern_str
     end
 
     # called by Agent to add new match pattern and collector
@@ -97,6 +97,10 @@ module Fluent
       @emit_error_handler.handle_emits_error(tag, es, e)
     end
 
+    def emit_error_event(tag, time, record, error)
+      @emit_error_handler.emit_error_event(tag, time, record, error)
+    end
+
     def match?(tag)
       !!find(tag)
     end

data/lib/fluent/filter.rb

@@ -4,12 +4,19 @@ module Fluent
     include PluginId
     include PluginLoggerMixin
 
+    attr_accessor :router
+
     def initialize
       super
     end
 
     def configure(conf)
       super
+
+      if label_name = conf['@label']
+        label = Engine.root_agent.find_label(label_name)
+        @router = label.event_router
+      end
     end
 
     def start
@@ -24,7 +31,11 @@ module Fluent
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
       es.each { |time, record|
-        new_es.add(time, filter(tag, time, record))
+        begin
+          new_es.add(time, filter(tag, time, record))
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
       }
       new_es
     end

data/lib/fluent/formatter.rb

@@ -77,11 +77,12 @@ module Fluent
       end
     end
 
-    class JSONFormatter
-      include Configurable
-      include HandleTagAndTimeMixin
-
-      config_param :time_as_epoch, :bool, :default => false
+    module StructuredFormatMixin
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :time_as_epoch, :bool, :default => false
+        }
+      end
 
       def configure(conf)
         super
@@ -99,10 +100,30 @@ module Fluent
       def format(tag, time, record)
         filter_record(tag, time, record)
         record[@time_key] = time if @time_as_epoch
+        format_record(record)
+      end
+    end
+
+    class JSONFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
         "#{Yajl.dump(record)}\n"
       end
     end
 
+    class MessagePackFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
+        record.to_msgpack
+      end
+    end
+
     class LabeledTSVFormatter
       include Configurable
       include HandleTagAndTimeMixin
@@ -121,6 +142,37 @@ module Fluent
       end
     end
 
+    class CsvFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+
+      config_param :delimiter, :default => ',' do |val|
+        ['\t', 'TAB'].include?(val) ? "\t" : val
+      end
+      config_param :force_quotes, :bool, :default => true
+      config_param :fields, :default => [] do |val|
+        val.split(',').map do |f|
+          f.strip!
+          f.size > 0 ? f : nil
+        end.compact
+      end
+
+      def initialize
+        super
+        require 'csv'
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        row = @fields.inject([]) do |memo, key|
+            memo << record[key]
+            memo
+        end
+        CSV.generate_line(row, :col_sep => @delimiter,
+                          :force_quotes => @force_quotes)
+      end
+    end
+
     class SingleValueFormatter
       include Configurable
 
@@ -134,41 +186,58 @@ module Fluent
       end
     end
 
+    class ProcWrappedFormatter
+      def initialize(proc)
+        @proc = proc
+      end
+
+      def configure(conf)
+      end
+
+      def format(tag, time, record)
+        @proc.call(tag, time, record)
+      end
+    end
+
     TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
     {
       'out_file' => Proc.new { OutFileFormatter.new },
       'json' => Proc.new { JSONFormatter.new },
+      'msgpack' => Proc.new { MessagePackFormatter.new },
       'ltsv' => Proc.new { LabeledTSVFormatter.new },
+      'csv' => Proc.new { CsvFormatter.new },
       'single_value' => Proc.new { SingleValueFormatter.new },
     }.each { |name, factory|
       TEMPLATE_REGISTRY.register(name, factory)
     }
 
     def self.register_template(name, factory_or_proc)
-      factory = if factory_or_proc.arity == 3
-                  Proc.new { factory_or_proc }
-                else
+      factory = if factory_or_proc.is_a?(Class) # XXXFormatter
+                  Proc.new { factory_or_proc.new }
+                elsif factory_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+                  Proc.new { ProcWrappedFormatter.new(factory_or_proc) }
+                else # Proc.new { XXXFormatter.new }
                   factory_or_proc
                 end
 
       TEMPLATE_REGISTRY.register(name, factory)
     end
 
+    def self.lookup(format)
+      TEMPLATE_REGISTRY.lookup(format).call
+    end
+
+    # Keep backward-compatibility
     def self.create(conf)
       format = conf['format']
       if format.nil?
         raise ConfigError, "'format' parameter is required"
       end
 
-      # built-in template
-      begin
-        factory = TEMPLATE_REGISTRY.lookup(format)
-      rescue ConfigError => e
-        raise ConfigError, "unknown format: '#{format}'"
+      formatter = lookup(format)
+      if formatter.respond_to?(:configure)
+        formatter.configure(conf)
       end
-
-      formatter = factory.call
-      formatter.configure(conf)
       formatter
     end
   end