fluentd 0.10.57 → 0.10.58

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2d8292e66a20b544c4f2f96165479f81573e7ead
-  data.tar.gz: 8ac541d791d8d50454e4050f6c4242da7b85a884
+  metadata.gz: 3a62e7c7181c7f247c4a76b42638d0a6b4195d8a
+  data.tar.gz: 12ccff6793701f59c0b67c17d66804a3d25ac818
 SHA512:
-  metadata.gz: b14d9f92215ea493b61825babac41710bef940e0d07f7a4955476f0e831257a3738ea244f84fcf1271011ced7fc0d4a5db20cd2e6759c488110f12ec94dc75d4
-  data.tar.gz: 480aaf1a575d9c0d1153f24a2d1c2c92fa1ff0308664790b9a023c8b55d5bbaabd2b8ed29ec6209f36ebd899b4a087d40e57cff5bc742150f03ea77cd73391a4
+  metadata.gz: 81a1d182345423ecbb41a94351ccb6666f449126128718187345b9e62c187a3f8ed50fa56cfdee19cd5817ed37620471d10d871ca339cd345ef2111e1da76e29
+  data.tar.gz: 7a61d7bd4b80b28373fb1ed7824add416a085d77a7ace5a229bb34ebb6965fadf7fbcbd5ff4d61ee89ce6166b05a17011b619d5e7d9e9e42ddade2d6828ab8f0

data/.travis.yml

@@ -14,6 +14,7 @@ os:
 branches:
   only:
     - master
+    - v0.10
 
 gemfile:
   - Gemfile

data/ChangeLog

@@ -1,3 +1,21 @@
+Release 0.10.58 - 2014/12/14
+
+* parser/formatter: Add base class and Plugin.new_xxx/Plugin.register_xxx APIs
+* parser: Fix blank column handling of TSVParser
+* formatter: Add new CSV formatter
+* formatter: Add msgpack format to built-in Formatter to dump records
+* input: Add in_dummy plugin
+* in_http: Add respond_with_empty_img parameter to return empty gif image
+* in_http: Add cors_allow_origins parameter to support CORS request
+* config: Add self.name to configure_proxy error message
+* config: Fix system config using double memory
+* config: Fix v1 config to support multiple tag match
+* config: Fix Config.bool_value regression for nil value
+* buffer: Prevent an exception with large num_retries
+* out_file: Don't create world writable directory in daemon mode
+* add router.emit and router= for compatibility with v0.12
+* Use router instead of Engine.emit
+
 Release 0.10.57 - 2014/11/12
 
 * buffer: Make total_queued_chunk_size thread-safe to avoid race condition with in_monitor_agent

data/fluent.conf

@@ -1,26 +1,31 @@
+# In v1 configuration, type and id are @ prefix parameters.
+# @type and @id are recommended. type and id are still available for backward compatibility
 
 ## built-in TCP input
 ## $ echo <json> | fluent-cat <tag>
 <source>
-  type forward
+  @type forward
+  @id forward_input
 </source>
 
 ## built-in UNIX socket input
 #<source>
-#  type unix
+#  @type unix
 #</source>
 
 # HTTP input
 # http://localhost:8888/<tag>?json=<json>
 <source>
-  type http
+  @type http
+  @id http_input
+
   port 8888
 </source>
 
 ## File input
 ## read apache logs with tag=apache.access
 #<source>
-#  type tail
+#  @type tail
 #  format apache
 #  path /var/log/httpd-access.log
 #  tag apache.access
@@ -31,32 +36,38 @@
 # http://localhost:24220/api/plugins?type=TYPE
 # http://localhost:24220/api/plugins?tag=MYTAG
 <source>
-  type monitor_agent
+  @type monitor_agent
+  @id monitor_agent_input
+
   port 24220
 </source>
 
 # Listen DRb for debug
 <source>
-  type debug_agent
+  @type debug_agent
+  @id debug_agent_input
+
   bind 127.0.0.1
   port 24230
 </source>
 
-
 ## match tag=apache.access and write to file
 #<match apache.access>
-#  type file
+#  @type file
 #  path /var/log/fluent/access
 #</match>
 
 ## match tag=debug.** and dump to console
 <match debug.**>
-  type stdout
+  @type stdout
+  @id stdout_output
 </match>
 
 # match tag=system.** and forward to another fluent server
 <match system.**>
-  type forward
+  @type forward
+  @id forward_output
+
   <server>
     host 192.168.0.11
   </server>
@@ -69,9 +80,9 @@
 
 ## match tag=myapp.** and forward and write to file
 #<match myapp.**>
-#  type copy
+#  @type copy
 #  <store>
-#    type forward
+#    @type forward
 #    buffer_type file
 #    buffer_path /var/log/fluent/myapp-forward
 #    retry_limit 50
@@ -81,20 +92,19 @@
 #    </server>
 #  </store>
 #  <store>
-#    type file
+#    @type file
 #    path /var/log/fluent/myapp
 #  </store>
 #</match>
 
 ## match fluent's internal events
 #<match fluent.**>
-#  type null
+#  @type null
 #</match>
 
 ## match not matched logs and write to file
 #<match **>
-#  type file
+#  @type file
 #  path /var/log/fluent/else
 #  compress gz
 #</match>
-

data/fluentd.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.has_rdoc = false
 
-  gem.required_ruby_version = '>= 1.9.2'
+  gem.required_ruby_version = '>= 1.9.3'
 
   gem.add_runtime_dependency("msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"])
   gem.add_runtime_dependency("json", [">= 1.4.3"])

data/lib/fluent/config.rb

@@ -44,17 +44,10 @@ module Fluent
 
   module PluginId
     def configure(conf)
-      @id = conf['id']
+      @id = conf['@id'] || conf['id']
       super
     end
 
-    def require_id
-      unless @id
-        raise ConfigError, "'id' parameter is required"
-      end
-      @id
-    end
-
     def plugin_id
       @id ? @id : "object:#{object_id.to_s(16)}"
     end

data/lib/fluent/config/basic_parser.rb

@@ -26,6 +26,7 @@ module Fluent
 
       LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
+      ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
       SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
 
       module ClassMethods

data/lib/fluent/config/configure_proxy.rb

@@ -68,13 +68,13 @@ module Fluent
           elsif a.is_a?(Hash)
             opts.merge!(a)
           else
-            raise ArgumentError, "wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
+            raise ArgumentError, "#{self.name}: wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
           end
         }
 
         type = opts[:type]
         if block && type
-          raise ArgumentError, "both of block and type cannot be specified"
+          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
         end
 
         begin
@@ -82,7 +82,7 @@ module Fluent
           block ||= Configurable.lookup_type(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "unknown config_argument type `#{type}'"
+          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
         end
 
         if opts.has_key?(:default)
@@ -94,7 +94,7 @@ module Fluent
 
       def config_argument(name, *args, &block)
         if @argument
-          raise ArgumentError, "config_argument called twice"
+          raise ArgumentError, "#{self.name}: config_argument called twice"
         end
         name, block, opts = parameter_configuration(name, *args, &block)
 
@@ -114,7 +114,7 @@ module Fluent
         name = name.to_sym
 
         if @defaults.has_key?(name)
-          raise ArgumentError, "default value specified twice for #{name}"
+          raise ArgumentError, "#{self.name}: default value specified twice for #{name}"
         end
 
         @defaults[name] = defval
@@ -123,13 +123,13 @@ module Fluent
 
       def config_section(name, *args, &block)
         unless block_given?
-          raise ArgumentError, "config_section requires block parameter"
+          raise ArgumentError, "#{self.name}: config_section requires block parameter"
         end
         name = name.to_sym
 
         opts = {}
         unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
-          raise ArgumentError, "unknown config_section arguments: #{args.inspect}"
+          raise ArgumentError, "#{self.name}: unknown config_section arguments: #{args.inspect}"
         end
 
         sub_proxy = ConfigureProxy.new(name, *args)

data/lib/fluent/config/types.rb

@@ -33,6 +33,7 @@ module Fluent
     end
 
     def self.bool_value(str)
+      return nil if str.nil?
       case str.to_s
       when 'true', 'yes'
         true

data/lib/fluent/config/v1_parser.rb

@@ -50,6 +50,7 @@ module Fluent
       end
 
       ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+      RESERVED_PARAMS = %W(@type @id @label)
 
       def parse_element(root_element, elem_name, attrs = {}, elems = [])
         while true
@@ -78,7 +79,7 @@ module Fluent
           elsif skip(/\</)
             e_name = scan(ELEMENT_NAME)
             spacing
-            e_arg = scan_nonquoted_string(/(?:#{SPACING}|\>)/)
+            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
             spacing
             unless skip(/\>/)
               parse_error! "expected '>'"

data/lib/fluent/engine.rb

@@ -352,30 +352,4 @@ module Fluent
   end
 
   Engine = EngineClass.new
-
-
-  module Test
-    @@test = false
-
-    def test?
-      @@test
-    end
-
-    def self.setup
-      @@test = true
-
-      Fluent.__send__(:remove_const, :Engine)
-      engine = Fluent.const_set(:Engine, EngineClass.new).init
-
-      engine.define_singleton_method(:now=) {|n|
-        @now = n.to_i
-      }
-      engine.define_singleton_method(:now) {
-        @now || super()
-      }
-
-      nil
-    end
-  end
 end
-

data/lib/fluent/env.rb

@@ -4,4 +4,5 @@ module Fluent
   DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
   DEFAULT_LISTEN_PORT = 24224
   DEFAULT_FILE_PERMISSION = 0644
+  DEFAULT_DIR_PERMISSION = 0755
 end

data/lib/fluent/formatter.rb

@@ -18,6 +18,18 @@
 module Fluent
   require 'fluent/registry'
 
+  class Formatter
+    include Configurable
+
+    def configure(conf)
+      super
+    end
+
+    def format(tag, time, record)
+      raise NotImplementedError, "Implement this method in child class"
+    end
+  end
+
   module TextFormatter
     module HandleTagAndTimeMixin
       def self.included(klass)
@@ -51,8 +63,7 @@ module Fluent
       end
     end
 
-    class OutFileFormatter
-      include Configurable
+    class OutFileFormatter < Formatter
       include HandleTagAndTimeMixin
 
       config_param :output_time, :bool, :default => true
@@ -65,10 +76,6 @@ module Fluent
         end
       end
 
-      def configure(conf)
-        super
-      end
-
       def format(tag, time, record)
         filter_record(tag, time, record)
         header = ''
@@ -78,11 +85,12 @@ module Fluent
       end
     end
 
-    class JSONFormatter
-      include Configurable
-      include HandleTagAndTimeMixin
-
-      config_param :time_as_epoch, :bool, :default => false
+    module StructuredFormatMixin
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :time_as_epoch, :bool, :default => false
+        }
+      end
 
       def configure(conf)
         super
@@ -100,12 +108,29 @@ module Fluent
       def format(tag, time, record)
         filter_record(tag, time, record)
         record[@time_key] = time if @time_as_epoch
+        format_record(record)
+      end
+    end
+
+    class JSONFormatter < Formatter
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
         "#{Yajl.dump(record)}\n"
       end
     end
 
-    class LabeledTSVFormatter
-      include Configurable
+    class MessagePackFormatter < Formatter
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
+        record.to_msgpack
+      end
+    end
+
+    class LabeledTSVFormatter < Formatter
       include HandleTagAndTimeMixin
 
       config_param :delimiter, :string, :default => "\t"
@@ -122,9 +147,37 @@ module Fluent
       end
     end
 
-    class SingleValueFormatter
-      include Configurable
+    class CsvFormatter < Formatter
+      include HandleTagAndTimeMixin
+
+      config_param :delimiter, :default => ',' do |val|
+        ['\t', 'TAB'].include?(val) ? "\t" : val
+      end
+      config_param :force_quotes, :bool, :default => true
+      config_param :fields, :default => [] do |val|
+        val.split(',').map do |f|
+          f.strip!
+          f.size > 0 ? f : nil
+        end.compact
+      end
+
+      def initialize
+        super
+        require 'csv'
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        row = @fields.inject([]) do |memo, key|
+            memo << record[key]
+            memo
+        end
+        CSV.generate_line(row, :col_sep => @delimiter,
+                          :force_quotes => @force_quotes)
+      end
+    end
 
+    class SingleValueFormatter < Formatter
       config_param :message_key, :string, :default => 'message'
       config_param :add_newline, :bool, :default => true
 
@@ -135,41 +188,58 @@ module Fluent
       end
     end
 
+    class ProcWrappedFormatter < Formatter
+      def initialize(proc)
+        @proc = proc
+      end
+
+      def configure(conf)
+      end
+
+      def format(tag, time, record)
+        @proc.call(tag, time, record)
+      end
+    end
+
     TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
     {
       'out_file' => Proc.new { OutFileFormatter.new },
       'json' => Proc.new { JSONFormatter.new },
+      'msgpack' => Proc.new { MessagePackFormatter.new },
       'ltsv' => Proc.new { LabeledTSVFormatter.new },
+      'csv' => Proc.new { CsvFormatter.new },
       'single_value' => Proc.new { SingleValueFormatter.new },
     }.each { |name, factory|
       TEMPLATE_REGISTRY.register(name, factory)
     }
 
     def self.register_template(name, factory_or_proc)
-      factory = if factory_or_proc.arity == 3
-                  Proc.new { factory_or_proc }
-                else
+      factory = if factory_or_proc.is_a?(Class) # XXXFormatter
+                  Proc.new { factory_or_proc.new }
+                elsif factory_or_proc.arity == 3 # Proc.new { |tag, time, record| }
+                  Proc.new { ProcWrappedFormatter.new(factory_or_proc) }
+                else # Proc.new { XXXFormatter.new }
                   factory_or_proc
                 end
 
       TEMPLATE_REGISTRY.register(name, factory)
     end
 
+    def self.lookup(format)
+      TEMPLATE_REGISTRY.lookup(format).call
+    end
+
+    # Keep backward-compatibility
     def self.create(conf)
       format = conf['format']
       if format.nil?
         raise ConfigError, "'format' parameter is required"
       end
 
-      # built-in template
-      begin
-        factory = TEMPLATE_REGISTRY.lookup(format)
-      rescue ConfigError => e
-        raise ConfigError, "unknown format: '#{format}'"
+      formatter = lookup(format)
+      if formatter.respond_to?(:configure)
+        formatter.configure(conf)
       end
-
-      formatter = factory.call
-      formatter.configure(conf)
       formatter
     end
   end