fluentd 0.12.20 → 0.12.21

data/lib/fluent/parser.rb

@@ -14,9 +14,19 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/registry'
+require 'time'
+require 'json'
+
+require 'yajl'
 
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/registry'
+require 'fluent/time'
+
+module Fluent
   class ParserError < StandardError
   end
 
@@ -27,7 +37,7 @@ module Fluent
     # 'configure()' may raise errors for unexpected configurations
     attr_accessor :estimate_current_event
 
-    config_param :keep_time_key, :bool, :default => false
+    config_param :keep_time_key, :bool, default: false
 
     def initialize
       super
@@ -113,9 +123,9 @@ module Fluent
 
       def self.included(klass)
         klass.instance_eval {
-          config_param :types, :string, :default => nil
-          config_param :types_delimiter, :string, :default => ','
-          config_param :types_label_delimiter, :string, :default => ':'
+          config_param :types, :string, default: nil
+          config_param :types_delimiter, :string, default: ','
+          config_param :types_label_delimiter, :string, default: ':'
         }
       end
 
@@ -162,8 +172,8 @@ module Fluent
     class RegexpParser < Parser
       include TypeConverter
 
-      config_param :time_key, :string, :default => 'time'
-      config_param :time_format, :string, :default => nil
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
 
       def initialize(regexp, conf={})
         super()
@@ -235,9 +245,9 @@ module Fluent
     end
 
     class JSONParser < Parser
-      config_param :time_key, :string, :default => 'time'
-      config_param :time_format, :string, :default => nil
-      config_param :json_parser, :string, :default => 'oj'
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+      config_param :json_parser, :string, default: 'oj'
 
       def configure(conf)
         super
@@ -250,7 +260,7 @@ module Fluent
         begin
           raise LoadError unless @json_parser == 'oj'
           require 'oj'
-          Oj.default_options = {:bigdecimal_load => :float}
+          Oj.default_options = {bigdecimal_load: :float}
           @load_proc = Oj.method(:load)
           @error_class = Oj::ParseError
         rescue LoadError
@@ -298,17 +308,17 @@ module Fluent
     class ValuesParser < Parser
       include TypeConverter
 
-      config_param :keys, :default => [] do |val|
+      config_param :keys, default: [] do |val|
         if val.start_with?('[') # This check is enough because keys parameter is simple. No '[' started column name.
           JSON.load(val)
         else
           val.split(",")
         end
       end
-      config_param :time_key, :string, :default => nil
-      config_param :time_format, :string, :default => nil
-      config_param :null_value_pattern, :string, :default => nil
-      config_param :null_empty_string, :bool, :default => false
+      config_param :time_key, :string, default: nil
+      config_param :time_format, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
 
       def configure(conf)
         super
@@ -377,7 +387,7 @@ module Fluent
     end
 
     class TSVParser < ValuesParser
-      config_param :delimiter, :string, :default => "\t"
+      config_param :delimiter, :string, default: "\t"
 
       def configure(conf)
         super
@@ -394,9 +404,9 @@ module Fluent
     end
 
     class LabeledTSVParser < ValuesParser
-      config_param :delimiter,       :string, :default => "\t"
-      config_param :label_delimiter, :string, :default =>  ":"
-      config_param :time_key, :string, :default =>  "time"
+      config_param :delimiter,       :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+      config_param :time_key, :string, default: "time"
 
       def configure(conf)
         conf['keys'] = conf['time_key'] || 'time'
@@ -437,7 +447,7 @@ module Fluent
     end
 
     class NoneParser < Parser
-      config_param :message_key, :string, :default => 'message'
+      config_param :message_key, :string, default: 'message'
 
       def parse(text)
         record = {}
@@ -526,8 +536,8 @@ module Fluent
       # From in_syslog default pattern
       REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
 
-      config_param :time_format, :string, :default => "%b %d %H:%M:%S"
-      config_param :with_priority, :bool, :default => false
+      config_param :time_format, :string, default: "%b %d %H:%M:%S"
+      config_param :with_priority, :bool, default: false
 
       def initialize
         super
@@ -586,7 +596,7 @@ module Fluent
     end
 
     class MultilineParser < Parser
-      config_param :format_firstline, :string, :default => nil
+      config_param :format_firstline, :string, default: nil
 
       FORMAT_MAX_NUM = 20
 

data/lib/fluent/plugin.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/config/error'
+
 module Fluent
   class PluginClass
     # This class is refactored using Fluent::Registry at v0.14
@@ -66,10 +68,12 @@ module Fluent
     end
 
     def new_parser(type)
+      require 'fluent/parser'
       TextParser.lookup(type)
     end
 
     def new_formatter(type)
+      require 'fluent/formatter'
       TextFormatter.lookup(type)
     end
 

data/lib/fluent/plugin/buf_file.rb

@@ -14,6 +14,13 @@
 #    limitations under the License.
 #
 
+require 'fileutils'
+require 'uri'
+
+require 'fluent/env'
+require 'fluent/plugin'
+require 'fluent/buffer'
+
 module Fluent
   class FileBufferChunk < BufferChunk
     def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
@@ -85,7 +92,7 @@ module Fluent
     desc 'The path where buffer chunks are stored.'
     config_param :buffer_path, :string
     desc 'If true, queued chunks are flushed at shutdown process.'
-    config_param :flush_at_shutdown, :bool, :default => false
+    config_param :flush_at_shutdown, :bool, default: false
 
     # 'symlink_path' is currently only for out_file.
     # That is the reason why this is not config_param, but attr_accessor.
@@ -112,7 +119,7 @@ module Fluent
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), mode: DEFAULT_DIR_PERMISSION
       super
     end
 

data/lib/fluent/plugin/buf_memory.rb

@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'stringio'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/buffer'
+
 module Fluent
   class MemoryBufferChunk < BufferChunk
     def initialize(key, data='')
@@ -71,7 +77,7 @@ module Fluent
     end
 
     desc 'If true, queued chunks are flushed at shutdown process. Otherwise queued chunks are discarded'
-    config_param :flush_at_shutdown, :bool, :default => true
+    config_param :flush_at_shutdown, :bool, default: true
     # Overwrite default BasicBuffer#buffer_queue_limit
     # to limit total memory usage upto 512MB.
     config_set_default :buffer_queue_limit, 64

data/lib/fluent/plugin/exec_util.rb

@@ -14,10 +14,15 @@
 #    limitations under the License.
 #
 
+require 'msgpack'
+require 'yajl'
+
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/parser'
+
 module Fluent
   module ExecUtil
-    require 'fluent/parser'
-
     SUPPORTED_FORMAT = {
       'tsv' => :tsv,
       'json' => :json,

data/lib/fluent/plugin/filter_grep.rb

@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'fluent/filter'
+require 'fluent/config/error'
+
 module Fluent
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
@@ -25,8 +28,8 @@ module Fluent
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, :default => nil }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, :default => nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
 
     # for test
     attr_reader :regexps
@@ -67,7 +70,7 @@ module Fluent
           result = record
         end
       rescue => e
-        log.warn "failed to grep events", :error_class => e.class, :error => e.message
+        log.warn "failed to grep events", error_class: e.class, error: e.message
         log.warn_backtrace
       end
       result

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -14,29 +14,31 @@
 #    limitations under the License.
 #
 
+require 'socket'
+require 'json'
 require 'ostruct'
 
+require 'fluent/filter'
+require 'fluent/config/error'
+require 'fluent/event'
+require 'fluent/time'
+
 module Fluent
   class RecordTransformerFilter < Filter
     Fluent::Plugin.register_filter('record_transformer', self)
 
-    def initialize
-      require 'socket'
-      super
-    end
-
     desc 'A comma-delimited list of keys to delete.'
-    config_param :remove_keys, :string, :default => nil
+    config_param :remove_keys, :string, default: nil
     desc 'A comma-delimited list of keys to keep.'
-    config_param :keep_keys, :string, :default => nil
+    config_param :keep_keys, :string, default: nil
     desc 'Create new Hash to transform incoming data'
-    config_param :renew_record, :bool, :default => false
+    config_param :renew_record, :bool, default: false
     desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
-    config_param :renew_time_key, :string, :default => nil
+    config_param :renew_time_key, :string, default: nil
     desc 'When set to true, the full Ruby syntax is enabled in the ${...} expression.'
-    config_param :enable_ruby, :bool, :default => false
+    config_param :enable_ruby, :bool, default: false
     desc 'Use original value type.'
-    config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
+    config_param :auto_typecast, :bool, default: false # false for lower version compatibility
 
     def configure(conf)
       super
@@ -60,8 +62,8 @@ module Fluent
       end
 
       placeholder_expander_params = {
-        :log           => log,
-        :auto_typecast => @auto_typecast,
+        log: log,
+        auto_typecast: @auto_typecast,
       }
       @placeholder_expander =
         if @enable_ruby
@@ -105,7 +107,7 @@ module Fluent
       end
       new_es
     rescue => e
-      log.warn "failed to reform records", :error_class => e.class, :error => e.message
+      log.warn "failed to reform records", error_class: e.class, error: e.message
       log.warn_backtrace
       log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
@@ -119,33 +121,34 @@ module Fluent
         value_str
       end
     rescue => e
-      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", :error_class => e.class, :error => e.message
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error_class: e.class, error: e.message
       value_str # emit as string
     end
 
     def reform(record, placeholder_values)
-      @placeholder_expander.prepare_placeholders(placeholder_values)
+      placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
-      new_record.merge!(expand_placeholders(@map))
+      new_record.merge!(expand_placeholders(@map, placeholders))
       @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
 
-    def expand_placeholders(value)
+    def expand_placeholders(value, placeholders)
       if value.is_a?(String)
-        new_value = @placeholder_expander.expand(value)
+        new_value = @placeholder_expander.expand(value, placeholders)
       elsif value.is_a?(Hash)
         new_value = {}
         value.each_pair do |k, v|
-          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
+          new_key = @placeholder_expander.expand(k, placeholders, true)
+          new_value[new_key] = expand_placeholders(v, placeholders)
         end
       elsif value.is_a?(Array)
         new_value = []
         value.each_with_index do |v, i|
-          new_value[i] = expand_placeholders(v)
+          new_value[i] = expand_placeholders(v, placeholders)
         end
       else
         new_value = value
@@ -172,6 +175,7 @@ module Fluent
       rev_tag_suffix.reverse!
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
@@ -208,36 +212,37 @@ module Fluent
           end
         end
 
-        @placeholders = placeholders
+        placeholders
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_if_unknown_placeholder($1)
-            return @placeholders[single_placeholder_matched[1]]
+            log_if_unknown_placeholder($1, placeholders)
+            return placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_if_unknown_placeholder($1)
-          @placeholders[$1]
+          log_if_unknown_placeholder($1, placeholders)
+          placeholders[$1]
         }
       end
 
       private
 
-      def log_if_unknown_placeholder(placeholder)
-        unless @placeholders.include?(placeholder)
+      def log_if_unknown_placeholder(placeholder, placeholders)
+        unless placeholders.include?(placeholder)
           log.warn "unknown placeholder `#{placeholder}` found"
         end
       end
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class RubyPlaceholderExpander
       attr_reader :log
 
@@ -259,12 +264,13 @@ module Fluent
         new_value = nil
         if value.is_a?(String)
           if @auto_typecast and !force_stringify
-            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
-              new_value = single_placeholder_matched[1]
+            num_placeholders = value.scan('${').size
+            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
+              new_value = value[2..-2] # ${..} => ..
             end
           end
           unless new_value
-            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+            new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx]
           end
         elsif value.is_a?(Hash)
           new_value = {}
@@ -283,31 +289,25 @@ module Fluent
       end
 
       def prepare_placeholders(placeholder_values)
-        @tag = placeholder_values['tag']
-        @time = placeholder_values['time']
-        @record = placeholder_values['record']
-        @tag_parts = placeholder_values['tag_parts']
-        @tag_prefix = placeholder_values['tag_prefix']
-        @tag_suffix = placeholder_values['tag_suffix']
-        @hostname = placeholder_values['hostname']
+        placeholder_values
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         @cleanroom_expander.expand(
           str,
-          @tag,
-          @time,
-          @record,
-          @tag_parts,
-          @tag_prefix,
-          @tag_suffix,
-          @hostname,
+          placeholders['tag'],
+          placeholders['time'],
+          placeholders['record'],
+          placeholders['tag_parts'],
+          placeholders['tag_prefix'],
+          placeholders['tag_suffix'],
+          placeholders['hostname'],
         )
       rescue => e
-        log.warn "failed to expand `#{str}`", :error_class => e.class, :error => e.message
+        log.warn "failed to expand `#{str}`", error_class: e.class, error: e.message
         log.warn_backtrace
         nil
       end
@@ -315,15 +315,16 @@ module Fluent
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           tags = tag_parts # for old version compatibility
-          @record = record # for old version compatibility
+          Thread.current[:record_transformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end
 
         # for old version compatibility
         def method_missing(name)
           key = name.to_s
-          if @record.has_key?(key)
-            @record[key]
+          record = Thread.current[:record_transformer_record]
+          if record.has_key?(key)
+            record[key]
           else
             raise NameError, "undefined local variable or method `#{key}'"
           end