fluentd 0.12.43 → 0.14.0

data/lib/fluent/plugin/filter.rb

@@ -0,0 +1,51 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/event'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Filter < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers :event_emitter
+
+      def filter(tag, time, record)
+        raise NotImplementedError, "BUG: filter plugins MUST implement this method"
+      end
+
+      def filter_stream(tag, es)
+        new_es = MultiEventStream.new
+        es.each do |time, record|
+          begin
+            filtered_record = filter(tag, time, record)
+            new_es.add(time, filtered_record) if filtered_record
+          rescue => e
+            router.emit_error_event(tag, time, record, e)
+          end
+        end
+        new_es
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_grep.rb

@@ -28,59 +28,32 @@ module Fluent
 
     REGEXP_MAX_NUM = 20
 
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
-    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil, deprecated: "Use <exclude> section" }
-
-    config_section :regexp, param_name: :regexps, multi: true do
-      desc "The field name to which the regular expression is applied."
-      config_param :key, :string
-      desc "The regular expression."
-      config_param :pattern do |value|
-        Regexp.compile(value)
-      end
-    end
-
-    config_section :exclude, param_name: :excludes, multi: true do
-      desc "The field name to which the regular expression is applied."
-      config_param :key, :string
-      desc "The regular expression."
-      config_param :pattern do |value|
-        Regexp.compile(value)
-      end
-    end
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil }
+    (1..REGEXP_MAX_NUM).each {|i| config_param :"exclude#{i}", :string, default: nil }
 
     # for test
-    attr_reader :_regexps
-    attr_reader :_excludes
+    attr_reader :regexps
+    attr_reader :excludes
 
     def configure(conf)
       super
 
-      @_regexps = {}
+      @regexps = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
         raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
-        @_regexps[key] = Regexp.compile(regexp)
+        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @regexps[key]
+        @regexps[key] = Regexp.compile(regexp)
       end
 
-      @_excludes = {}
+      @excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
         raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
-        @_excludes[key] = Regexp.compile(exclude)
-      end
-
-      @regexps.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
-        @_regexps[e.key] = e.pattern
-      end
-      @excludes.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
-        @_excludes[e.key] = e.pattern
+        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @excludes[key]
+        @excludes[key] = Regexp.compile(exclude)
       end
     end
 
@@ -88,16 +61,16 @@ module Fluent
       result = nil
       begin
         catch(:break_loop) do
-          @_regexps.each do |key, regexp|
+          @regexps.each do |key, regexp|
             throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
           end
-          @_excludes.each do |key, exclude|
+          @excludes.each do |key, exclude|
             throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
           end
           result = record
         end
       rescue => e
-        log.warn "failed to grep events", error_class: e.class, error: e.message
+        log.warn "failed to grep events", error: e
         log.warn_backtrace
       end
       result

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -28,9 +28,9 @@ module Fluent
     Fluent::Plugin.register_filter('record_transformer', self)
 
     desc 'A comma-delimited list of keys to delete.'
-    config_param :remove_keys, :array, default: nil
+    config_param :remove_keys, :string, default: nil
     desc 'A comma-delimited list of keys to keep.'
-    config_param :keep_keys, :array, default: nil
+    config_param :keep_keys, :string, default: nil
     desc 'Create new Hash to transform incoming data'
     config_param :renew_record, :bool, default: false
     desc 'Specify field name of the record to overwrite the time of events. Its value must be unix time.'
@@ -52,8 +52,13 @@ module Fluent
         end
       end
 
+      if @remove_keys
+        @remove_keys = @remove_keys.split(',')
+      end
+
       if @keep_keys
         raise Fluent::ConfigError, "`renew_record` must be true to use `keep_keys`" unless @renew_record
+        @keep_keys = @keep_keys.split(',')
       end
 
       placeholder_expander_params = {
@@ -90,18 +95,19 @@ module Fluent
       last_record = nil
       es.each do |time, record|
         last_record = record # for debug log
-        placeholder_values['time'] = @placeholder_expander.time_value(time)
-        placeholder_values['record'] = record
+        placeholder_values.merge!({
+          'time'     => @placeholder_expander.time_value(time),
+          'record'   => record,
+        })
         new_record = reform(record, placeholder_values)
         if @renew_time_key && new_record.has_key?(@renew_time_key)
-          time = new_record[@renew_time_key].to_i
+          time = EventTime.from_time(Time.at(new_record[@renew_time_key].to_f))
         end
-        @remove_keys.each { |k| new_record.delete(k) } if @remove_keys
         new_es.add(time, new_record)
       end
       new_es
     rescue => e
-      log.warn "failed to reform records", error_class: e.class, error: e.message
+      log.warn "failed to reform records", error: e
       log.warn_backtrace
       log.debug "map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
@@ -115,7 +121,7 @@ module Fluent
         value_str
       end
     rescue => e
-      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error_class: e.class, error: e.message
+      log.warn "failed to parse #{value_str} as json. Assuming #{value_str} is a string", error: e
       value_str # emit as string
     end
 
@@ -123,10 +129,9 @@ module Fluent
       placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
       new_record = @renew_record ? {} : record.dup
-      @keep_keys.each do |k|
-        new_record[k] = record[k] if record.has_key?(k)
-      end if @keep_keys && @renew_record
+      @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
       new_record.merge!(expand_placeholders(@map, placeholders))
+      @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       new_record
     end
@@ -199,9 +204,7 @@ module Fluent
             end
           elsif value.kind_of?(Hash) # record, etc
             value.each do |k, v|
-              unless placeholder_values.has_key?(k) # prevent overwriting reserved keys such as tag
-                placeholders.store("${#{k}}", v) # foo
-              end
+              placeholders.store("${#{k}}", v) # foo
               placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
             end
           else # string, interger, float, and others?
@@ -217,7 +220,7 @@ module Fluent
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
       def expand(str, placeholders, force_stringify = false)
-        if @auto_typecast && !force_stringify
+        if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
             log_if_unknown_placeholder($1, placeholders)
@@ -260,9 +263,9 @@ module Fluent
       def preprocess_map(value, force_stringify = false)
         new_value = nil
         if value.is_a?(String)
-          if @auto_typecast && !force_stringify
+          if @auto_typecast and !force_stringify
             num_placeholders = value.scan('${').size
-            if num_placeholders == 1 && value.start_with?('${') && value.end_with?('}')
+            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
               new_value = value[2..-2] # ${..} => ..
             end
           end
@@ -304,7 +307,7 @@ module Fluent
           placeholders['hostname'],
         )
       rescue => e
-        log.warn "failed to expand `#{str}`", error_class: e.class, error: e.message
+        log.warn "failed to expand `#{str}`", error: e
         log.warn_backtrace
         nil
       end
@@ -312,6 +315,7 @@ module Fluent
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           tags = tag_parts # for old version compatibility
+          _ = tags # to suppress "unused variable" warning for tags
           Thread.current[:record_transformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end

data/lib/fluent/plugin/formatter.rb

@@ -0,0 +1,93 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+require 'fluent/mixin' # for TimeFormatter
+
+module Fluent
+  module Plugin
+    class Formatter < Base
+      include OwnedByMixin
+
+      configured_in :format
+
+      def format(tag, time, record)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      # Mixins for formatter plugins
+      module HandleTagAndTimeMixin
+        def self.included(klass)
+          klass.instance_eval {
+            config_param :include_time_key, :bool, default: false
+            config_param :time_key, :string, default: 'time'
+            config_param :time_format, :string, default: nil
+            config_param :time_as_epoch, :bool, default: false
+            config_param :include_tag_key, :bool, default: false
+            config_param :tag_key, :string, default: 'tag'
+            config_param :localtime, :bool, default: true
+            config_param :timezone, :string, default: nil
+          }
+        end
+
+        def configure(conf)
+          super
+
+          if conf['utc']
+            @localtime = false
+          end
+          @timef = Fluent::TimeFormatter.new(@time_format, @localtime, @timezone)
+          if @time_as_epoch && !@include_time_key
+            log.warn "time_as_epoch will be ignored because include_time_key is false"
+          end
+        end
+
+        def filter_record(tag, time, record)
+          if @include_tag_key
+            record[@tag_key] = tag
+          end
+          if @include_time_key
+            if @time_as_epoch
+              record[@time_key] = time.to_i
+            else
+              record[@time_key] = @timef.format(time)
+            end
+          end
+        end
+      end
+
+      module StructuredFormatMixin
+        def format(tag, time, record)
+          filter_record(tag, time, record)
+          format_record(record)
+        end
+      end
+    end
+
+    class ProcWrappedFormatter < Formatter
+      def initialize(proc)
+        super()
+        @proc = proc
+      end
+
+      def format(tag, time, record)
+        @proc.call(tag, time, record)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_csv.rb

@@ -0,0 +1,48 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CsvFormatter < Formatter
+      Plugin.register_formatter('csv', self)
+
+      include HandleTagAndTimeMixin
+
+      config_param :delimiter, default: ',' do |val|
+        ['\t', 'TAB'].include?(val) ? "\t" : val
+      end
+      config_param :force_quotes, :bool, default: true
+      # "array" looks good for type of :fields, but this implementation removes tailing comma
+      # TODO: Is it needed to support tailing comma?
+      config_param :fields, default: [] do |val|
+        val.split(',').map{|f| f.strip!; f.empty? ? nil : f }.compact
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        row = @fields.inject([]) do |memo, key|
+            memo << record[key]
+            memo
+        end
+        CSV.generate_line(row, col_sep: @delimiter,
+                          force_quotes: @force_quotes)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_hash.rb

@@ -0,0 +1,32 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class HashFormatter < Formatter
+      Plugin.register_formatter('hash', self)
+
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
+        "#{record.to_s}\n"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/formatter_json.rb

@@ -0,0 +1,47 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/formatter'
+
+module Fluent
+  module Plugin
+    class JSONFormatter < Formatter
+      Plugin.register_formatter('json', self)
+
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      config_param :json_parser, :string, default: 'oj'
+
+      def configure(conf)
+        super
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = {mode: :compat}
+          @dump_proc = Oj.method(:dump)
+        rescue LoadError
+          @dump_proc = Yajl.method(:dump)
+        end
+      end
+
+      def format_record(record)
+        "#{@dump_proc.call(record)}\n"
+      end
+    end
+  end
+end