fluentd 0.14.4-x86-mingw32

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -0,0 +1,42 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Plugin
+    module OwnedByMixin
+      def owner=(plugin)
+        @_owner = plugin
+
+        @_plugin_id = plugin.plugin_id
+        @_plugin_id_configured = plugin.plugin_id_configured?
+
+        @log = plugin.log
+      end
+
+      def owner
+        if instance_variable_defined?("@_owner")
+          @_owner
+        end
+      end
+
+      def log
+        if instance_variable_defined?("@log")
+          @log
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser.rb

@@ -0,0 +1,175 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+require 'fluent/mixin' # for TypeConverter
+require 'fluent/time'
+require 'fluent/plugin/string_util'
+
+require 'strptime'
+
+module Fluent
+  module Plugin
+    class Parser < Base
+      include OwnedByMixin
+
+      class ParserError < StandardError; end
+
+      configured_in :parse
+
+      # SET false BEFORE CONFIGURE, to return nil when time not parsed
+      attr_accessor :estimate_current_event
+
+      config_param :keep_time_key, :bool, default: false
+
+      def initialize
+        super
+        @estimate_current_event = true
+      end
+
+      def parse(text)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def call(*a, &b)
+        # Keep backward compatibility for existing plugins
+        # TODO: warn when deprecated
+        parse(*a, &b)
+      end
+
+      class TimeParser
+        def initialize(time_format)
+          @cache1_key = nil
+          @cache1_time = nil
+          @cache2_key = nil
+          @cache2_time = nil
+          @parser =
+            if time_format
+              begin
+                strptime = Strptime.new(time_format)
+                Proc.new { |value| Fluent::EventTime.from_time(strptime.exec(value)) }
+              rescue
+                Proc.new { |value| Fluent::EventTime.from_time(Time.strptime(value, time_format)) }
+              end
+            else
+              Proc.new { |value| Fluent::EventTime.parse(value) }
+            end
+        end
+
+        # TODO: new cache mechanism using format string
+        def parse(value)
+          unless value.is_a?(String)
+            raise ParserError, "value must be string: #{value}"
+          end
+
+          if @cache1_key == value
+            return @cache1_time
+          elsif @cache2_key == value
+            return @cache2_time
+          else
+            begin
+              time = @parser.call(value)
+            rescue => e
+              raise ParserError, "invalid time format: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+            end
+            @cache1_key = @cache2_key
+            @cache1_time = @cache2_time
+            @cache2_key = value
+            @cache2_time = time
+            return time
+          end
+        end
+      end
+    end
+
+    class ValuesParser < Parser
+      include Fluent::TypeConverter
+
+      config_param :keys, :array, default: []
+      config_param :time_key, :string, default: nil
+      config_param :time_format, :string, default: nil
+      config_param :null_value_pattern, :string, default: nil
+      config_param :null_empty_string, :bool, default: false
+
+      def configure(conf)
+        super
+
+        if @time_key && !@keys.include?(@time_key) && @estimate_current_event
+          raise ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
+        end
+
+        if @time_format && !@time_key
+          raise ConfigError, "time_format parameter is ignored because time_key parameter is not set. at #{conf.inspect}"
+        end
+
+        @time_parser = TimeParser.new(@time_format)
+
+        if @null_value_pattern
+          @null_value_pattern = Regexp.new(@null_value_pattern)
+        end
+
+        @mutex = Mutex.new
+      end
+
+      def values_map(values)
+        record = Hash[keys.zip(values.map { |value| convert_value_to_nil(value) })]
+
+        if @time_key
+          value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+          time = if value.nil?
+                   if @estimate_current_event
+                     Fluent::EventTime.now
+                   else
+                     nil
+                   end
+                 else
+                   @mutex.synchronize { @time_parser.parse(value) }
+                 end
+        elsif @estimate_current_event
+          time = Fluent::EventTime.now
+        else
+          time = nil
+        end
+
+        convert_field_type!(record) if @type_converters
+
+        return time, record
+      end
+
+      private
+
+      def convert_field_type!(record)
+        @type_converters.each_key { |key|
+          if value = record[key]
+            record[key] = convert_type(key, value)
+          end
+        }
+      end
+
+      def convert_value_to_nil(value)
+        if value and @null_empty_string
+          value = (value == '') ? nil : value
+        end
+        if value and @null_value_pattern
+          value = ::Fluent::StringUtil.match_regexp(@null_value_pattern, value) ? nil : value
+        end
+        value
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class ApacheParser < RegexpParser
+      Plugin.register_parser("apache", self)
+
+      config_set_default :expression, %q{/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache2.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class Apache2Parser < Parser
+      Plugin.register_parser('apache2', self)
+
+      REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
+      TIME_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
+
+      def initialize
+        super
+        @time_parser = TimeParser.new(TIME_FORMAT)
+        @mutex = Mutex.new
+      end
+
+      def patterns
+        {'format' => REGEXP, 'time_format' => TIME_FORMAT}
+      end
+
+      def parse(text)
+        m = REGEXP.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        host = m['host']
+        host = (host == '-') ? nil : host
+
+        user = m['user']
+        user = (user == '-') ? nil : user
+
+        time = m['time']
+        time = @mutex.synchronize { @time_parser.parse(time) }
+
+        method = m['method']
+        path = m['path']
+
+        code = m['code'].to_i
+        code = nil if code == 0
+
+        size = m['size']
+        size = (size == '-') ? nil : size.to_i
+
+        referer = m['referer']
+        referer = (referer == '-') ? nil : referer
+
+        agent = m['agent']
+        agent = (agent == '-') ? nil : agent
+
+        record = {
+          "host" => host,
+          "user" => user,
+          "method" => method,
+          "path" => path,
+          "code" => code,
+          "size" => size,
+          "referer" => referer,
+          "agent" => agent,
+        }
+        record["time"] = m['time'] if @keep_time_key
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_apache_error.rb

@@ -0,0 +1,26 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class ApacheErrorParser < RegexpParser
+      Plugin.register_parser("apache_error", self)
+      config_set_default :expression, %q{/^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$/}
+    end
+  end
+end

data/lib/fluent/plugin/parser_csv.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'csv'
+
+module Fluent
+  module Plugin
+    class CSVParser < ValuesParser
+      Plugin.register_parser('csv', self)
+
+      config_param :delimiter, :string, default: ','
+
+      def parse(text)
+        yield values_map(CSV.parse_line(text, col_sep: @delimiter))
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_json.rb

@@ -0,0 +1,79 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/time'
+
+require 'yajl'
+
+module Fluent
+  module Plugin
+    class JSONParser < Parser
+      Plugin.register_parser('json', self)
+
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+      config_param :json_parser, :string, default: 'oj'
+
+      def configure(conf)
+        super
+
+        unless @time_format.nil?
+          @time_parser = TimeParser.new(@time_format)
+          @mutex = Mutex.new
+        end
+
+        begin
+          raise LoadError unless @json_parser == 'oj'
+          require 'oj'
+          Oj.default_options = {bigdecimal_load: :float, mode: :strict}
+          @load_proc = Oj.method(:load)
+          @error_class = Oj::ParseError
+        rescue LoadError
+          @load_proc = Yajl.method(:load)
+          @error_class = Yajl::ParseError
+        end
+      end
+
+      def parse(text)
+        record = @load_proc.call(text)
+
+        value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+        if value
+          if @time_format
+            time = @mutex.synchronize { @time_parser.parse(value) }
+          else
+            begin
+              time = Fluent::EventTime.from_time(Time.at(value.to_f))
+            rescue => e
+              raise ParserError, "invalid time value: value = #{value}, error_class = #{e.class.name}, error = #{e.message}"
+            end
+          end
+        else
+          if @estimate_current_event
+            time = Fluent::EventTime.now
+          else
+            time = nil
+          end
+        end
+
+        yield time, record
+      rescue @error_class
+        yield nil, nil
+      end
+    end
+  end
+end