fluentd222 1.16.2-x86_64-linux

data/lib/fluent/plugin/parser_ltsv.rb

@@ -0,0 +1,51 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class LabeledTSVParser < Parser
+      Plugin.register_parser('ltsv', self)
+
+      desc 'The delimiter character (or string) of TSV values'
+      config_param :delimiter, :string, default: "\t"
+      desc 'The delimiter pattern of TSV values'
+      config_param :delimiter_pattern, :regexp, default: nil
+      desc 'The delimiter character between field name and value'
+      config_param :label_delimiter, :string, default: ":"
+
+      config_set_default :time_key, 'time'
+
+      def configure(conf)
+        super
+        @delimiter = @delimiter_pattern || @delimiter
+      end
+
+      def parse(text)
+        r = {}
+        text.split(@delimiter).each do |pair|
+          if pair.include? @label_delimiter
+            key, value = pair.split(@label_delimiter, 2)
+            r[key] = value
+          end
+        end
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Plugin
+    class MessagePackParser < Parser
+      Plugin.register_parser('msgpack', self)
+
+      def configure(conf)
+        super
+        @unpacker = Fluent::MessagePackFactory.engine_factory.unpacker
+      end
+
+      def parser_type
+        :binary
+      end
+
+      def parse(data)
+        @unpacker.feed_each(data) do |obj|
+          yield convert_values(parse_time(obj), obj)
+        end
+      end
+      alias parse_partial_data parse
+
+      def parse_io(io, &block)
+        u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
+        u.each do |obj|
+          time, record = convert_values(parse_time(obj), obj)
+          yield time, record
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_multiline.rb

@@ -0,0 +1,152 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class MultilineParser < Parser
+      Plugin.register_parser('multiline', self)
+
+      desc 'Specify regexp pattern for start line of multiple lines'
+      config_param :format_firstline, :string, default: nil
+      desc 'Enable an option returning line as unmatched_line'
+      config_param :unmatched_lines, :string, default: nil
+
+      FORMAT_MAX_NUM = 20
+
+      class MultilineRegexpParser < Fluent::Plugin::RegexpParser
+        def parse(text)
+          m = @expression.match(text)
+          unless m
+            yield nil, nil
+            return m
+          end
+
+          r = {}
+          m.names.each do |name|
+            if (value = m[name])
+              r[name] = value
+            end
+          end
+
+          time, record = convert_values(parse_time(r), r)
+
+          yield(time, record)
+          m
+        end
+      end
+
+      def configure(conf)
+        super
+
+        formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
+        begin
+          regexp = Regexp.new(formats, Regexp::MULTILINE)
+          if regexp.named_captures.empty?
+            raise "No named captures"
+          end
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/m" }, [])
+          @parser = Fluent::Plugin::MultilineParser::MultilineRegexpParser.new
+          @parser.configure(conf + regexp_conf)
+        rescue => e
+          raise Fluent::ConfigError, "Invalid regexp '#{formats}': #{e}"
+        end
+
+        if @format_firstline
+          check_format_regexp(@format_firstline, 'format_firstline')
+          @firstline_regex = Regexp.new(@format_firstline[1..-2])
+        end
+      end
+
+      def parse(text, &block)
+        loop do
+          m =
+            if @unmatched_lines
+              @parser.call(text) do |time, record|
+                if time && record
+                  yield(time, record)
+                else
+                  yield(Fluent::EventTime.now, { 'unmatched_line' => text })
+                end
+              end
+            else
+              @parser.call(text, &block)
+            end
+
+          return if m.nil?
+
+          text = m.post_match
+          if text.start_with?("\n")
+            text = text[1..-1]
+          end
+
+          return if text.empty?
+        end
+      end
+
+      def has_firstline?
+        !!@format_firstline
+      end
+
+      def firstline?(text)
+        @firstline_regex.match?(text)
+      end
+
+      private
+
+      def parse_formats(conf)
+        check_format_range(conf)
+
+        prev_format = nil
+        (1..FORMAT_MAX_NUM).map { |i|
+          format = conf["format#{i}"]
+          if (i > 1) && prev_format.nil? && !format.nil?
+            raise Fluent::ConfigError, "Jump of format index found. format#{i - 1} is missing."
+          end
+          prev_format = format
+          next if format.nil?
+
+          check_format_regexp(format, "format#{i}")
+          format
+        }
+      end
+
+      def check_format_range(conf)
+        invalid_formats = conf.keys.select { |k|
+          m = k.match(/^format(\d+)$/)
+          m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
+        }
+        unless invalid_formats.empty?
+          raise Fluent::ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
+        end
+      end
+
+      def check_format_regexp(format, key)
+        if format[0] == '/' && format[-1] == '/'
+          begin
+            Regexp.new(format[1..-2], Regexp::MULTILINE)
+          rescue => e
+            raise Fluent::ConfigError, "Invalid regexp in #{key}: #{e}"
+          end
+        else
+          raise Fluent::ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_nginx.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser_regexp'
+
+module Fluent
+  module Plugin
+    class NginxParser < RegexpParser
+      Plugin.register_parser("nginx", self)
+
+      config_set_default :expression, /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+\"?(?<http_x_forwarded_for>[^\"]*)\"?)?)?$/
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_none.rb

@@ -0,0 +1,36 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class NoneParser < Parser
+      Plugin.register_parser('none', self)
+
+      desc 'Field name to contain logs'
+      config_param :message_key, :string, default: 'message'
+
+      def parse(text)
+        record = {@message_key => text}
+        time = @estimate_current_event ? Fluent::EventTime.now : nil
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_regexp.rb

@@ -0,0 +1,68 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class RegexpParser < Parser
+      Plugin.register_parser("regexp", self)
+
+      desc 'Regular expression for matching logs'
+      config_param :expression, :regexp
+      desc 'Ignore case in matching'
+      config_param :ignorecase, :bool, default: false, deprecated: "Use /pattern/i instead, this option is no longer effective"
+      desc 'Build regular expression as a multline mode'
+      config_param :multiline, :bool, default: false, deprecated: "Use /pattern/m instead, this option is no longer effective"
+
+      config_set_default :time_key, 'time'
+
+      def configure(conf)
+        super
+        # For compat layer
+        if @ignorecase || @multiline
+          options = 0
+          options |= Regexp::IGNORECASE if @ignorecase
+          options |= Regexp::MULTILINE if @multiline
+          @expression = Regexp.compile(@expression.source, options)
+        end
+        @regexp = @expression # For backward compatibility
+
+        if @expression.named_captures.empty?
+          raise Fluent::ConfigError, "No named captures in 'expression' parameter. The regexp must have at least one named capture"
+        end
+      end
+
+      def parse(text)
+        m = @expression.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        r = {}
+        m.names.each do |name|
+          if value = m[name]
+            r[name] = value
+          end
+        end
+
+        time, record = convert_values(parse_time(r), r)
+        yield time, record
+      end
+    end
+  end
+end