fluentd 0.14.4-x86-mingw32

data/lib/fluent/plugin/parser_ltsv.rb

@@ -0,0 +1,50 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class LabeledTSVParser < ValuesParser
+      Plugin.register_parser('ltsv', self)
+
+      config_param :delimiter,       :string, default: "\t"
+      config_param :label_delimiter, :string, default: ":"
+      config_param :time_key, :string, default: "time"
+
+      def configure(conf)
+        # this assignment is not to raise ConfigError in ValuesParser#configure
+        conf['keys'] = conf['time_key'] || 'time'
+        super(conf)
+      end
+
+      def parse(text)
+        # TODO: thread unsafe: @keys might be changed by other threads
+        @keys  = []
+        values = []
+
+        text.split(delimiter).each do |pair|
+          key, value = pair.split(label_delimiter, 2)
+          @keys.push(key)
+          values.push(value)
+        end
+
+        yield values_map(values)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_multiline.rb

@@ -0,0 +1,104 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class MultilineParser < Parser
+      Plugin.register_parser('multiline', self)
+
+      config_param :format_firstline, :string, default: nil
+
+      FORMAT_MAX_NUM = 20
+
+      def configure(conf)
+        super
+
+        formats = parse_formats(conf).compact.map { |f| f[1..-2] }.join
+        begin
+          regexp = Regexp.new(formats, Regexp::MULTILINE)
+          if regexp.named_captures.empty?
+            raise "No named captures"
+          end
+          regexp_conf = Fluent::Config::Element.new("", "", { "expression" => "/#{formats}/", "multiline" => true }, [])
+          @parser = Fluent::Plugin::RegexpParser.new
+          @parser.configure(conf + regexp_conf)
+        rescue => e
+          raise ConfigError, "Invalid regexp '#{formats}': #{e}"
+        end
+
+        if @format_firstline
+          check_format_regexp(@format_firstline, 'format_firstline')
+          @firstline_regex = Regexp.new(@format_firstline[1..-2])
+        end
+      end
+
+      def parse(text, &block)
+        @parser.call(text, &block)
+      end
+
+      def has_firstline?
+        !!@format_firstline
+      end
+
+      def firstline?(text)
+        @firstline_regex.match(text)
+      end
+
+      private
+
+      def parse_formats(conf)
+        check_format_range(conf)
+
+        prev_format = nil
+        (1..FORMAT_MAX_NUM).map { |i|
+          format = conf["format#{i}"]
+          if (i > 1) && prev_format.nil? && !format.nil?
+            raise ConfigError, "Jump of format index found. format#{i - 1} is missing."
+          end
+          prev_format = format
+          next if format.nil?
+
+          check_format_regexp(format, "format#{i}")
+          format
+        }
+      end
+
+      def check_format_range(conf)
+        invalid_formats = conf.keys.select { |k|
+          m = k.match(/^format(\d+)$/)
+          m ? !((1..FORMAT_MAX_NUM).include?(m[1].to_i)) : false
+        }
+        unless invalid_formats.empty?
+          raise ConfigError, "Invalid formatN found. N should be 1 - #{FORMAT_MAX_NUM}: " + invalid_formats.join(",")
+        end
+      end
+
+      def check_format_regexp(format, key)
+        if format[0] == '/' && format[-1] == '/'
+          begin
+            Regexp.new(format[1..-2], Regexp::MULTILINE)
+          rescue => e
+            raise ConfigError, "Invalid regexp in #{key}: #{e}"
+          end
+        else
+          raise ConfigError, "format should be Regexp, need //, in #{key}: '#{format}'"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_nginx.rb

@@ -0,0 +1,28 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+module Fluent
+  module Plugin
+    class NginxParser < RegexpParser
+      Plugin.register_parser("nginx", self)
+
+      config_set_default :expression, %q{/^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/}
+      config_set_default :time_format, "%d/%b/%Y:%H:%M:%S %z"
+    end
+  end
+end

data/lib/fluent/plugin/parser_none.rb

@@ -0,0 +1,36 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class NoneParser < Parser
+      Plugin.register_parser('none', self)
+
+      config_param :message_key, :string, default: 'message'
+
+      def parse(text)
+        record = {}
+        record[@message_key] = text
+        time = @estimate_current_event ? Fluent::EventTime.now : nil
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_regexp.rb

@@ -0,0 +1,73 @@
+module Fluent
+  module Plugin
+    class RegexpParser < Parser
+      include Fluent::Compat::TypeConverter
+
+      Plugin.register_parser("regexp", self)
+
+      config_param :expression, :string, default: ""
+      config_param :ignorecase, :bool, default: false
+      config_param :multiline, :bool, default: false
+      config_param :time_key, :string, default: 'time'
+      config_param :time_format, :string, default: nil
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+        @time_parser = TimeParser.new(@time_format)
+        unless @expression.empty?
+          if @expression[0] == "/" && @expression[-1] == "/"
+            regexp_option = 0
+            regexp_option |= Regexp::IGNORECASE if @ignorecase
+            regexp_option |= Regexp::MULTILINE if @multiline
+            @regexp = Regexp.new(@expression[1..-2], regexp_option)
+          else
+            raise Fluent::ConfigError, "expression must start with `/` and end with `/`: #{@expression}"
+          end
+        end
+      end
+
+      def parse(text)
+        m = @regexp.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each do |name|
+          if value = m[name]
+            if name == @time_key
+              time = @mutex.synchronize { @time_parser.parse(value) }
+              if @keep_time_key
+                record[name] = if @type_converters.nil?
+                                 value
+                               else
+                                 convert_type(name, value)
+                               end
+              end
+            else
+              record[name] = if @type_converters.nil?
+                               value
+                             else
+                               convert_type(name, value)
+                             end
+            end
+          end
+        end
+
+        if @estimate_current_event
+          time ||= Fluent::EventTime.now
+        end
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_syslog.rb

@@ -0,0 +1,82 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class SyslogParser < Parser
+      Plugin.register_parser('syslog', self)
+
+      # From existence TextParser pattern
+      REGEXP = /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+      # From in_syslog default pattern
+      REGEXP_WITH_PRI = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
+
+      config_param :time_format, :string, default: "%b %d %H:%M:%S"
+      config_param :with_priority, :bool, default: false
+
+      def initialize
+        super
+        @mutex = Mutex.new
+      end
+
+      def configure(conf)
+        super
+
+        @regexp = @with_priority ? REGEXP_WITH_PRI : REGEXP
+        @time_parser = TimeParser.new(@time_format)
+      end
+
+      def patterns
+        {'format' => @regexp, 'time_format' => @time_format}
+      end
+
+      def parse(text)
+        m = @regexp.match(text)
+        unless m
+          yield nil, nil
+          return
+        end
+
+        time = nil
+        record = {}
+
+        m.names.each { |name|
+          if value = m[name]
+            case name
+            when "pri"
+              record['pri'] = value.to_i
+            when "time"
+              time = @mutex.synchronize { @time_parser.parse(value.gsub(/ +/, ' ')) }
+              record[name] = value if @keep_time_key
+            else
+              record[name] = value
+            end
+          end
+        }
+
+        if @estimate_current_event
+          time ||= Fluent::EventTime.now
+        end
+
+        yield time, record
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_tsv.rb

@@ -0,0 +1,37 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/parser'
+require 'fluent/time'
+
+module Fluent
+  module Plugin
+    class TSVParser < ValuesParser
+      Plugin.register_parser('tsv', self)
+
+      config_param :delimiter, :string, default: "\t"
+
+      def configure(conf)
+        super
+        @key_num = @keys.length
+      end
+
+      def parse(text)
+        yield values_map(text.split(@delimiter, @key_num))
+      end
+    end
+  end
+end

data/lib/fluent/plugin/socket_util.rb

@@ -0,0 +1,22 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/compat/socket_util'
+
+module Fluent
+  # obsolete
+  SocketUtil = Fluent::Compat::SocketUtil
+end

data/lib/fluent/plugin/storage.rb

@@ -0,0 +1,84 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/plugin/owned_by_mixin'
+
+module Fluent
+  module Plugin
+    class Storage < Base
+      include OwnedByMixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :storage
+
+      config_param :persistent,        :bool, default: false # load/save with all operations
+      config_param :autosave,          :bool, default: true
+      config_param :autosave_interval, :time, default: 10
+      config_param :save_at_shutdown,  :bool, default: true
+
+      def self.validate_key(key)
+        raise ArgumentError, "key must be a string (or symbol for to_s)" unless key.is_a?(String) || key.is_a?(Symbol)
+        key.to_s
+      end
+
+      attr_accessor :log
+
+      def persistent_always?
+        false
+      end
+
+      def synchronized?
+        false
+      end
+
+      def implementation
+        self
+      end
+
+      def load
+        # load storage data from any data source, or initialize storage internally
+      end
+
+      def save
+        # save internal data store into data source (to be loaded)
+      end
+
+      def get(key)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def fetch(key, defval)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def put(key, value)
+        # return value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def delete(key)
+        # return deleted value
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def update(key, &block) # transactional get-and-update
+        raise NotImplementedError, "Implement this method in child class"
+      end
+    end
+  end
+end