fluentd 0.12.40 → 1.6.2

data/lib/fluent/plugin/compressable.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'stringio'
+require 'zlib'
+
+module Fluent
+  module Plugin
+    module Compressable
+      def compress(data, **kwargs)
+        output_io = kwargs[:output_io]
+        io = output_io || StringIO.new
+        Zlib::GzipWriter.wrap(io) do |gz|
+          gz.write data
+        end
+
+        output_io || io.string
+      end
+
+      # compressed_data is String like `compress(data1) + compress(data2) + ... + compress(dataN)`
+      # https://www.ruby-forum.com/topic/971591#979503
+      def decompress(compressed_data = nil, output_io: nil, input_io: nil)
+        case
+        when input_io && output_io
+          io_decompress(input_io, output_io)
+        when input_io
+          output_io = StringIO.new
+          io = io_decompress(input_io, output_io)
+          io.string
+        when compressed_data.nil? || compressed_data.empty?
+          # check compressed_data(String) is 0 length
+          compressed_data
+        when output_io
+          # execute after checking compressed_data is empty or not
+          io = StringIO.new(compressed_data)
+          io_decompress(io, output_io)
+        else
+          string_decompress(compressed_data)
+        end
+      end
+
+      private
+
+      def string_decompress(compressed_data)
+        io = StringIO.new(compressed_data)
+
+        out = ''
+        loop do
+          gz = Zlib::GzipReader.new(io)
+          out << gz.read
+          unused = gz.unused
+          gz.finish
+
+          break if unused.nil?
+          adjust = unused.length
+          io.pos -= adjust
+        end
+
+        out
+      end
+
+      def io_decompress(input, output)
+        loop do
+          gz = Zlib::GzipReader.new(input)
+          v = gz.read
+          output.write(v)
+          unused = gz.unused
+          gz.finish
+
+          break if unused.nil?
+          adjust = unused.length
+          input.pos -= adjust
+        end
+
+        output
+      end
+    end
+  end
+end

data/lib/fluent/plugin/exec_util.rb

@@ -14,114 +14,9 @@
 #    limitations under the License.
 #
 
-require 'msgpack'
-require 'yajl'
-
-require 'fluent/engine'
-require 'fluent/plugin'
-require 'fluent/parser'
+require 'fluent/compat/exec_util'
 
 module Fluent
-  module ExecUtil
-    SUPPORTED_FORMAT = {
-      'tsv' => :tsv,
-      'json' => :json,
-      'msgpack' => :msgpack,
-    }
-
-    class Parser
-      def initialize(on_message)
-        @on_message = on_message
-      end
-    end
-
-    class TextParserWrapperParser < Parser
-      def initialize(conf, on_message)
-        @parser = Plugin.new_parser(conf['format'])
-        @parser.configure(conf)
-        super(on_message)
-      end
-
-      def call(io)
-        io.each_line(&method(:each_line))
-      end
-
-      def each_line(line)
-        line.chomp!
-        @parser.parse(line) { |time, record|
-          @on_message.call(record, time)
-        }
-      end
-    end
-
-    class TSVParser < Parser
-      def initialize(keys, on_message)
-        @keys = keys
-        super(on_message)
-      end
-
-      def call(io)
-        io.each_line(&method(:each_line))
-      end
-
-      def each_line(line)
-        line.chomp!
-        vals = line.split("\t")
-
-        record = Hash[@keys.zip(vals)]
-
-        @on_message.call(record)
-      end
-    end
-
-    class JSONParser < Parser
-      def call(io)
-        y = Yajl::Parser.new
-        y.on_parse_complete = @on_message
-        y.parse(io)
-      end
-    end
-
-    class MessagePackParser < Parser
-      def call(io)
-        @u = Fluent::Engine.msgpack_factory.unpacker(io)
-        begin
-          @u.each(&@on_message)
-        rescue EOFError
-        end
-      end
-    end
-
-    class Formatter
-    end
-
-    class TSVFormatter < Formatter
-      def initialize(in_keys)
-        @in_keys = in_keys
-        super()
-      end
-
-      def call(record, out)
-        last = @in_keys.length-1
-        for i in 0..last
-          key = @in_keys[i]
-          out << record[key].to_s
-          out << "\t" if i != last
-        end
-        out << "\n"
-      end
-    end
-
-    class JSONFormatter < Formatter
-      def call(record, out)
-        out << Yajl.dump(record) << "\n"
-      end
-    end
-
-    class MessagePackFormatter < Formatter
-      def call(record, out)
-        record.to_msgpack(out)
-      end
-    end
-  end
+  # obsolete
+  ExecUtil = Fluent::Compat::ExecUtil
 end

data/lib/fluent/plugin/file_util.rb

@@ -14,39 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module FileUtil
-    # Check file is writable if file exists
-    # Check directory is writable if file does not exist
-    #
-    # @param [String] path File path
-    # @return [Boolean] file is writable or not
-    def writable?(path)
-      return false if File.directory?(path)
-      return File.writable?(path) if File.exist?(path)
-
-      dirname = File.dirname(path)
-      return false if !File.directory?(dirname)
-      File.writable?(dirname)
-    end
-    module_function :writable?
-
-    # Check file is writable in conjunction wtih mkdir_p(dirname(path))
-    #
-    # @param [String] path File path
-    # @return [Boolean] file writable or not
-    def writable_p?(path)
-      return false if File.directory?(path)
-      return File.writable?(path) if File.exist?(path)
+require 'fluent/compat/file_util'
 
-      dirname = File.dirname(path)
-      until File.exist?(dirname)
-        dirname = File.dirname(dirname)
-      end
-
-      return false if !File.directory?(dirname)
-      File.writable?(dirname)
-    end
-    module_function :writable_p?
-  end
+module Fluent
+  # obsolete
+  FileUtil = Fluent::Compat::FileUtil
 end

data/lib/fluent/plugin/file_wrapper.rb

@@ -0,0 +1,120 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module FileWrapper
+    def self.open(*args)
+      io = WindowsFile.new(*args).io
+      if block_given?
+        v = yield io
+        io.close
+        v
+      else
+        io
+      end
+    end
+
+    def self.stat(path)
+      f = WindowsFile.new(path)
+      s = f.stat
+      f.close
+      s
+    end
+  end
+
+  module WindowsFileExtension
+    attr_reader :path
+
+    def stat
+      s = super
+      s.instance_variable_set :@ino, @ino
+      def s.ino; @ino; end
+      s
+    end
+  end
+
+  # To open and get stat with setting FILE_SHARE_DELETE
+  class WindowsFile
+    require 'windows/file'
+    require 'windows/error'
+    require 'windows/handle'
+    require 'windows/nio'
+
+    include Windows::Error
+    include Windows::File
+    include Windows::Handle
+    include Windows::NIO
+
+    def initialize(path, mode='r', sharemode=FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE)
+      @path = path
+      @file_handle = INVALID_HANDLE_VALUE
+      @mode = mode
+
+
+      access, creationdisposition, seektoend = case mode.delete('b')
+      when "r" ; [FILE_GENERIC_READ                     , OPEN_EXISTING, false]
+      when "r+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , false]
+      when "w" ; [FILE_GENERIC_WRITE                    , CREATE_ALWAYS, false]
+      when "w+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, CREATE_ALWAYS, false]
+      when "a" ; [FILE_GENERIC_WRITE                    , OPEN_ALWAYS  , true]
+      when "a+"; [FILE_GENERIC_READ | FILE_GENERIC_WRITE, OPEN_ALWAYS  , true]
+      else raise "unknown mode '#{mode}'"
+      end
+
+      @file_handle = CreateFile.call(@path, access, sharemode,
+                     0, creationdisposition, FILE_ATTRIBUTE_NORMAL, 0)
+      if @file_handle == INVALID_HANDLE_VALUE
+        err = GetLastError.call
+        if err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND || err == ERROR_ACCESS_DENIED
+          raise SystemCallError.new(2)
+        end
+        raise SystemCallError.new(err)
+      end
+    end
+
+    def close
+      CloseHandle.call(@file_handle)
+      @file_handle = INVALID_HANDLE_VALUE
+    end
+
+    def io
+      fd = _open_osfhandle(@file_handle, 0)
+      raise Errno::ENOENT if fd == -1
+      io = File.for_fd(fd, @mode)
+      io.instance_variable_set :@ino, self.ino
+      io.instance_variable_set :@path, @path
+      io.extend WindowsFileExtension
+      io
+    end
+
+    def ino
+      by_handle_file_information = '\0'*(4+8+8+8+4+4+4+4+4+4)   #72bytes
+
+      unless GetFileInformationByHandle.call(@file_handle, by_handle_file_information)
+        return 0
+      end
+
+      by_handle_file_information.unpack("I11Q1")[11] # fileindex
+    end
+
+    def stat
+      s = File.stat(@path)
+      s.instance_variable_set :@ino, self.ino
+      def s.ino; @ino; end
+      s
+    end
+  end
+end if Fluent.windows?

data/lib/fluent/plugin/filter.rb

@@ -0,0 +1,93 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/event'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Filter < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers_internal :event_emitter
+
+      attr_reader :has_filter_with_time
+
+      def initialize
+        super
+        @has_filter_with_time = has_filter_with_time?
+      end
+
+      def filter(tag, time, record)
+        raise NotImplementedError, "BUG: filter plugins MUST implement this method"
+      end
+
+      def filter_with_time(tag, time, record)
+        raise NotImplementedError, "BUG: filter plugins MUST implement this method"
+      end
+
+      def filter_stream(tag, es)
+        new_es = MultiEventStream.new
+        if @has_filter_with_time
+          es.each do |time, record|
+            begin
+              filtered_time, filtered_record = filter_with_time(tag, time, record)
+              new_es.add(filtered_time, filtered_record) if filtered_time && filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        else
+          es.each do |time, record|
+            begin
+              filtered_record = filter(tag, time, record)
+              new_es.add(time, filtered_record) if filtered_record
+            rescue => e
+              router.emit_error_event(tag, time, record, e)
+            end
+          end
+        end
+        new_es
+      end
+
+      private
+
+      def has_filter_with_time?
+        implmented_methods = self.class.instance_methods(false)
+        # Plugins that override `filter_stream` don't need check,
+        # because they may not call `filter` or `filter_with_time`
+        # for example fluentd/lib/fluent/plugin/filter_record_transformer.rb
+        return nil if implmented_methods.include?(:filter_stream)
+        case
+        when [:filter, :filter_with_time].all? { |e| implmented_methods.include?(e) }
+          raise "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
+        when implmented_methods.include?(:filter)
+          false
+        when implmented_methods.include?(:filter_with_time)
+          true
+        else
+          raise NotImplementedError, "BUG: Filter plugins MUST implement either `filter` or `filter_with_time`"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/filter_grep.rb

@@ -14,18 +14,28 @@
 #    limitations under the License.
 #
 
-require 'fluent/filter'
+require 'fluent/plugin/filter'
 require 'fluent/config/error'
+require 'fluent/plugin/string_util'
 
-module Fluent
+module Fluent::Plugin
   class GrepFilter < Filter
     Fluent::Plugin.register_filter('grep', self)
 
     def initialize
       super
-      require 'fluent/plugin/string_util'
+
+      @_regexp_and_conditions = nil
+      @_exclude_and_conditions = nil
+      @_regexp_or_conditions = nil
+      @_exclude_or_conditions = nil
     end
 
+    # for test
+    attr_reader :_regexp_and_conditions, :_exclude_and_conditions, :_regexp_or_conditions, :_exclude_or_conditions
+
+    helpers :record_accessor
+
     REGEXP_MAX_NUM = 20
 
     (1..REGEXP_MAX_NUM).each {|i| config_param :"regexp#{i}",  :string, default: nil, deprecated: "Use <regexp> section" }
@@ -35,72 +45,145 @@ module Fluent
       desc "The field name to which the regular expression is applied."
       config_param :key, :string
       desc "The regular expression."
-      config_param :pattern do |value|
-        Regexp.compile(value)
-      end
+      config_param :pattern, :regexp
     end
 
     config_section :exclude, param_name: :excludes, multi: true do
       desc "The field name to which the regular expression is applied."
       config_param :key, :string
       desc "The regular expression."
-      config_param :pattern do |value|
-        Regexp.compile(value)
+      config_param :pattern, :regexp
+    end
+
+    config_section :and, param_name: :and_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
       end
     end
 
-    # for test
-    attr_reader :_regexps
-    attr_reader :_excludes
+    config_section :or, param_name: :or_conditions, multi: true do
+      config_section :regexp, param_name: :regexps, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+      config_section :exclude, param_name: :excludes, multi: true do
+        desc "The field name to which the regular expression is applied."
+        config_param :key, :string
+        desc "The regular expression."
+        config_param :pattern, :regexp
+      end
+    end
 
     def configure(conf)
       super
 
-      @_regexps = {}
+      regexp_and_conditions = {}
+      regexp_or_conditions = {}
+      exclude_and_conditions = {}
+      exclude_or_conditions = {}
+
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["regexp#{i}"]
         key, regexp = conf["regexp#{i}"].split(/ /, 2)
-        raise ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
-        raise ConfigError, "regexp#{i} contains a duplicated key, #{key}" if @_regexps[key]
-        @_regexps[key] = Regexp.compile(regexp)
+        raise Fluent::ConfigError, "regexp#{i} does not contain 2 parameters" unless regexp
+        raise Fluent::ConfigError, "regexp#{i} contains a duplicated key, #{key}" if regexp_and_conditions[key]
+        regexp_and_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(regexp))
       end
 
-      @_excludes = {}
       (1..REGEXP_MAX_NUM).each do |i|
         next unless conf["exclude#{i}"]
         key, exclude = conf["exclude#{i}"].split(/ /, 2)
-        raise ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
-        raise ConfigError, "exclude#{i} contains a duplicated key, #{key}" if @_excludes[key]
-        @_excludes[key] = Regexp.compile(exclude)
+        raise Fluent::ConfigError, "exclude#{i} does not contain 2 parameters" unless exclude
+        raise Fluent::ConfigError, "exclude#{i} contains a duplicated key, #{key}" if exclude_or_conditions[key]
+        exclude_or_conditions[key] = Expression.new(record_accessor_create(key), Regexp.compile(exclude))
       end
 
+      if @regexps.size > 1
+        log.info "Top level multiple <regexp> is intepreted as 'and' condition"
+      end
       @regexps.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_regexps.key?(e.key)
-        @_regexps[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if regexp_and_conditions.key?(e.key)
+        regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+      end
+
+      if @excludes.size > 1
+        log.info "Top level multiple <exclude> is intepreted as 'or' condition"
       end
       @excludes.each do |e|
-        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if @_excludes.key?(e.key)
-        @_excludes[e.key] = e.pattern
+        raise Fluent::ConfigError, "Duplicate key: #{e.key}" if exclude_or_conditions.key?(e.key)
+        exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
       end
+
+      @and_conditions.each do |and_condition|
+        if !and_condition.regexps.empty? && !and_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <and>"
+        end
+        and_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if regexp_and_conditions.key?(e.key)
+          regexp_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        and_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <and>: #{e.key}" if exclude_and_conditions.key?(e.key)
+          exclude_and_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+      end
+
+      @or_conditions.each do |or_condition|
+        if !or_condition.regexps.empty? && !or_condition.excludes.empty?
+          raise Fluent::ConfigError, "Do not specify both <regexp> and <exclude> in <or>"
+        end
+        or_condition.regexps.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if regexp_or_conditions.key?(e.key)
+          regexp_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+        or_condition.excludes.each do |e|
+          raise Fluent::ConfigError, "Duplicate key in <or>: #{e.key}" if exclude_or_conditions.key?(e.key)
+          exclude_or_conditions[e.key] = Expression.new(record_accessor_create(e.key), e.pattern)
+        end
+      end
+
+      @_regexp_and_conditions = regexp_and_conditions.values unless regexp_and_conditions.empty?
+      @_exclude_and_conditions = exclude_and_conditions.values unless exclude_and_conditions.empty?
+      @_regexp_or_conditions = regexp_or_conditions.values unless regexp_or_conditions.empty?
+      @_exclude_or_conditions = exclude_or_conditions.values unless exclude_or_conditions.empty?
     end
 
     def filter(tag, time, record)
-      result = nil
       begin
-        catch(:break_loop) do
-          @_regexps.each do |key, regexp|
-            throw :break_loop unless ::Fluent::StringUtil.match_regexp(regexp, record[key].to_s)
-          end
-          @_excludes.each do |key, exclude|
-            throw :break_loop if ::Fluent::StringUtil.match_regexp(exclude, record[key].to_s)
-          end
-          result = record
+        if @_regexp_and_conditions && @_regexp_and_conditions.any? { |expression| !expression.match?(record) }
+          return nil
+        end
+        if @_regexp_or_conditions && @_regexp_or_conditions.none? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_and_conditions && @_exclude_and_conditions.all? { |expression| expression.match?(record) }
+          return nil
+        end
+        if @_exclude_or_conditions && @_exclude_or_conditions.any? { |expression| expression.match?(record) }
+          return nil
         end
       rescue => e
-        log.warn "failed to grep events", error_class: e.class, error: e.message
+        log.warn "failed to grep events", error: e
         log.warn_backtrace
       end
-      result
+      record
+    end
+
+    Expression = Struct.new(:key, :pattern) do
+      def match?(record)
+        ::Fluent::StringUtil.match_regexp(pattern, key.call(record).to_s)
+      end
     end
   end
 end