fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/plugin.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,13 +13,13 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class PluginClass
-    # This class is refactored using Fluent::Registry at v0.14
-
     def initialize
       @input = {}
       @output = {}
+      @filter = {}
       @buffer = {}
     end
 
@@ -33,16 +31,12 @@ module Fluent
       register_impl('output', @output, type, klass)
     end
 
-    def register_buffer(type, klass)
-      register_impl('buffer', @buffer, type, klass)
-    end
-
-    def register_parser(type, klass)
-      TextParser.register_template(type, klass)
+    def register_filter(type, klass)
+      register_impl('filter', @filter, type, klass)
     end
 
-    def register_formatter(type, klass)
-      TextFormatter.register_template(type, klass)
+    def register_buffer(type, klass)
+      register_impl('buffer', @buffer, type, klass)
     end
 
     def new_input(type)
@@ -53,16 +47,12 @@ module Fluent
       new_impl('output', @output, type)
     end
 
-    def new_buffer(type)
-      new_impl('buffer', @buffer, type)
+    def new_filter(type)
+      new_impl('filter', @filter, type)
     end
 
-    def new_parser(type)
-      TextParser.lookup(type)
-    end
-
-    def new_formatter(type)
-      TextFormatter.lookup(type)
+    def new_buffer(type)
+      new_impl('buffer', @buffer, type)
     end
 
     def load_plugins
@@ -108,6 +98,8 @@ module Fluent
         path = "fluent/plugin/in_#{type}"
       when 'output'
         path = "fluent/plugin/out_#{type}"
+      when 'filter'
+        path = "fluent/plugin/filter_#{type}"
       when 'buffer'
         path = "fluent/plugin/buf_#{type}"
       else
@@ -126,33 +118,16 @@ module Fluent
       end
 
       # search gems
-      if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
-        specs = Gem::Specification.find_all {|spec|
-          spec.contains_requirable_file? path
-        }
-
-        # prefer newer version
-        specs = specs.sort_by {|spec| spec.version }
-        if spec = specs.last
-          spec.require_paths.each {|lib|
-            file = "#{spec.full_gem_path}/#{lib}/#{path}"
-            require file
-          }
-        end
-
-        # backward compatibility for rubygems < 1.8
-      elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
-        #files = Gem.find_files(path).sort
-        specs = Gem.searcher.find_all(path)
+      specs = Gem::Specification.find_all { |spec|
+        spec.contains_requirable_file? path
+      }
 
-        # prefer newer version
-        specs = specs.sort_by {|spec| spec.version }
-        specs.reverse_each {|spec|
-          files = Gem.searcher.matching_files(spec, path)
-          unless files.empty?
-            require files.first
-            break
-          end
+      # prefer newer version
+      specs = specs.sort_by { |spec| spec.version }
+      if spec = specs.last
+        spec.require_paths.each { |lib|
+          file = "#{spec.full_gem_path}/#{lib}/#{path}"
+          require file
         }
       end
     end
@@ -160,4 +135,3 @@ module Fluent
 
   Plugin = PluginClass.new
 end
-

data/lib/fluent/plugin/buf_file.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class FileBufferChunk < BufferChunk
     def initialize(key, path, unique_id, mode="a+", symlink_path = nil)
@@ -84,7 +83,6 @@ module Fluent
     end
 
     config_param :buffer_path, :string
-    config_param :flush_at_shutdown, :bool, :default => false
 
     attr_accessor :symlink_path
 
@@ -92,9 +90,9 @@ module Fluent
       super
 
       if @@buffer_paths.has_key?(@buffer_path)
-        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['@type'] || conf['type']}, buffer_path = #{@buffer_path}"
+        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['type']}, buffer_path = #{@buffer_path}"
       else
-        @@buffer_paths[@buffer_path] = conf['@type'] || conf['type']
+        @@buffer_paths[@buffer_path] = conf['type']
       end
 
       if pos = @buffer_path.index('*')
@@ -105,10 +103,15 @@ module Fluent
         @buffer_path_suffix = ".log"
       end
 
+      if flush_at_shutdown = conf['flush_at_shutdown']
+        @flush_at_shutdown = true
+      else
+        @flush_at_shutdown = false
+      end
     end
 
     def start
-      FileUtils.mkdir_p File.dirname(@buffer_path_prefix + "path"), :mode => DEFAULT_DIR_PERMISSION
+      FileUtils.mkdir_p File.dirname(@buffer_path_prefix+"path")
       super
     end
 

data/lib/fluent/plugin/buf_memory.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class MemoryBufferChunk < BufferChunk
     def initialize(key, data='')

data/lib/fluent/plugin/buf_zfile.rb

@@ -0,0 +1,75 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class ZFileBufferChunk < FileBufferChunk
+    def initialize(path)
+      super(path)
+      @z = Zlib::Deflate.new
+    end
+
+    def <<(data)
+      zdata = @z.deflate(data, Z_NO_FLUSH)
+      unless zdata.empty?
+        super(zdata)
+      end
+    end
+
+    def close_write
+      zdata = @z.flush
+      unless zdata.empty?
+        @file.write(zdata)
+        @size += zdata.bytesize
+      end
+      super
+    end
+
+    def close
+      @z.close
+      super
+    end
+
+    def purge
+      @z.close
+      super
+    end
+
+    #def open(&block)
+    #end
+  end
+
+  class ZFileBuffer < FileBuffer
+    #Plugin.register_buffer('zfile', self)
+
+    def initialize
+      require 'zlib'
+      super
+    end
+
+    def resume_queue
+      queue = resume_queue_paths.map {|path|
+        ZFileBufferChunk.new(path)
+      }
+      top = new_chunk
+      return queue, top
+    end
+
+    def new_chunk
+      path = new_chunk_path
+      ZFileBufferChunk.new(path)
+    end
+  end
+end

data/lib/fluent/plugin/exec_util.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   module ExecUtil
     SUPPORTED_FORMAT = {

data/lib/fluent/plugin/in_debug_agent.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class DebugAgentInput < Input
     Plugin.register_input('debug_agent', self)

data/lib/fluent/plugin/in_exec.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class ExecInput < Input
     Plugin.register_input('exec', self)
@@ -22,7 +21,6 @@ module Fluent
     def initialize
       super
       require 'fluent/plugin/exec_util'
-      require 'fluent/timezone'
     end
 
     SUPPORTED_FORMAT = {
@@ -55,11 +53,6 @@ module Fluent
         @localtime = false
       end
 
-      if conf['timezone']
-        @timezone = conf['timezone']
-        Fluent::Timezone.validate!(@timezone)
-      end
-
       if !@tag && !@tag_key
         raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end

data/lib/fluent/plugin/in_forward.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,9 +13,8 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-
 
+module Fluent
   class ForwardInput < Input
     Plugin.register_input('forward', self)
 
@@ -54,19 +51,12 @@ module Fluent
       @loop.attach(@hbr)
 
       @thread = Thread.new(&method(:run))
-      @cached_unpacker = $use_msgpack_5 ? nil : MessagePack::Unpacker.new
     end
 
     def shutdown
       @loop.watchers.each {|w| w.detach }
       @loop.stop
       @usock.close
-      unless support_blocking_timeout?
-        listen_address = (@bind == '0.0.0.0' ? '127.0.0.1' : @bind)
-        # This line is for connecting listen socket to stop the event loop.
-        # We should use more better approach, e.g. using pipe, fixing cool.io with timeout, etc.
-        TCPSocket.open(listen_address, @port) {|sock| } # FIXME @thread.join blocks without this line
-      end
       @thread.join
       @lsock.close
     end
@@ -89,11 +79,7 @@ module Fluent
     #end
 
     def run
-      if support_blocking_timeout?
-        @loop.run(@blocking_timeout)
-      else
-        @loop.run
-      end
+      @loop.run(@blocking_timeout)
     rescue => e
       log.error "unexpected error", :error => e, :error_class => e.class
       log.error_backtrace
@@ -101,10 +87,6 @@ module Fluent
 
     protected
 
-    def support_blocking_timeout?
-      @loop.method(:run).arity.nonzero?
-    end
-
     # message Entry {
     #   1: long time
     #   2: object record
@@ -144,7 +126,7 @@ module Fluent
 
       if entries.class == String
         # PackedForward
-        es = MessagePackEventStream.new(entries, @cached_unpacker)
+        es = MessagePackEventStream.new(entries)
         router.emit_stream(tag, es)
 
       elsif entries.class == Array

data/lib/fluent/plugin/in_gc_stat.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class GCStatInput < Input
     Plugin.register_input('gc_stat', self)

data/lib/fluent/plugin/in_http.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   class HttpInput < Input
     Plugin.register_input('http', self)
@@ -29,18 +28,14 @@ module Fluent
       super
     end
 
-    EMPTY_GIF_IMAGE = "GIF89a\u0001\u0000\u0001\u0000\x80\xFF\u0000\xFF\xFF\xFF\u0000\u0000\u0000,\u0000\u0000\u0000\u0000\u0001\u0000\u0001\u0000\u0000\u0002\u0002D\u0001\u0000;".force_encoding("UTF-8")
-
     config_param :port, :integer, :default => 9880
     config_param :bind, :string, :default => '0.0.0.0'
     config_param :body_size_limit, :size, :default => 32*1024*1024  # TODO default
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
     config_param :add_http_headers, :bool, :default => false
-    config_param :add_remote_addr, :bool, :default => false
     config_param :format, :string, :default => 'default'
-    config_param :cors_allow_origins, :array, :default => nil
-    config_param :respond_with_empty_img, :bool, :default => false
+    config_param :blocking_timeout, :time, :default => 0.5
 
     def configure(conf)
       super
@@ -48,8 +43,9 @@ module Fluent
       m = if @format == 'default'
             method(:parse_params_default)
           else
-            @parser = Plugin.new_parser(@format)
-            @parser.configure(conf)
+            parser = TextParser.new
+            parser.configure(conf)
+            @parser = parser
             method(:parse_params_with_parser)
           end
       (class << self; self; end).module_eval do
@@ -89,9 +85,7 @@ module Fluent
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
         #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request),
-                                       @body_size_limit, @format, log,
-                                       @cors_allow_origins)
+        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
         @lsock.listen(@backlog) unless @backlog.nil?
 
         @loop = Coolio::Loop.new
@@ -110,20 +104,12 @@ module Fluent
     end
 
     def run
-      if support_blocking_timeout?
-        @loop.run(0.5)
-      else
-        @loop.run
-      end
+      @loop.run(@blocking_timeout)
     rescue
       log.error "unexpected error", :error=>$!.to_s
       log.error_backtrace
     end
 
-    def support_blocking_timeout?
-      @loop.method(:run).arity.nonzero?
-    end
-
     def on_request(path_info, params)
       begin
         path = path_info[1..-1]  # remove /
@@ -132,11 +118,7 @@ module Fluent
 
         # Skip nil record
         if record.nil?
-          if @respond_with_empty_img
-            return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
-          else
-            return ["200 OK", {'Content-type'=>'text/plain'}, ""]
-          end
+          return ["200 OK", {'Content-type'=>'text/plain'}, ""]
         end
 
         if @add_http_headers
@@ -147,10 +129,6 @@ module Fluent
           }
         end
 
-        if @add_remote_addr
-          record['REMOTE_ADDR'] = params['REMOTE_ADDR']
-        end
-
         time = if param_time = params['time']
                  param_time = param_time.to_i
                  param_time.zero? ? Engine.now : param_time
@@ -164,10 +142,10 @@ module Fluent
       # TODO server error
       begin
         # Support batched requests
-        if record.is_a?(Array)
+        if record.is_a?(Array)           
           mes = MultiEventStream.new
           record.each do |single_record|
-            single_time = single_record.delete("time") || time
+            single_time = single_record.delete("time") || time 
             mes.add(single_time, single_record)
           end
           router.emit_stream(tag, mes)
@@ -178,11 +156,7 @@ module Fluent
         return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
 
-      if @respond_with_empty_img
-        return ["200 OK", {'Content-type'=>'image/gif; charset=utf-8'}, EMPTY_GIF_IMAGE]
-      else
-        return ["200 OK", {'Content-type'=>'text/plain'}, ""]
-      end
+      return ["200 OK", {'Content-type'=>'text/plain'}, ""]
     end
 
     private
@@ -202,25 +176,25 @@ module Fluent
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @parser.parse(content) { |time, record|
-          raise "Received event is not #{@format}: #{content}" if record.nil?
-          return time, record
-        }
+        time, record = @parser.parse(content)
+        raise "Received event is not #{@format}: #{content}" if record.nil?
+        return time, record
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
     end
 
     class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit, format, log, cors_allow_origins)
+      def initialize(io, km, callback, body_size_limit, format, log)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
+        @content_type = ""
         @next_close = false
         @format = format
         @log = log
-        @cors_allow_origins = cors_allow_origins
+
         @idle = 0
         @km.add(self)
 
@@ -255,14 +229,12 @@ module Fluent
       def on_headers_complete(headers)
         expect = nil
         size = nil
-
         if @parser.http_version == [1, 1]
           @keep_alive = true
         else
           @keep_alive = false
         end
         @env = {}
-        @content_type = ""
         headers.each_pair {|k,v|
           @env["HTTP_#{k.gsub('-','_').upcase}"] = v
           case k
@@ -278,8 +250,6 @@ module Fluent
             elsif v =~ /Keep-alive/i
               @keep_alive = true
             end
-          when /Origin/i
-            @origin  = v
           end
         }
         if expect
@@ -308,17 +278,6 @@ module Fluent
       def on_message_complete
         return if closing?
 
-        # CORS check
-        # ==========
-        # For every incoming request, we check if we have some CORS
-        # restrictions and white listed origins through @cors_allow_origins.
-        unless @cors_allow_origins.nil?
-          unless @cors_allow_origins.include?(@origin)
-            send_response_and_close("403 Forbidden", {'Connection' => 'close'}, "")
-            return
-          end
-        end
-
         @env['REMOTE_ADDR'] = @remote_addr if @remote_addr
 
         uri = URI.parse(@parser.request_url)
@@ -388,3 +347,4 @@ module Fluent
     end
   end
 end
+