fluent-plugin-aliyun-oss 0.0.1

data/lib/fluent/plugin/mns/message.rb

@@ -0,0 +1,52 @@
+require 'rexml/document'
+
+module Fluent
+  module Plugin
+    module MNS
+      # Class for Aliyun MNS Message.
+      class Message
+        include REXML
+
+        attr_reader :queue, :id, :body_md5, :body, :receipt_handle, :enqueue_at,
+                    :first_enqueue_at, :next_visible_at, :dequeue_count, :priority
+
+        def initialize(queue, content)
+          @queue = queue
+
+          doc = Document.new(content)
+          doc.elements[1].each do |e|
+            if e.node_type == :element
+              if e.name == 'MessageId'
+                @id = e.text
+              elsif e.name == 'MessageBodyMD5'
+                @body_md5 = e.text
+              elsif e.name == 'MessageBody'
+                @body = e.text
+              elsif e.name == 'EnqueueTime'
+                @enqueue_at = e.text.to_i
+              elsif e.name == 'FirstDequeueTime'
+                @first_enqueue_at = e.text.to_i
+              elsif e.name == 'DequeueCount'
+                @dequeue_count = e.text.to_i
+              elsif e.name == 'Priority'
+                @priority = e.text.to_i
+              elsif e.name == 'ReceiptHandle'
+                @receipt_handle = e.text
+              elsif e.name == 'NextVisibleTime'
+                @next_visible_at = e.text.to_i
+              end
+            end
+          end
+
+          # verify body
+          md5 = Digest::MD5.hexdigest(body).upcase
+          unless md5 == body_md5
+            raise Exception,
+                  'Invalid MNS Body, MD5 does not match, '\
+                  "MD5 #{body_md5}, expect MD5 #{md5}, Body: #{body}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/mns/request.rb

@@ -0,0 +1,81 @@
+require 'uri'
+require 'rexml/document'
+
+module Fluent
+  module Plugin
+    module MNS
+      # Class for Aliyun MNS Request.
+      class Request
+        include REXML
+
+        attr_reader :log, :uri, :method, :body, :content_md5, :content_type,
+                    :content_length, :mns_headers, :access_key_id,
+                    :access_key_secret, :endpoint
+
+        def initialize(opts, headers, params)
+          @log = opts[:log]
+          conf = {
+            host: opts[:endpoint],
+            path: opts[:path]
+          }
+
+          conf[:query] = URI.encode_www_form(params) unless params.empty?
+          @uri = URI::HTTP.build(conf)
+          @method = opts[:method].to_s.downcase
+          @mns_headers = headers.merge('x-mns-version' => '2015-06-06')
+          @access_key_id = opts[:access_key_id]
+          @access_key_secret = opts[:access_key_secret]
+
+          log.info uri.to_s
+        end
+
+        def content(type, values = {})
+          ns = 'http://mns.aliyuncs.com/doc/v1/'
+          builder = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+            xml.send(type.to_sym, xmlns: ns) do |b|
+              values.each { |k, v| b.send k.to_sym, v }
+            end
+          end
+          @body = builder.to_xml
+          @content_md5 = Base64.encode64(Digest::MD5.hexdigest(body)).chop
+          @content_length = body.size
+          @content_type = 'text/xml;charset=utf-8'
+        end
+
+        def execute
+          date = DateTime.now.httpdate
+          headers = {
+            'Authorization' => authorization(date),
+            'Content-Length' => content_length || 0,
+            'Content-Type' => content_type,
+            'Content-MD5' => content_md5,
+            'Date' => date,
+            'Host' => uri.host
+          }.merge(@mns_headers).reject { |k, v| v.nil? }
+
+          begin
+            RestClient.send *[method, uri.to_s, headers, body].compact
+          rescue RestClient::Exception => e
+            doc = Document.new(e.response.to_s)
+            doc.elements[1].each do |e|
+              next unless e.node_type == :element
+              return nil if (e.name == 'Code') && (e.text == 'MessageNotExist')
+            end
+
+            log.error e.response
+
+            raise e
+          end
+        end
+
+        def authorization(date)
+          canonical_resource = [uri.path, uri.query].compact.join('?')
+          canonical_headers = mns_headers.sort.collect { |k, v| "#{k.downcase}:#{v}" }.join("\n")
+          signature = [method.to_s.upcase, content_md5 || '', content_type || '', date, canonical_headers, canonical_resource].join("\n")
+          sha1 = OpenSSL::HMAC.digest('sha1', access_key_secret, signature)
+          "MNS #{access_key_id}:#{Base64.encode64(sha1).chop}"
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_compressor_gzip_command.rb

@@ -0,0 +1,55 @@
+module Fluent
+  module Plugin
+    class OSSOutput
+      # This class uses gzip command to compress chunks.
+      class GzipCommandCompressor < Compressor
+        OSSOutput.register_compressor('gzip_command', self)
+
+        config_param :command_parameter, :string, default: ''
+
+        def configure(conf)
+          super
+
+          check_command('gzip')
+        end
+
+        def ext
+          'gz'.freeze
+        end
+
+        def content_type
+          'application/x-gzip'.freeze
+        end
+
+        def compress(chunk, file)
+          path = if @buffer_type == 'file'
+                   chunk.path
+                 else
+                   out = Tempfile.new('chunk-gzip-out-')
+                   out.binmode
+                   chunk.write_to(out)
+                   out.close
+                   out.path
+                 end
+
+          res = system "gzip #{@command_parameter} -c #{path} > #{file.path}"
+
+          unless res
+            log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+            begin
+              file.truncate(0)
+              gw = Zlib::GzipWriter.new(file)
+              chunk.write_to(gw)
+              gw.close
+            ensure
+              gw.close rescue nil
+            end
+          end
+
+        ensure
+          out.close(true) rescue nil unless @buffer_type == 'file'
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_compressor_lzma2.rb

@@ -0,0 +1,45 @@
+module Fluent
+  module Plugin
+    class OSSOutput
+      # This class uses xz command to compress chunks.
+      class LZMA2Compressor < Compressor
+        OSSOutput.register_compressor('lzma2', self)
+
+        config_param :command_parameter, :string, default: '-qf0'
+
+        def configure(conf)
+          super
+          check_command('xz', 'LZMA2')
+        end
+
+        def ext
+          'xz'.freeze
+        end
+
+        def content_type
+          'application/x-xz'.freeze
+        end
+
+        def compress(chunk, file)
+          path = if @buffer_type == 'file'
+                   chunk.path
+                 else
+                   out = Tempfile.new('chunk-xz-out-')
+                   out.binmode
+                   chunk.write_to(out)
+                   out.close
+                   out.path
+                 end
+
+          res = system "xz #{@command_parameter} -c #{path} > #{file.path}"
+          unless res
+            log.warn "failed to execute xz command, status = #{$?}"
+            raise Fluent::Exception, "failed to execute xz command, status = #{$?}"
+          end
+        ensure
+          out.close(true) rescue nil unless @buffer_type == 'file'
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_compressor_lzo.rb

@@ -0,0 +1,45 @@
+module Fluent
+  module Plugin
+    class OSSOutput
+      # This class uses lzop command to compress chunks.
+      class LZOCompressor < Compressor
+        OSSOutput.register_compressor('lzo', self)
+
+        config_param :command_parameter, :string, default: '-qf1'
+
+        def configure(conf)
+          super
+          check_command('lzop', 'LZO')
+        end
+
+        def ext
+          'lzo'.freeze
+        end
+
+        def content_type
+          'application/x-lzop'.freeze
+        end
+
+        def compress(chunk, file)
+          path = if @buffer_type == 'file'
+                   chunk.path
+                 else
+                   out = Tempfile.new('chunk-lzo-out-')
+                   out.binmode
+                   chunk.write_to(out)
+                   out.close
+                   out.path
+                 end
+
+          res = system "lzop #{@command_parameter} -c #{path} > #{file.path}"
+          unless res
+            log.warn "failed to execute lzop command, status = #{$?}"
+            raise Fluent::Exception, "failed to execute lzop command, status = #{$?}"
+          end
+        ensure
+          out.close(true) rescue nil unless @buffer_type == 'file'
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_decompressor_gzip_command.rb

@@ -0,0 +1,41 @@
+module Fluent
+  module Plugin
+    class OSSInput
+      # This class uses gzip command to decompress chunks.
+      class GzipCommandDecompressor < Decompressor
+        OSSInput.register_decompressor('gzip_command', self)
+
+        config_param :command_parameter, :string, default: '-dc'
+
+        def configure(conf)
+          super
+          check_command('gzip')
+        end
+
+        def ext
+          'gz'.freeze
+        end
+
+        def content_type
+          'application/x-gzip'.freeze
+        end
+
+        def decompress(io)
+          path = io.path
+
+          out, err, status = Open3.capture3("gzip #{@command_parameter} #{path}")
+          if status.success?
+            out
+          else
+            log.warn "failed to execute gzip command, #{err.to_s.gsub("\n",'')}, fallback to GzipReader."
+
+            begin
+              io.rewind
+              Zlib::GzipReader.wrap(io)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_decompressor_lzma2.rb

@@ -0,0 +1,36 @@
+module Fluent
+  module Plugin
+    class OSSInput
+      # This class uses xz command to decompress chunks.
+      class LZMA2Decompressor < Decompressor
+        OSSInput.register_decompressor('lzma2', self)
+
+        config_param :command_parameter, :string, default: '-qdc'
+
+        def configure(conf)
+          super
+          check_command('xz', 'LZMA')
+        end
+
+        def ext
+          'xz'.freeze
+        end
+
+        def content_type
+          'application/x-xz'.freeze
+        end
+
+        def decompress(io)
+          path = io.path
+
+          out, err, status = Open3.capture3("xz #{@command_parameter} #{path}")
+          if status.success?
+            out
+          else
+            raise err.to_s.chomp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/oss_decompressor_lzo.rb

@@ -0,0 +1,36 @@
+module Fluent
+  module Plugin
+    class OSSInput
+      # This class uses lzop command to decompress chunks.
+      class LZODecompressor < Decompressor
+        OSSInput.register_decompressor('lzo', self)
+
+        config_param :command_parameter, :string, default: '-qdc'
+
+        def configure(conf)
+          super
+          check_command('lzop', 'LZO')
+        end
+
+        def ext
+          'lzo'.freeze
+        end
+
+        def content_type
+          'application/x-lzop'.freeze
+        end
+
+        def decompress(io)
+          path = io.path
+
+          out, err, status = Open3.capture3("lzop #{@command_parameter} #{path}")
+          if status.success?
+            out
+          else
+            raise err.to_s.chomp
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_oss.rb

@@ -0,0 +1,423 @@
+require 'fluent/plugin/output'
+require 'aliyun/oss'
+require 'aliyun/sts'
+
+# This is Fluent OSS Output Plugin
+# Usage:
+# In order to write output data to OSS, you should add configurations like below
+# <match pattern>
+#   @type oss
+#   endpoint              <OSS endpoint to connect to>
+#   bucket                <Your bucket name>
+#   access_key_id         <Your access key id>
+#   access_key_secret     <Your access secret key>
+#   path                  <Path prefix of the files on OSS>
+#   key_format            %{path}%{time_slice}_%{index}.%{file_extension}
+# if you want to use ${tag} or %Y/%m/%d/ like syntax in path/key_format,
+# need to specify tag for ${tag} and time for %Y/%m/%d in <buffer> argument.
+#   <buffer tag,time>
+#     @type file
+#     path /var/log/fluent/oss
+#     timekey 3600 # 1 hour partition
+#     timekey_wait 10m
+#     timekey_use_utc true # use utc
+#   </buffer>
+#   <format>
+#     @type json
+#   </format>
+# </match>
+module Fluent
+  # Fluent OSS Plugin
+  module Plugin
+    # OSSOutput class implementation
+    class OSSOutput < Output
+      Fluent::Plugin.register_output('oss', self)
+
+      helpers :compat_parameters, :formatter, :inject
+
+      desc 'OSS endpoint to connect to'
+      config_param :endpoint, :string
+      desc 'Your bucket name'
+      config_param :bucket, :string
+      desc 'Your access key id'
+      config_param :access_key_id, :string
+      desc 'Your access secret key'
+      config_param :access_key_secret, :string
+      desc 'Path prefix of the files on OSS'
+      config_param :path, :string, default: 'fluent/logs'
+      config_param :upload_crc_enable, :bool, default: true
+      config_param :download_crc_enable, :bool, default: true
+      desc 'Timeout for open connections'
+      config_param :open_timeout, :integer, default: 10
+      desc 'Timeout for read response'
+      config_param :read_timeout, :integer, default: 120
+
+      desc 'OSS SDK log directory'
+      config_param :oss_sdk_log_dir, :string, default: '/var/log/td-agent'
+
+      desc 'The format of OSS object keys'
+      config_param :key_format, :string, default: '%{path}/%{time_slice}_%{index}_%{thread_id}.%{file_extension}'
+      desc 'Archive format on OSS'
+      config_param :store_as, :string, default: 'gzip'
+      desc 'Create OSS bucket if it does not exists'
+      config_param :auto_create_bucket, :bool, default: false
+      desc 'Overwrite already existing path'
+      config_param :overwrite, :bool, default: false
+      desc 'Check bucket if exists or not'
+      config_param :check_bucket, :bool, default: true
+      desc 'Check object before creation'
+      config_param :check_object, :bool, default: true
+      desc 'The length of `%{hex_random}` placeholder(4-16)'
+      config_param :hex_random_length, :integer, default: 4
+      desc '`sprintf` format for `%{index}`'
+      config_param :index_format, :string, default: '%d'
+      desc 'Given a threshold to treat events as delay, output warning logs if delayed events were put into OSS'
+      config_param :warn_for_delay, :time, default: nil
+
+      DEFAULT_FORMAT_TYPE = 'out_file'.freeze
+
+      config_section :format do
+        config_set_default :@type, DEFAULT_FORMAT_TYPE
+      end
+
+      config_section :buffer do
+        config_set_default :chunk_keys, ['time']
+        config_set_default :timekey, (60 * 60 * 24)
+      end
+
+      MAX_HEX_RANDOM_LENGTH = 16
+
+      def configure(conf)
+        compat_parameters_convert(conf, :buffer, :formatter, :inject)
+
+        super
+
+        raise Fluent::ConfigError, 'Invalid oss endpoint' if @endpoint.nil?
+
+        if @hex_random_length > MAX_HEX_RANDOM_LENGTH
+          raise Fluent::ConfigError, 'hex_random_length parameter must be '\
+                "less than or equal to #{MAX_HEX_RANDOM_LENGTH}"
+        end
+
+        unless @index_format =~ /^%(0\d*)?[dxX]$/
+          raise Fluent::ConfigError, 'index_format parameter should follow '\
+                '`%[flags][width]type`. `0` is the only supported flag, '\
+                'and is mandatory if width is specified. '\
+                '`d`, `x` and `X` are supported types'
+        end
+
+        begin
+          @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(buffer_type: @buffer_config[:@type], log: log)
+        rescue StandardError => e
+          log.warn "'#{@store_as}' not supported. Use 'text' instead: error = #{e.message}"
+          @compressor = TextCompressor.new
+        end
+
+        @compressor.configure(conf)
+
+        @formatter = formatter_create
+
+        process_key_format
+
+        unless @check_object
+          if config.has_key?('key_format')
+            log.warn "set 'check_object false' and key_format is "\
+                    'specified. Check key_format is unique in each '\
+                    'write. If not, existing file will be overwritten.'
+          else
+            log.warn "set 'check_object false' and key_format is "\
+                    'not specified. Use '\
+                    "'%{path}/%{time_slice}_%{hms_slice}_%{thread_id}.%{file_extension}' "\
+                    'for key_format'
+            @key_format = '%{path}/%{time_slice}_%{hms_slice}_%{thread_id}.%{file_extension}'
+          end
+        end
+
+        @configured_time_slice_format = conf['time_slice_format']
+        @values_for_oss_object_chunk = {}
+        @time_slice_with_tz = Fluent::Timezone.formatter(
+          @timekey_zone,
+          @configured_time_slice_format || timekey_to_timeformat(@buffer_config['timekey']))
+      end
+
+      def timekey_to_timeformat(timekey)
+        case timekey
+        when nil          then ''
+        when 0...60       then '%Y%m%d-%H_%M_%S' # 60 exclusive
+        when 60...3600    then '%Y%m%d-%H_%M'
+        when 3600...86400 then '%Y%m%d-%H'
+        else                   '%Y%m%d'
+        end
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def initialize
+        super
+        @compressor = nil
+        @uuid_flush_enabled = false
+      end
+
+      def start
+        @oss_sdk_log_dir += '/' unless @oss_sdk_log_dir.end_with?('/')
+        Aliyun::Common::Logging.set_log_file(@oss_sdk_log_dir + Aliyun::Common::Logging::DEFAULT_LOG_FILE)
+        create_oss_client unless @oss
+
+        ensure_bucket if @check_bucket
+        super
+      end
+
+      def format(tag, time, record)
+        r = inject_values_to_record(tag, time, record)
+        @formatter.format(tag, time, r)
+      end
+
+      def write(chunk)
+        index = 0
+        metadata = chunk.metadata
+        time_slice = if metadata.timekey.nil?
+                       ''.freeze
+                     else
+                       @time_slice_with_tz.call(metadata.timekey)
+                     end
+
+        @values_for_oss_object_chunk[chunk.unique_id] ||= {
+          '%{hex_random}' => hex_random(chunk)
+        }
+
+        if @check_object
+          exist_key = nil
+          begin
+            values_for_oss_key = {
+              '%{path}' => @path,
+              '%{thread_id}' => Thread.current.object_id.to_s,
+              '%{file_extension}' => @compressor.ext,
+              '%{time_slice}' => time_slice,
+              '%{index}' => sprintf(@index_format, index)
+            }.merge!(@values_for_oss_object_chunk[chunk.unique_id])
+
+            values_for_oss_key['%{uuid_flush}'.freeze] = uuid_random if @uuid_flush_enabled
+
+            key = @key_format.gsub(/%{[^}]+}/) do |matched_key|
+              values_for_oss_key.fetch(matched_key, matched_key)
+            end
+            key = extract_placeholders(key, chunk)
+            key = key.gsub(/%{[^}]+}/, values_for_oss_key)
+
+            if (index > 0) && (key == exist_key)
+              if @overwrite
+                log.warn "#{key} already exists, but will overwrite"
+                break
+              else
+                raise "duplicated path is generated. use %{index} in key_format: path = #{key}"
+              end
+            end
+
+            index += 1
+            exist_key = key
+          end while @bucket_handler.object_exists?(key)
+        else
+          hms_slice = Time.now.utc.strftime('%H%M%S')
+          hms_slice = Time.now.strftime('%H%M%S') if @local_time
+
+          values_for_oss_key = {
+            '%{path}' => @path,
+            '%{thread_id}' => Thread.current.object_id.to_s,
+            '%{file_extension}' => @compressor.ext,
+            '%{time_slice}' => time_slice,
+            '%{hms_slice}' => hms_slice
+          }.merge!(@values_for_oss_object_chunk[chunk.unique_id])
+
+          values_for_oss_key['%{uuid_flush}'.freeze] = uuid_random if @uuid_flush_enabled
+
+          key = @key_format.gsub(/%{[^}]+}/) do |matched_key|
+            values_for_oss_key.fetch(matched_key, matched_key)
+          end
+          key = extract_placeholders(key, chunk)
+          key = key.gsub(/%{[^}]+}/, values_for_oss_key)
+        end
+
+        out_file = Tempfile.new('oss-fluent-')
+        out_file.binmode
+        begin
+          @compressor.compress(chunk, out_file)
+          out_file.rewind
+          log.info "out_oss: write chunk #{dump_unique_id_hex(chunk.unique_id)} with metadata #{chunk.metadata} to oss://#{@bucket}/#{key}, size #{out_file.size}"
+
+          start = Time.now.to_i
+          @bucket_handler.put_object(key, file: out_file, content_type: @compressor.content_type)
+
+          log.debug "out_oss: write oss://#{@bucket}/#{key} used #{Time.now.to_i - start} seconds, size #{out_file.length}"
+          @values_for_oss_object_chunk.delete(chunk.unique_id)
+
+          if @warn_for_delay
+            if Time.at(chunk.metadata.timekey) < Time.now - @warn_for_delay
+              log.warn "out_oss: delayed events were put to oss://#{@bucket}/#{key}"
+            end
+          end
+        ensure
+          out_file.close(true) rescue nil
+        end
+      end
+
+      def create_oss_client
+        @oss = Aliyun::OSS::Client.new(
+          endpoint: @endpoint,
+          access_key_id: @access_key_id,
+          access_key_secret: @access_key_secret,
+          download_crc_enable: @download_crc_enable,
+          upload_crc_enable: @upload_crc_enable,
+          open_timeout: @open_timeout,
+          read_timeout: @read_timeout
+        )
+      end
+
+      def process_key_format
+        if @key_format.include?('%{uuid_flush}')
+          # verify uuidtools
+          begin
+            require 'uuidtools'
+          rescue LoadError
+            raise Fluent::ConfigError, 'uuidtools gem not found.'\
+                  ' Install uuidtools gem first'
+          end
+
+          begin
+            uuid_random
+          rescue => e
+            raise Fluent::ConfigError, "generating uuid doesn't work. "\
+                  "Can't use %{uuid_flush} on this environment. #{e}"
+          end
+
+          @uuid_flush_enabled = true
+        end
+      end
+
+      def uuid_random
+        ::UUIDTools::UUID.random_create.to_s
+      end
+
+      def hex_random(chunk)
+        unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
+        # unique_hex is like (time_sec, time_usec, rand) => reversing gives more randomness
+        unique_hex.reverse!
+        unique_hex[0...@hex_random_length]
+      end
+
+      def ensure_bucket
+        unless @oss.bucket_exist?(@bucket)
+          if @auto_create_bucket
+            log.info "creating bucket #{@bucket} on #{@endpoint}"
+            @oss.create_bucket(@bucket)
+          else
+            raise "the specified bucket does not exist: bucket = #{@bucket}"
+          end
+        end
+
+        @bucket_handler = @oss.get_bucket(@bucket)
+      end
+
+      # Compression base class.
+      class Compressor
+        include Fluent::Configurable
+
+        attr_reader :log
+
+        def initialize(opts = {})
+          super()
+          @buffer_type = opts[:buffer_type]
+          @log = opts[:log]
+        end
+
+        def configure(conf)
+          super
+        end
+
+        def ext; end
+
+        def content_type; end
+
+        def compress(chunk, file); end
+
+        private
+
+        def check_command(command, encode = nil)
+          require 'open3'
+
+          encode = command if encode.nil?
+          begin
+            Open3.capture3("#{command} -V")
+          rescue Errno::ENOENT
+            raise Fluent::ConfigError,
+                  "'#{command}' utility must be in PATH for #{encode} compression"
+          end
+        end
+      end
+
+      # Gzip compression.
+      class GzipCompressor < Compressor
+        def ext
+          'gz'.freeze
+        end
+
+        def content_type
+          'application/x-gzip'.freeze
+        end
+
+        def compress(chunk, file)
+          out = Zlib::GzipWriter.new(file)
+          chunk.write_to(out)
+          out.finish
+        ensure
+          begin
+            out.finish
+          rescue StandardError
+            nil
+          end
+        end
+      end
+
+      # Text output format.
+      class TextCompressor < Compressor
+        def ext
+          'txt'.freeze
+        end
+
+        def content_type
+          'text/plain'.freeze
+        end
+
+        def compress(chunk, file)
+          chunk.write_to(file)
+        end
+      end
+
+      # Json compression.
+      class JsonCompressor < TextCompressor
+        def ext
+          'json'.freeze
+        end
+
+        def content_type
+          'application/json'.freeze
+        end
+      end
+
+      COMPRESSOR_REGISTRY = Fluent::Registry.new(:oss_compressor_type,
+                                                 'fluent/plugin/oss_compressor_')
+      {
+        'gzip' => GzipCompressor,
+        'json' => JsonCompressor,
+        'text' => TextCompressor
+      }.each do |name, compressor|
+        COMPRESSOR_REGISTRY.register(name, compressor)
+      end
+
+      def self.register_compressor(name, compressor)
+        COMPRESSOR_REGISTRY.register(name, compressor)
+      end
+    end
+  end
+end