fluent-plugin-s3 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e12d64135d362d4053421a2e3d9ac0b90f01582
-  data.tar.gz: cdf020286a2e57aa6cd21b25629db8d781a9284f
+  metadata.gz: 3db75bd6afaa816615c066329d862d2134aa63cb
+  data.tar.gz: 29a26bf2d9d7dc34d915a0f5444f5f510547df1e
 SHA512:
-  metadata.gz: 33b2ea7ceffef945de20e6ea4602c01c750d3c7a4e01329401cf39ab549cc4a7d0c8f28e5ea51e727a3e227a5e13d1616cb1a9b3e2d0c6086451da0e054073dd
-  data.tar.gz: 921e4017bd14f0fe3b17cde337a1608fd6aeb20e33f2e2b1b0b5adeee0cecd66141221652ee947fc44bea3ac209cbe3e17523b2ff30707bbc1b0a78cfcd60067
+  metadata.gz: c10a4b62d9ebab53a1fd3535001d972f46df95ce0df20432df9b73a81aac5a379d1e8967132bb557386030078ec95991658e651cb28963a294c2ae36c8b52899
+  data.tar.gz: e46f3df727e359413cb8ec6b4bc8697346ccc2f58ebf5e70a363f9c3b463052ce974be574c7ffc3ccee0f2174e060fd0b2b49c7d8f2da0f63306d68fd28dde1a

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 0.5.0 - 2014/12/07
+
+  * Make compression algorithm pluggable
+  * Remove format_json parameter
+  * Remove s3_endpoint parameter
+  * Relax fluentd version restriction to support 0.12 or later
+
+
 Release 0.4.3 - 2014/11/10
 
   * Change API check message to consider region mismatch

data/README.rdoc

@@ -21,7 +21,7 @@ Simply use RubyGems:
       aws_key_id YOUR_AWS_KEY_ID
       aws_sec_key YOUR_AWS_SECRET/KEY
       s3_bucket YOUR_S3_BUCKET_NAME
-      s3_endpoint s3-ap-northeast-1.amazonaws.com
+      s3_region ap-northeast-1
       s3_object_key_format %{path}%{time_slice}_%{index}.%{file_extension}
       path logs/
       buffer_path /var/log/fluent/s3
@@ -39,8 +39,6 @@ Simply use RubyGems:
 
 [s3_region] s3 region name. For example, US West (Oregon) Region is "us-west-2". The full list of regions are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region. We recommend using `s3_region` instead of `s3_endpoint`.
 
-[s3_endpoint] s3 endpoint name. For example, US West (Oregon) Region is "s3-us-west-2.amazonaws.com". The full list of endpoints are available here. > http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
-
 [s3_object_key_format] The format of S3 object keys. You can use several built-in variables:
 
 - %{path}
@@ -88,6 +86,8 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
 - text
 - lzo (Need lzop command)
 
+See 'Use your compression algorithm' section for adding another format.
+
 [format] Change one line format in the S3 object. Supported formats are "out_file", "json", "ltsv" and "single_value".
 
 - out_file (default).
@@ -106,7 +106,7 @@ At this format, "time" and "tag" are omitted.
 But you can set these information to the record by setting "include_tag_key" / "tag_key" and "include_time_key" / "time_key" option.
 If you set following configuration in S3 output:
 
-    format_json true
+    format json
     include_time_key true
     time_key log_time # default is time
 
@@ -165,6 +165,37 @@ Refer to the {AWS documentation}[http://docs.aws.amazon.com/IAM/latest/UserGuide
 
 Using {IAM roles}[http://docs.aws.amazon.com/IAM/latest/UserGuide/WorkingWithRoles.html] with a properly configured IAM policy are preferred over embedding access keys on EC2 instances.
 
+== Use your compression algorithm
+
+s3 plugin has plugabble compression mechanizm like Fleuntd\'s input / output plugin.
+If you set 'store_as xxx', s3 plugin searches `fluent/plugin/s3_compressor_xxx.rb`.
+You can define your compression with 'S3Output::Compressor' class. Compressor API is here:
+
+    module Fluent
+      class S3Output
+        class XXXCompressor < Compressor
+          S3Output.register_compressor('xxx', self)
+
+          # Used to file extension
+          def ext
+            'xxx'
+          end
+
+          # Used to file content type
+          def content_type
+            'application/x-xxx'
+          end
+
+          # chunk is buffer chunk. tmp is destination file for upload
+          def compress(chunk, tmp)
+            # call command or something
+          end
+        end
+      end
+    end
+
+See bundled Compressor classes for more detail.
+
 == Website, license, et. al.
 
 Web site::  http://fluentd.org/

data/VERSION

@@ -1 +1 @@
-0.4.3
+0.5.0

data/fluent-plugin-s3.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", "~> 0.10.49"
+  gem.add_dependency "fluentd", [">= 0.10.49", "< 2"]
   gem.add_dependency "aws-sdk", "~> 1.38"
   gem.add_dependency "yajl-ruby", "~> 1.0"
   gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"

data/lib/fluent/plugin/out_s3.rb

@@ -4,31 +4,24 @@ module Fluent
   class S3Output < Fluent::TimeSlicedOutput
     Fluent::Plugin.register_output('s3', self)
 
-    unless method_defined?(:log)
-      define_method(:log) { $log }
-    end
-
     def initialize
       super
       require 'aws-sdk'
       require 'zlib'
       require 'time'
       require 'tempfile'
-      require 'open3'
 
-      @use_ssl = true
+      @compressor = nil
     end
 
     config_param :path, :string, :default => ""
-
+    config_param :use_ssl, :bool, :default => true
     config_param :aws_key_id, :string, :default => nil
     config_param :aws_sec_key, :string, :default => nil
     config_param :s3_bucket, :string
     config_param :s3_region, :string, :default => nil
-    config_param :s3_endpoint, :string, :default => nil
     config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
     config_param :store_as, :string, :default => "gzip"
-    config_param :command_parameter, :string, :default => nil
     config_param :auto_create_bucket, :bool, :default => true
     config_param :check_apikey_on_start, :bool, :default => true
     config_param :proxy_uri, :string, :default => nil
@@ -46,40 +39,20 @@ module Fluent
     def configure(conf)
       super
 
-      if use_ssl = conf['use_ssl']
-        if use_ssl.empty?
-          @use_ssl = true
-        else
-          @use_ssl = Config.bool_value(use_ssl)
-          if @use_ssl.nil?
-            raise ConfigError, "'true' or 'false' is required for use_ssl option on s3 output"
-          end
-        end
+      if conf.has_key?('s3_endpoint')
+        raise ConfigError, "s3_endpoint parameter is removed. Use s3_region instead"
       end
 
-      @ext, @mime_type = case @store_as
-                         when 'gzip'
-                           ['gz', 'application/x-gzip']
-                         when 'lzo'
-                           check_command('lzop', 'LZO')
-                           @command_parameter = '-qf1' if @command_parameter.nil?
-                           ['lzo', 'application/x-lzop']
-                         when 'lzma2'
-                           check_command('xz', 'LZMA2')
-                           @command_parameter = '-qf0' if @command_parameter.nil?
-                           ['xz', 'application/x-xz']
-                         when 'json'
-                           ['json', 'application/json']
-                         else
-                           ['txt', 'text/plain']
-                         end
-
-      if format_json = conf['format_json']
-        $log.warn "format_json is deprecated. Use 'format json' instead"
-        conf['format'] = 'json'
-      else
-        conf['format'] = @format
+      begin
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new
+      rescue => e
+        $log.warn "#{@store_as} not found. Use 'text' instead"
+        @compressor = TextCompressor.new
       end
+      @compressor.configure(conf)
+
+      # TODO: use Plugin.new_formatter instead of TextFormatter.create
+      conf['format'] = @format
       @formatter = TextFormatter.create(conf)
 
       if @localtime
@@ -101,7 +74,6 @@ module Fluent
         options[:secret_access_key] = @aws_sec_key
       end
       options[:region] = @s3_region if @s3_region
-      options[:endpoint] = @s3_endpoint if @s3_endpoint
       options[:proxy_uri] = @proxy_uri if @proxy_uri
       options[:use_ssl] = @use_ssl
 
@@ -125,7 +97,7 @@ module Fluent
         values_for_s3_object_key = {
           "path" => path,
           "time_slice" => chunk.key,
-          "file_extension" => @ext,
+          "file_extension" => @compressor.ext,
           "index" => i
         }
         s3path = @s3_object_key_format.gsub(%r(%{[^}]+})) { |expr|
@@ -141,33 +113,11 @@ module Fluent
 
       tmp = Tempfile.new("s3-")
       begin
-        if @store_as == "gzip"
-          w = Zlib::GzipWriter.new(tmp)
-          chunk.write_to(w)
-          w.close
-        elsif @store_as == "lzo"
-          w = Tempfile.new("chunk-tmp")
-          chunk.write_to(w)
-          w.close
-          tmp.close
-          # We don't check the return code because we can't recover lzop failure.
-          system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
-        elsif @store_as == "lzma2"
-          w = Tempfile.new("chunk-xz-tmp")
-          chunk.write_to(w)
-          w.close
-          tmp.close
-          system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
-        else
-          chunk.write_to(tmp)
-          tmp.close
-        end
-        @bucket.objects[s3path].write(Pathname.new(tmp.path), {:content_type => @mime_type,
+        @compressor.compress(chunk, tmp)
+        @bucket.objects[s3path].write(Pathname.new(tmp.path), {:content_type => @compressor.content_type,
                                                                :reduced_redundancy => @reduced_redundancy})
       ensure
         tmp.close(true) rescue nil
-        w.close rescue nil
-        w.unlink rescue nil
       end
     end
 
@@ -190,12 +140,91 @@ module Fluent
       raise "can't call S3 API. Please check your aws_key_id / aws_sec_key or s3_region configuration"
     end
 
-    def check_command(command, algo)
-      begin
-        Open3.capture3("#{command} -V")
-      rescue Errno::ENOENT
-        raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+    class Compressor
+      include Configurable
+
+      def configure(conf)
+        super
+      end
+
+      def ext
+      end
+
+      def content_type
+      end
+
+      def compress(chunk, tmp)
+      end
+
+      private
+
+      def check_command(command, algo = nil)
+        require 'open3'
+
+        algo = command if algo.nil?
+        begin
+          Open3.capture3("#{command} -V")
+        rescue Errno::ENOENT
+          raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+        end
       end
     end
+
+    class GzipCompressor < Compressor
+      def ext
+        'gz'.freeze
+      end
+
+      def content_type
+        'application/x-gzip'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Zlib::GzipWriter.new(tmp)
+        chunk.write_to(w)
+        w.close
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+
+    class TextCompressor < Compressor
+      def ext
+        'txt'.freeze
+      end
+
+      def content_type
+        'text/plain'.freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk.write_to(tmp)
+        tmp.close
+      end
+    end
+
+    class JsonCompressor < TextCompressor
+      def ext
+        'json'.freeze
+      end
+
+      def content_type
+        'application/json'.freeze
+      end
+    end
+
+    COMPRESSOR_REGISTRY = Registry.new(:s3_compressor_type, 'fluent/plugin/s3_compressor_')
+    {
+      'gzip' => GzipCompressor,
+      'json' => JsonCompressor,
+      'text' => TextCompressor
+    }.each { |name, compressor|
+      COMPRESSOR_REGISTRY.register(name, compressor)
+    }
+
+    def self.register_compressor(name, compressor)
+      COMPRESSOR_REGISTRY.register(name, compressor)
+    end
   end
 end

data/lib/fluent/plugin/s3_compressor_lzma2.rb

@@ -0,0 +1,34 @@
+module Fluent
+  class S3Output
+    class LZMA2Compressor < Compressor
+      S3Output.register_compressor('lzma2', self)
+
+      config_param :command_parameter, :string, :default => '-qf0'
+
+      def configure(conf)
+        super
+        check_command('xz', 'LZMA2')
+      end
+
+      def ext
+        'xz'.freeze
+      end
+
+      def content_type
+        'application/x-xz'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-xz-tmp")
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        # We don't check the return code because we can't recover lzop failure.
+        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/s3_compressor_lzo.rb

@@ -0,0 +1,34 @@
+module Fluent
+  class S3Output
+    class LZOCompressor < Compressor
+      S3Output.register_compressor('lzo', self)
+
+      config_param :command_parameter, :string, :default => '-qf1'
+
+      def configure(conf)
+        super
+        check_command('lzop', 'LZO')
+      end
+
+      def ext
+        'lzo'.freeze
+      end
+
+      def content_type
+        'application/x-lzop'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-tmp")
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        # We don't check the return code because we can't recover lzop failure.
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/test/test_out_s3.rb

@@ -42,32 +42,32 @@ class S3OutputTest < Test::Unit::TestCase
     assert_equal 'test_bucket', d.instance.s3_bucket
     assert_equal 'log', d.instance.path
     assert d.instance.instance_variable_get(:@use_ssl)
-    assert_equal 'gz', d.instance.instance_variable_get(:@ext)
-    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@mime_type)
+    assert_equal 'gz', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/x-gzip', d.instance.instance_variable_get(:@compressor).content_type
   end
 
   def test_configure_with_mime_type_json
     conf = CONFIG.clone
     conf << "\nstore_as json\n"
     d = create_driver(conf)
-    assert_equal 'json', d.instance.instance_variable_get(:@ext)
-    assert_equal 'application/json', d.instance.instance_variable_get(:@mime_type)
+    assert_equal 'json', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/json', d.instance.instance_variable_get(:@compressor).content_type
   end
 
   def test_configure_with_mime_type_text
     conf = CONFIG.clone
     conf << "\nstore_as text\n"
     d = create_driver(conf)
-    assert_equal 'txt', d.instance.instance_variable_get(:@ext)
-    assert_equal 'text/plain', d.instance.instance_variable_get(:@mime_type)
+    assert_equal 'txt', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'text/plain', d.instance.instance_variable_get(:@compressor).content_type
   end
 
   def test_configure_with_mime_type_lzo
     conf = CONFIG.clone
     conf << "\nstore_as lzo\n"
     d = create_driver(conf)
-    assert_equal 'lzo', d.instance.instance_variable_get(:@ext)
-    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@mime_type)
+    assert_equal 'lzo', d.instance.instance_variable_get(:@compressor).ext
+    assert_equal 'application/x-lzop', d.instance.instance_variable_get(:@compressor).content_type
   rescue => e
     # TODO: replace code with disable lzop command
     assert(e.is_a?(Fluent::ConfigError))
@@ -147,22 +147,8 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
   end
 
-  def test_format_with_format_json_deprecated
-    config = [CONFIG, 'format_json true'].join("\n")
-    d = create_driver(config)
-
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
-
-    d.expect_format %[{"a":1}\n]
-    d.expect_format %[{"a":2}\n]
-
-    d.run
-  end
-
   def test_format_with_format_json_included_tag
-    config = [CONFIG, 'format_json true', 'include_tag_key true'].join("\n")
+    config = [CONFIG, 'format json', 'include_tag_key true'].join("\n")
     d = create_driver(config)
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -9,22 +9,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-11 00:00:00.000000000 Z
+date: 2014-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.49
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.49
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -110,6 +116,8 @@ files:
 - VERSION
 - fluent-plugin-s3.gemspec
 - lib/fluent/plugin/out_s3.rb
+- lib/fluent/plugin/s3_compressor_lzma2.rb
+- lib/fluent/plugin/s3_compressor_lzo.rb
 - test/test_out_s3.rb
 homepage: https://github.com/fluent/fluent-plugin-s3
 licenses: []