fluent-plugin-s3 0.3.6 → 0.3.7

data/.travis.yml

@@ -5,7 +5,7 @@ rvm:
   - 1.9.3
   - 2.0.0
   - 2.1.0
-  - rbx-19mode
+  - rbx
 
 gemfile:
  - Gemfile
@@ -19,4 +19,4 @@ script: bundle exec rake test
 
 matrix:
   allow_failures:
-    - rvm: rbx-19mode
+    - rvm: rbx

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 0.3.7 - 2014/03/07
+
+  * Support lzma2 compression using 'xz' command
+    https://github.com/fluent/fluent-plugin-s3/pull/41
+  * Relax aws-sdk gem requirement
+    https://github.com/fluent/fluent-plugin-s3/pull/42
+
+
 Release 0.3.6 - 2014/02/05
 
   * Support 'log_level' option

data/README.rdoc

@@ -1,4 +1,4 @@
-= Amazon S3 output plugin for Fluent event collector
+= Amazon S3 output plugin for {Fluentd}[http://github.com/fluent/fluentd]
 
 == Overview
 
@@ -103,8 +103,12 @@ The {fluent-mixin-config-placeholders}[https://github.com/tagomoris/fluent-mixin
 [utc] Use UTC instead of local time.
 
 
-== Copyright
-
-Copyright:: Copyright (c) 2011 Sadayuki Furuhashi
-License::   Apache License, Version 2.0
+== Website, license, et. al.
 
+Web site::  http://fluentd.org/
+Documents:: http://docs.fluentd.org/
+Source repository:: http://github.com/fluent
+Discussion:: http://groups.google.com/group/fluentd
+Author:: Sadayuki Furuhashi
+Copyright:: (c) 2011 FURUHASHI Sadayuki
+License:: Apache License, Version 2.0

data/VERSION

@@ -1 +1 @@
-0.3.6
+0.3.7

data/fluent-plugin-s3.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ['lib']
 
   gem.add_dependency "fluentd", "~> 0.10.0"
-  gem.add_dependency "aws-sdk", "~> 1.8.2"
+  gem.add_dependency "aws-sdk", ">= 1.8.2"
   gem.add_dependency "yajl-ruby", "~> 1.0"
   gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
   gem.add_development_dependency "rake", ">= 0.9.2"

data/lib/fluent/plugin/out_s3.rb

@@ -35,6 +35,7 @@ class S3Output < Fluent::TimeSlicedOutput
   config_param :s3_endpoint, :string, :default => nil
   config_param :s3_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
   config_param :store_as, :string, :default => "gzip"
+  config_param :command_parameter, :string, :default => nil
   config_param :auto_create_bucket, :bool, :default => true
   config_param :check_apikey_on_start, :bool, :default => true
   config_param :proxy_uri, :string, :default => nil
@@ -69,17 +70,21 @@ class S3Output < Fluent::TimeSlicedOutput
     end
 
     @ext, @mime_type = case @store_as
-      when 'gzip' then ['gz', 'application/x-gzip']
-      when 'lzo' then
-        begin
-          Open3.capture3('lzop -V')
-        rescue Errno::ENOENT
-          raise ConfigError, "'lzop' utility must be in PATH for LZO compression"
-        end
-        ['lzo', 'application/x-lzop']
-      when 'json' then ['json', 'application/json']
-      else ['txt', 'text/plain']
-    end
+                       when 'gzip'
+                         ['gz', 'application/x-gzip']
+                       when 'lzo'
+                         check_command('lzop', 'LZO')
+                         @command_parameter = '-qf1' if @command_parameter.nil?
+                         ['lzo', 'application/x-lzop']
+                       when 'lzma2'
+                         check_command('xz', 'LZMA2')
+                         @command_parameter = '-qf0' if @command_parameter.nil?
+                         ['xz', 'application/x-xz']
+                       when 'json'
+                         ['json', 'application/json']
+                       else
+                         ['txt', 'text/plain']
+                       end
 
     @timef = TimeFormatter.new(@time_format, @localtime)
 
@@ -109,8 +114,8 @@ class S3Output < Fluent::TimeSlicedOutput
     @s3 = AWS::S3.new(options)
     @bucket = @s3.buckets[@s3_bucket]
 
-    ensure_bucket
     check_apikeys if @check_apikey_on_start
+    ensure_bucket
   end
 
   def format(tag, time, record)
@@ -162,7 +167,13 @@ class S3Output < Fluent::TimeSlicedOutput
         w.close
         tmp.close
         # We don't check the return code because we can't recover lzop failure.
-        system "lzop -qf1 -o #{tmp.path} #{w.path}"
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      elsif @store_as == "lzma2"
+        w = Tempfile.new("chunk-xz-tmp")
+        chunk.write_to(w)
+        w.close
+        tmp.close
+        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
       else
         chunk.write_to(tmp)
         tmp.close
@@ -194,6 +205,14 @@ class S3Output < Fluent::TimeSlicedOutput
   rescue
     raise "aws_key_id or aws_sec_key is invalid. Please check your configuration"
   end
+
+  def check_command(command, algo)
+    begin
+      Open3.capture3("#{command} -V")
+    rescue Errno::ENOENT
+      raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+    end
+  end
 end
 
 

data/test/out_s3.rb

@@ -27,6 +27,9 @@ class S3OutputTest < Test::Unit::TestCase
 
       private
 
+      def ensure_bucket
+      end
+
       def check_apikeys
       end
     end.configure(conf)
@@ -214,8 +217,7 @@ class S3OutputTest < Test::Unit::TestCase
   def test_write_with_custom_s3_object_key_format
     # Assert content of event logs which are being sent to S3
     s3obj = flexmock(AWS::S3::S3Object)
-    s3obj.should_receive(:exists?).with_any_args.
-      and_return { false }
+    s3obj.should_receive(:exists?).with_any_args.and_return { false }
     s3obj.should_receive(:write).with(
       on { |pathname|
         data = nil
@@ -226,7 +228,7 @@ class S3OutputTest < Test::Unit::TestCase
           gz.close
         }
         assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
-                         %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
+                     %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
                      data
 
         pathname.to_s.match(%r|s3-|)
@@ -244,12 +246,8 @@ class S3OutputTest < Test::Unit::TestCase
       }
 
     # Partial mock the S3Bucket, not to make an actual connection to Amazon S3
-    flexmock(AWS::S3::Bucket).new_instances do |bucket|
-      bucket.should_receive(:objects).with_any_args.
-        and_return {
-          s3obj_col
-        }
-    end
+    s3bucket, _ = setup_mocks(true)
+    s3bucket.should_receive(:objects).with_any_args.and_return { s3obj_col }
 
     # We must use TimeSlicedOutputTestDriver instead of BufferedOutputTestDriver,
     # to make assertions on chunks' keys
@@ -263,9 +261,9 @@ class S3OutputTest < Test::Unit::TestCase
     d.run
   end
 
-  def setup_mocks
+  def setup_mocks(exists_return = false)
     s3bucket = flexmock(AWS::S3::Bucket)
-    s3bucket.should_receive(:exists?).with_any_args.and_return { false }
+    s3bucket.should_receive(:exists?).with_any_args.and_return { exists_return }
     s3bucket_col = flexmock(AWS::S3::BucketCollection)
     s3bucket_col.should_receive(:[]).with_any_args.and_return { s3bucket }
     flexmock(AWS::S3).new_instances do |bucket|

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
+  prerelease: 
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,20 +30,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.8.2
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fluent-mixin-config-placeholders
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -69,29 +78,33 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.2.0
 description: Amazon S3 output plugin for Fluentd event collector
@@ -113,26 +126,33 @@ files:
 - test/out_s3.rb
 homepage: https://github.com/fluent/fluent-plugin-s3
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -3014844547885413145
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: -3014844547885413145
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Amazon S3 output plugin for Fluentd event collector
 test_files:
 - test/out_s3.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 7f3399bf6de1662008eb3172d029a449f367a8f9
-  data.tar.gz: 533c1fdc074a07ae577463bc342fd537f25919e1
-SHA512:
-  metadata.gz: 1e4e4cf633290fe4630797453305199cb884a96722cf0bc0cdeb96dde0c9b759aae36fa18453872938c8c2d775047972950356371f400781ab446b418c6e2682
-  data.tar.gz: f8f2128b80bb3fc305f53db40ff30172d5b8fbb21406bdd1c9055c7b7c9e4c3bbac197aee160a499f86079ed6a29e34ff44b3d5300df360a95528c196f11c673