fluent-plugin-azurestorage 0.0.7 → 0.0.8

data/Rakefile

@@ -1,14 +1,14 @@
-
-require 'bundler'
-Bundler::GemHelper.install_tasks
-
-require 'rake/testtask'
-
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/test_*.rb']
-  test.verbose = true
-end
-
-task :default => [:build]
-
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -1 +1 @@
-0.0.7
+0.0.8

data/fluent-plugin-azurestorage.gemspec

@@ -1,26 +1,26 @@
-# encoding: utf-8
-$:.push File.expand_path('../lib', __FILE__)
-
-Gem::Specification.new do |gem|
-  gem.name        = "fluent-plugin-azurestorage"
-  gem.description = "Azure Storage output plugin for Fluentd event collector"
-  gem.license     = "Apache-2.0"
-  gem.homepage    = "https://github.com/htgc/fluent-plugin-azurestorage"
-  gem.summary     = gem.description
-  gem.version     = File.read("VERSION").strip
-  gem.authors     = ["Hidemasa Togashi"]
-  gem.email       = ["togachiro@gmail.com"]
-  gem.has_rdoc    = false
-  #gem.platform    = Gem::Platform::RUBY
-  gem.files       = `git ls-files`.split("\n")
-  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
-  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  gem.require_paths = ['lib']
-
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_dependency "azure", "0.6.4"
-  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
-  gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency "test-unit", ">= 3.0.8"
-  gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
-end
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-azurestorage"
+  gem.description = "Azure Storage output plugin for Fluentd event collector"
+  gem.license     = "Apache-2.0"
+  gem.homepage    = "https://github.com/htgc/fluent-plugin-azurestorage"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Hidemasa Togashi"]
+  gem.email       = ["togachiro@gmail.com"]
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
+  gem.add_dependency "azure", [">= 0.7.1", "<= 0.7.7"]
+  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "test-unit-rr", ">= 1.0.3"
+end

data/lib/fluent/plugin/azurestorage_compressor_gzip_command.rb

@@ -1,51 +1,51 @@
-module Fluent
-  class AzureStorageOutput
-    class GzipCommandCompressor < Compressor
-      AzureStorageOutput.register_compressor('gzip_command', self)
-
-      config_param :command_parameter, :string, :default => ''
-
-      def configure(conf)
-        super
-        check_command('gzip')
-      end
-
-      def ext
-        'gz'.freeze
-      end
-
-      def content_type
-        'application/x-gzip'.freeze
-      end
-
-      def compress(chunk, tmp)
-        chunk_is_file = @buffer_type == 'file'
-        path = if chunk_is_file
-                 chunk.path
-               else
-                 w = Tempfile.new("chunk-gzip-tmp")
-                 chunk.write_to(w)
-                 w.close
-                 w.path
-               end
-
-        res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
-        unless res
-          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
-          begin
-            tmp.truncate(0)
-            gw = Zlib::GzipWriter.new(tmp)
-            chunk.write_to(gw)
-            gw.close
-          ensure
-            gw.close rescue nil
-          end
-        end
-      ensure
-        unless chunk_is_file
-          w.close(true) rescue nil
-        end
-      end
-    end
-  end
-end
+module Fluent
+  class AzureStorageOutput
+    class GzipCommandCompressor < Compressor
+      AzureStorageOutput.register_compressor('gzip_command', self)
+
+      config_param :command_parameter, :string, :default => ''
+
+      def configure(conf)
+        super
+        check_command('gzip')
+      end
+
+      def ext
+        'gz'.freeze
+      end
+
+      def content_type
+        'application/x-gzip'.freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk_is_file = @buffer_type == 'file'
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-gzip-tmp")
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+
+        res = system "gzip #{@command_parameter} -c #{path} > #{tmp.path}"
+        unless res
+          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+          begin
+            tmp.truncate(0)
+            gw = Zlib::GzipWriter.new(tmp)
+            chunk.write_to(gw)
+            gw.close
+          ensure
+            gw.close rescue nil
+          end
+        end
+      ensure
+        unless chunk_is_file
+          w.close(true) rescue nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/azurestorage_compressor_lzma2.rb

@@ -1,34 +1,34 @@
-module Fluent
-  class AzureStorageOutput
-    class LZMA2Compressor < Compressor
-      AzureStorageOutput.register_compressor('lzma2', self)
-
-      config_param :command_parameter, :string, :default => '-qf0'
-
-      def configure(conf)
-        super
-        check_command('xz', 'LZMA2')
-      end
-
-      def ext
-        'xz'.freeze
-      end
-
-      def content_type
-        'application/x-xz'.freeze
-      end
-
-      def compress(chunk, tmp)
-        w = Tempfile.new("chunk-xz-tmp")
-        chunk.write_to(w)
-        w.close
-
-        # We don't check the return code because we can't recover lzop failure.
-        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
-      ensure
-        w.close rescue nil
-        w.unlink rescue nil
-      end
-    end
-  end
-end
+module Fluent
+  class AzureStorageOutput
+    class LZMA2Compressor < Compressor
+      AzureStorageOutput.register_compressor('lzma2', self)
+
+      config_param :command_parameter, :string, :default => '-qf0'
+
+      def configure(conf)
+        super
+        check_command('xz', 'LZMA2')
+      end
+
+      def ext
+        'xz'.freeze
+      end
+
+      def content_type
+        'application/x-xz'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-xz-tmp")
+        chunk.write_to(w)
+        w.close
+
+        # We don't check the return code because we can't recover lzop failure.
+        system "xz #{@command_parameter} -c #{w.path} > #{tmp.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/azurestorage_compressor_lzo.rb

@@ -1,34 +1,34 @@
-module Fluent
-  class AzureStorageOutput
-    class LZOCompressor < Compressor
-      AzureStorageOutput.register_compressor('lzo', self)
-
-      config_param :command_parameter, :string, :default => '-qf1'
-
-      def configure(conf)
-        super
-        check_command('lzop', 'LZO')
-      end
-
-      def ext
-        'lzo'.freeze
-      end
-
-      def content_type
-        'application/x-lzop'.freeze
-      end
-
-      def compress(chunk, tmp)
-        w = Tempfile.new("chunk-tmp")
-        chunk.write_to(w)
-        w.close
-
-        # We don't check the return code because we can't recover lzop failure.
-        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
-      ensure
-        w.close rescue nil
-        w.unlink rescue nil
-      end
-    end
-  end
-end
+module Fluent
+  class AzureStorageOutput
+    class LZOCompressor < Compressor
+      AzureStorageOutput.register_compressor('lzo', self)
+
+      config_param :command_parameter, :string, :default => '-qf1'
+
+      def configure(conf)
+        super
+        check_command('lzop', 'LZO')
+      end
+
+      def ext
+        'lzo'.freeze
+      end
+
+      def content_type
+        'application/x-lzop'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Tempfile.new("chunk-tmp")
+        chunk.write_to(w)
+        w.close
+
+        # We don't check the return code because we can't recover lzop failure.
+        system "lzop #{@command_parameter} -o #{tmp.path} #{w.path}"
+      ensure
+        w.close rescue nil
+        w.unlink rescue nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_azurestorage.rb

@@ -1,248 +1,248 @@
-module Fluent
-  require 'fluent/mixin/config_placeholders'
-
-  class AzureStorageOutput < Fluent::TimeSlicedOutput
-    Fluent::Plugin.register_output('azurestorage', self)
-
-    def initialize
-      super
-      require 'azure'
-      require 'fluent/plugin/upload_service'
-      require 'zlib'
-      require 'time'
-      require 'tempfile'
-
-      @compressor = nil
-    end
-
-    config_param :path, :string, :default => ""
-    config_param :azure_storage_account, :string, :default => nil
-    config_param :azure_storage_access_key, :string, :default => nil, :secret => true
-    config_param :azure_container, :string, :default => nil
-    config_param :azure_storage_type, :string, :default => "blob"
-    config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
-    config_param :store_as, :string, :default => "gzip"
-    config_param :auto_create_container, :bool, :default => true
-    config_param :format, :string, :default => "out_file"
-    config_param :command_parameter, :string, :default => nil
-
-    attr_reader :bs
-
-    include Fluent::Mixin::ConfigPlaceholders
-
-    def placeholders
-      [:percent]
-    end
-
-    def configure(conf)
-      super
-
-      begin
-        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
-      rescue => e
-        $log.warn "#{@store_as} not found. Use 'text' instead"
-        @compressor = TextCompressor.new
-      end
-      @compressor.configure(conf)
-
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
-
-      if @localtime
-        @path_slicer = Proc.new {|path|
-          Time.now.strftime(path)
-        }
-      else
-        @path_slicer = Proc.new {|path|
-          Time.now.utc.strftime(path)
-        }
-      end
-
-      if @azure_container.nil?
-        raise ConfigError, 'azure_container is needed'
-      end
-
-      @storage_type = case @azure_storage_type
-                        when 'tables'
-                          raise NotImplementedError
-                        when 'queues'
-                          raise NotImplementedError
-                        else
-                          'blob'
-                      end
-    end
-
-    def start
-      super
-
-      if (!@azure_storage_account.nil? && !@azure_storage_access_key.nil?)
-        Azure.configure do |config|
-          config.storage_account_name = @azure_storage_account
-          config.storage_access_key   = @azure_storage_access_key
-        end
-      end
-      @bs = Azure::BlobService.new
-      @bs.extend UploadService
-
-      ensure_container
-    end
-
-    def format(tag, time, record)
-      @formatter.format(tag, time, record)
-    end
-
-    def write(chunk)
-      i = 0
-      previous_path = nil
-
-      begin
-        path = @path_slicer.call(@path)
-        values_for_object_key = {
-          "path" => path,
-          "time_slice" => chunk.key,
-          "file_extension" => @compressor.ext,
-          "index" => i,
-          "uuid_flush" => uuid_random
-        }
-        storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
-          values_for_object_key[expr[2...expr.size-1]]
-        }
-        if (i > 0) && (storage_path == previous_path)
-          raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
-        end
-
-        i += 1
-        previous_path = storage_path
-      end while blob_exists?(@azure_container, storage_path)
-
-      tmp = Tempfile.new("azure-")
-      begin
-        @compressor.compress(chunk, tmp)
-        tmp.close
-
-        options = {}
-        options[:content_type] = @compressor.content_type
-        options[:container] = @azure_container
-        options[:blob] = storage_path
-
-        @bs.upload(tmp.path, options)
-      end
-    end
-
-    private
-    def ensure_container
-      if ! @bs.list_containers.find { |c| c.name == @azure_container }
-        if @auto_create_container
-          @bs.create_container(@azure_container)
-        else
-          raise "The specified container does not exist: container = #{@azure_container}"
-        end
-      end
-    end
-
-    class Compressor
-      include Configurable
-
-      def initialize(opts = {})
-        super()
-        @buffer_type = opts[:buffer_type]
-        @log = opts[:log]
-      end
-
-      attr_reader :buffer_type, :log
-
-      def configure(conf)
-        super
-      end
-
-      def ext
-      end
-
-      def content_type
-      end
-
-      def compress(chunk, tmp)
-      end
-
-      private
-
-      def check_command(command, algo = nil)
-        require 'open3'
-
-        algo = command if algo.nil?
-        begin
-          Open3.capture3("#{command} -V")
-        rescue Errno::ENOENT
-          raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
-        end
-      end
-    end
-
-    class GzipCompressor < Compressor
-      def ext
-        'gz'.freeze
-      end
-
-      def content_type
-        'application/x-gzip'.freeze
-      end
-
-      def compress(chunk, tmp)
-        w = Zlib::GzipWriter.new(tmp)
-        chunk.write_to(w)
-        w.finish
-      ensure
-        w.finish rescue nil
-      end
-    end
-
-    class TextCompressor < Compressor
-      def ext
-        'txt'.freeze
-      end
-
-      def content_type
-        'text/plain'.freeze
-      end
-
-      def compress(chunk, tmp)
-        chunk.write_to(tmp)
-      end
-    end
-
-    class JsonCompressor < TextCompressor
-      def ext
-        'json'.freeze
-      end
-
-      def content_type
-        'application/json'.freeze
-      end
-    end
-
-    COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
-    {
-        'gzip' => GzipCompressor,
-        'json' => JsonCompressor,
-        'text' => TextCompressor
-    }.each { |name, compressor|
-      COMPRESSOR_REGISTRY.register(name, compressor)
-    }
-
-    def self.register_compressor(name, compressor)
-      COMPRESSOR_REGISTRY.register(name, compressor)
-    end
-
-    def blob_exists?(container, blob)
-      begin
-        @bs.get_blob_properties(container, blob)
-        true
-      rescue Azure::Core::Http::HTTPError => ex
-        raise if ex.status_code != 404
-        false
-      rescue Exception => e
-        raise e.message
-      end
-    end
-  end
-end
+module Fluent
+  require 'fluent/mixin/config_placeholders'
+
+  class AzureStorageOutput < Fluent::TimeSlicedOutput
+    Fluent::Plugin.register_output('azurestorage', self)
+
+    def initialize
+      super
+      require 'azure'
+      require 'fluent/plugin/upload_service'
+      require 'zlib'
+      require 'time'
+      require 'tempfile'
+
+      @compressor = nil
+    end
+
+    config_param :path, :string, :default => ""
+    config_param :azure_storage_account, :string, :default => nil
+    config_param :azure_storage_access_key, :string, :default => nil, :secret => true
+    config_param :azure_container, :string, :default => nil
+    config_param :azure_storage_type, :string, :default => "blob"
+    config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
+    config_param :store_as, :string, :default => "gzip"
+    config_param :auto_create_container, :bool, :default => true
+    config_param :format, :string, :default => "out_file"
+    config_param :command_parameter, :string, :default => nil
+
+    attr_reader :bs
+
+    include Fluent::Mixin::ConfigPlaceholders
+
+    def placeholders
+      [:percent]
+    end
+
+    def configure(conf)
+      super
+
+      begin
+        @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
+      rescue => e
+        $log.warn "#{@store_as} not found. Use 'text' instead"
+        @compressor = TextCompressor.new
+      end
+      @compressor.configure(conf)
+
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+
+      if @localtime
+        @path_slicer = Proc.new {|path|
+          Time.now.strftime(path)
+        }
+      else
+        @path_slicer = Proc.new {|path|
+          Time.now.utc.strftime(path)
+        }
+      end
+
+      if @azure_container.nil?
+        raise ConfigError, 'azure_container is needed'
+      end
+
+      @storage_type = case @azure_storage_type
+                        when 'tables'
+                          raise NotImplementedError
+                        when 'queues'
+                          raise NotImplementedError
+                        else
+                          'blob'
+                      end
+    end
+
+    def start
+      super
+
+      if (!@azure_storage_account.nil? && !@azure_storage_access_key.nil?)
+        Azure.configure do |config|
+          config.storage_account_name = @azure_storage_account
+          config.storage_access_key   = @azure_storage_access_key
+        end
+      end
+      @bs = Azure::Blob::BlobService.new
+      @bs.extend UploadService
+
+      ensure_container
+    end
+
+    def format(tag, time, record)
+      @formatter.format(tag, time, record)
+    end
+
+    def write(chunk)
+      i = 0
+      previous_path = nil
+
+      begin
+        path = @path_slicer.call(@path)
+        values_for_object_key = {
+          "path" => path,
+          "time_slice" => chunk.key,
+          "file_extension" => @compressor.ext,
+          "index" => i,
+          "uuid_flush" => uuid_random
+        }
+        storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) { |expr|
+          values_for_object_key[expr[2...expr.size-1]]
+        }
+        if (i > 0) && (storage_path == previous_path)
+          raise "duplicated path is generated. use %{index} in azure_object_key_format: path = #{storage_path}"
+        end
+
+        i += 1
+        previous_path = storage_path
+      end while blob_exists?(@azure_container, storage_path)
+
+      tmp = Tempfile.new("azure-")
+      begin
+        @compressor.compress(chunk, tmp)
+        tmp.close
+
+        options = {}
+        options[:content_type] = @compressor.content_type
+        options[:container] = @azure_container
+        options[:blob] = storage_path
+
+        @bs.upload(tmp.path, options)
+      end
+    end
+
+    private
+    def ensure_container
+      if ! @bs.list_containers.find { |c| c.name == @azure_container }
+        if @auto_create_container
+          @bs.create_container(@azure_container)
+        else
+          raise "The specified container does not exist: container = #{@azure_container}"
+        end
+      end
+    end
+
+    class Compressor
+      include Configurable
+
+      def initialize(opts = {})
+        super()
+        @buffer_type = opts[:buffer_type]
+        @log = opts[:log]
+      end
+
+      attr_reader :buffer_type, :log
+
+      def configure(conf)
+        super
+      end
+
+      def ext
+      end
+
+      def content_type
+      end
+
+      def compress(chunk, tmp)
+      end
+
+      private
+
+      def check_command(command, algo = nil)
+        require 'open3'
+
+        algo = command if algo.nil?
+        begin
+          Open3.capture3("#{command} -V")
+        rescue Errno::ENOENT
+          raise ConfigError, "'#{command}' utility must be in PATH for #{algo} compression"
+        end
+      end
+    end
+
+    class GzipCompressor < Compressor
+      def ext
+        'gz'.freeze
+      end
+
+      def content_type
+        'application/x-gzip'.freeze
+      end
+
+      def compress(chunk, tmp)
+        w = Zlib::GzipWriter.new(tmp)
+        chunk.write_to(w)
+        w.finish
+      ensure
+        w.finish rescue nil
+      end
+    end
+
+    class TextCompressor < Compressor
+      def ext
+        'txt'.freeze
+      end
+
+      def content_type
+        'text/plain'.freeze
+      end
+
+      def compress(chunk, tmp)
+        chunk.write_to(tmp)
+      end
+    end
+
+    class JsonCompressor < TextCompressor
+      def ext
+        'json'.freeze
+      end
+
+      def content_type
+        'application/json'.freeze
+      end
+    end
+
+    COMPRESSOR_REGISTRY = Registry.new(:azurestorage_compressor_type, 'fluent/plugin/azurestorage_compressor_')
+    {
+        'gzip' => GzipCompressor,
+        'json' => JsonCompressor,
+        'text' => TextCompressor
+    }.each { |name, compressor|
+      COMPRESSOR_REGISTRY.register(name, compressor)
+    }
+
+    def self.register_compressor(name, compressor)
+      COMPRESSOR_REGISTRY.register(name, compressor)
+    end
+
+    def blob_exists?(container, blob)
+      begin
+        @bs.get_blob_properties(container, blob)
+        true
+      rescue Azure::Core::Http::HTTPError => ex
+        raise if ex.status_code != 404
+        false
+      rescue Exception => e
+        raise e.message
+      end
+    end
+  end
+end