fluent-plugin-azurestorage-gen2 0.2.0 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4b4636ae463e1d010ac8b6c069c1c9e47d226a9
-  data.tar.gz: b215a1aa8a4c8c9e768b38f1ee3a318b8d5f42a2
+  metadata.gz: 02170d63c314b7bb91fb1722b0ca1a4dccad9bd0
+  data.tar.gz: 7fc221dc0be643681800be67c954634caa7df518
 SHA512:
-  metadata.gz: 8c7eb7bd110a30963f4cc5b99303829d234b9285a7a29e4b9eb3115511e0c7870f06f94cf237b60ce8cd76e74a0c634a71341747b0991b5080f7dbab2f94ec3f
-  data.tar.gz: a02576ae303d449b2d2301c88d80e076792acf58d90b50fafa4822a3ab96581f1566ccb052cd153d42a323fb4563575b648271f89a9220d815ba1a3d5f953be2
+  metadata.gz: ad9f64e7f2af1d65177e683aa808df6187d9ace11a0f976a925a11d31e74db15c324b10035d4610ff387b5f2b3a1939c4f8c2cf5f7f25e4a237c16350b4ec620
+  data.tar.gz: eeff3b1a4c7ea0b48e207c270e1679ba770c2bca21146f879bfb893a1fca314419c5a7a6c35a1437ca4967dbd87a0d703c81b66a16d1326e187da90e2fcf1ac1

data/Makefile

@@ -8,8 +8,6 @@ install: gem
 push: gem
 	gem push fluent-plugin-azurestorage-gen2*.gem
 
-tag-and-branch:
+tag:
 	git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
-	git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
-	git push origin "v$$(cat VERSION)"
-	git push -u origin "release/$$(cat VERSION)"
+	git push origin "v$$(cat VERSION)"

data/README.md

@@ -3,13 +3,13 @@
 [![Build Status](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2.svg?branch=master)](https://travis-ci.org/oleewere/fluent-plugin-azurestorage-gen2)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-azurestorage-gen2.svg)](http://badge.fury.io/rb/fluent-plugin-azurestorage-gen2)
-![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total)
+![](https://ruby-gem-downloads-badge.herokuapp.com/fluent-plugin-azurestorage-gen2?type=total&metric=true)
 
 ## Requirements
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.2.0 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -34,9 +34,10 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_object_key_format          %{path}-%{index}.%{file_extension}
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
+  file_extension                   log # only used with store_as none
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
   auto_create_container            true
+  store_as                         gzip
   format                           single_value
   <buffer tag,time>
     @type file
@@ -61,8 +62,8 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_oauth_secret               <my client secret>
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
+  store_as                         gzip
   auto_create_container            true
   format                           single_value
   <buffer tag,time>
@@ -105,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
 
 ### azure_oauth_refresh_interval
 
-OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
+OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
 
 ### azure_oauth_use_azure_cli (Preview)
 
@@ -121,7 +122,19 @@ This plugin create container if not exist when you set 'auto_create_container' t
 
 ### skip_container_check
 
-You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform these operations.
+You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
+
+### enable_retry
+
+If you set this option, operations can be retried in the buffer. Default value is false. (Used for create/update/flush Blob operations)
+
+### startup_fail_on_error
+
+If that setting is disabled, the worker won't fail on initialization (getting first access token) error. The default value is true.
+
+### url_domain_suffix
+
+The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
 
 ### azure_object_key_format
 

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.5

data/fluent-plugin-azurestorage-gen2.gemspec

@@ -4,7 +4,7 @@ $:.push File.expand_path('../lib', __FILE__)
 Gem::Specification.new do |gem|
   gem.name = "fluent-plugin-azurestorage-gen2"
   gem.description = "Azure Storage output plugin for Fluentd event collector"
-  gem.license = "Apache-2.0"
+  gem.license = "MIT License"
   gem.homepage = "https://github.com/oleewere/fluent-plugin-azurestorage-gen2"
   gem.summary = gem.description
   gem.version = File.read("VERSION").strip

data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb

@@ -24,6 +24,7 @@ module Fluent
                  chunk.path
                else
                  w = Tempfile.new("chunk-gzip-tmp")
+                 w.binmode
                  chunk.write_to(w)
                  w.close
                  w.path

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-xz-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -27,19 +27,21 @@ module Fluent::Plugin
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
         config_param :azure_oauth_use_azure_cli, :bool, :default => false
-        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
+        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
         config_param :file_extension, :string, :default => "log"
         config_param :store_as, :string, :default => "none"
         config_param :auto_create_container, :bool, :default => false
         config_param :skip_container_check, :bool, :default => false
+        config_param :enable_retry, :bool, :default => false
+        config_param :startup_fail_on_error, :bool, :default => true
+        config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :command_parameter, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
-        URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
         ABFS_API_VERSION = "2018-11-09"
         AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
@@ -127,23 +129,25 @@ module Fluent::Plugin
                     log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
                     upload_blob(raw_data, metadata)
                 end
+                chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
             else
                 tmp = Tempfile.new("azure-")
+                tmp.binmode
                 begin
                     @compressor.compress(chunk, tmp)
-                    tmp.close
+                    tmp.rewind
                     generate_log_name(metadata, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
                         generate_log_name(metadata, @current_index)
                     end
-                    log.debug "Start uploading temp file: #{tmp.path}"
+                    log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
                     upload_blob(content, metadata)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
-                    tmp.unlink
+                    tmp.close(true) rescue nil
                 end
             end
 
@@ -151,7 +155,7 @@ module Fluent::Plugin
 
         private
         def upload_blob(content, metadata)
-            log.debug "azurestorage_gen2:  Uploading blob: #{@azure_storage_path}"
+            log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
             existing_content_length = get_blob_properties(@azure_storage_path)
             if existing_content_length == 0
                 create_blob(@azure_storage_path)
@@ -183,7 +187,19 @@ module Fluent::Plugin
         def setup_access_token
             if @azure_storage_access_key.nil?
                 @get_token_lock = Concurrent::ReadWriteLock.new
-                acquire_access_token
+                if @startup_fail_on_error
+                    acquire_access_token
+                else
+                    while true
+                        begin
+                            acquire_access_token
+                            break
+                        rescue Exception => e
+                            log.warn("#{e.message}, acquired token failed, wait 20 seconds until next retry.")
+                            sleep 20
+                        end
+                    end
+                end
                 if @azure_oauth_refresh_interval > 0
                     log.info("azurestorage_gen2: Start getting access token every #{@azure_oauth_refresh_interval} seconds.")
                     @get_token_task = Concurrent::TimerTask.new(
@@ -217,7 +233,7 @@ module Fluent::Plugin
         # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
         private
         def acquire_access_token_msi
-            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azures.com/" }
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
             end
@@ -266,7 +282,7 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -293,12 +309,12 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating container '#{@azure_container}' request timed out."
+                    raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
                 else
                     raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
@@ -313,16 +329,16 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating blob '#{blob_path}' request timed out."
+                    raise_error  "Creating blob '#{blob_path}' request timed out."
                 elsif response.code == 409
                     log.debug "azurestorage_gen2: Blob already exists: #{blob_path}"
                 else
-                    raise Fluent::UnrecoverableError, "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Creating blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -330,24 +346,24 @@ module Fluent::Plugin
 
         private
         def append_blob_block(blob_path, content, position)
-            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
+            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
             datestamp = create_request_date
             headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"Content-Length" => content.length}
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Appending blob #{blob_path}' request timed out."
+                    raise_error  "Appending blob #{blob_path}' request timed out."
                 elsif response.code == 404
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has not found. Error code: #{response.code}", 404)
                 elsif response.code == 409
                     raise AppendBlobResponseError.new("Blob '#{blob_path}' has conflict. Error code: #{response.code}", 409)
                 else
-                    raise Fluent::UnrecoverableError, "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Appending blob '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -361,14 +377,14 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Bloub '#{blob_path}' flush request timed out."
+                    raise_error  "Bloub '#{blob_path}' flush request timed out."
                 else
-                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -382,18 +398,18 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
                   content_length = response.headers['Content-Length'].to_i
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Get blob properties '#{blob_path}' request timed out."
+                    raise_error  "Get blob properties '#{blob_path}' request timed out."
                 elsif response.code == 404
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' does not exist. Creating it if needed..."
                     content_length = 0
                 else
-                    raise Fluent::UnrecoverableError, "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Get blob properties '#{blob_path}' request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -506,6 +522,15 @@ module Fluent::Plugin
             Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
         end
 
+        private
+        def raise_error(error_message)
+            if @enable_retry
+                raise BlobOperationError, error_message
+            else
+                raise Fluent::UnrecoverableError,  error_message
+            end
+        end
+
         def uuid_random
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
@@ -622,4 +647,10 @@ module Fluent::Plugin
           super(message)
         end
     end
+
+    class BlobOperationError < StandardError
+        def initialize(message="Default message")
+            super(message)
+        end
+    end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.5
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-04 00:00:00.000000000 Z
+date: 2020-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -188,7 +188,7 @@ files:
 - test/plugin/test_out_azurestorage_gen2.rb
 homepage: https://github.com/oleewere/fluent-plugin-azurestorage-gen2
 licenses:
-- Apache-2.0
+- MIT License
 metadata: {}
 post_install_message: 
 rdoc_options: []