fluent-plugin-azurestorage-gen2 0.1.4 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13b891a2c06d9a6a18d088c8cf02eb781d3538eb
-  data.tar.gz: cc89a73ec41eed32b5a7f3a4d104c3ac80422642
+  metadata.gz: c631dda137f661c5692855eb46d97696df14a5e0
+  data.tar.gz: 174ace2e26b75f25b12a38d3c76726d3526f0866
 SHA512:
-  metadata.gz: da9ade031001af9fea91952af3da3fea6fa32dac029b4b2ae84cb9ac92c2b68e2fd7f9232380ec38fd96990ad53b07d9a16ec11699c12ed4b3e2978a72c03364
-  data.tar.gz: 76d5f0f943d2647b388852d4c4d274c0af6c8cfae9b3440afaed6b5cb4db240106efe106e974cb8ad744fe7f3ecb4deee143271e42ef67f2f9db184427b96143
+  metadata.gz: 5f135ffbe42f292666e66bca2b84a8c1f49d3ddf727460e376c95658b0382419a9008b110371d48314ccef6af0dc2f3cff5b0ae64730ae29222571dd20333163
+  data.tar.gz: e81ba7ac31051ac5813ab4f3c845ee866f7dce3061c16a72fc5f3164e4b53c9c345c1135820762671162f092b195191a93ff6650a2562fc61cb12ea9a4b97ff2

data/Makefile

@@ -8,8 +8,6 @@ install: gem
 push: gem
 	gem push fluent-plugin-azurestorage-gen2*.gem
 
-tag-and-branch:
+tag:
 	git tag "v$$(cat VERSION)" $(RELEASE_COMMIT)
-	git checkout -b "release/$$(cat VERSION)" $(RELEASE_COMMIT)
-	git push origin "v$$(cat VERSION)"
-	git push -u origin "release/$$(cat VERSION)"
+	git push origin "v$$(cat VERSION)"

data/README.md

@@ -9,7 +9,7 @@
 
 | fluent-plugin-azurestorage-gen2 | fluentd | ruby |
 |------------------------|---------|------|
-| >= 0.1.3 | >= v0.14.0 | >= 2.4 |
+| >= 0.1.0 | >= v0.14.0 | >= 2.4 |
 
 ## Overview
 
@@ -34,9 +34,11 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_object_key_format          %{path}-%{index}.%{file_extension}
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
+  file_extension                   log # only used with store_as none
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
   auto_create_container            true
+  store_as                         gzip
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -60,9 +62,10 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_oauth_secret               <my client secret>
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
-  file_extension                   log
   path                             "/cluster-logs/myfolder/${tag[1]}-#{Socket.gethostname}-%M"
+  store_as                         gzip
   auto_create_container            true
+  format                           single_value
   <buffer tag,time>
     @type file
     path /var/log/fluent/azurestorage-buffer
@@ -103,7 +106,7 @@ OAuth client secret that is used for OAuth based authentication. Required if OAu
 
 ### azure_oauth_refresh_interval
 
-OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used.
+OAuth2 access token refreshment interval in second. Only applies when MSI / OAuth authentication is used. (default value is 59 minutes)
 
 ### azure_oauth_use_azure_cli (Preview)
 
@@ -117,6 +120,14 @@ Azure Storage Container name
 
 This plugin create container if not exist when you set 'auto_create_container' to true.
 
+### skip_container_check
+
+You can skip the initial container listing (and container creation) operations at startup. That can be useful if the user is not allowed to perform this operation.
+
+### enable_retry
+
+If you set this option, operations can be retried in the buffer. Default value is false. (Currently only flush command is supported)
+
 ### azure_object_key_format
 
 The format of Azure Storage object keys. You can use several built-in variables:
@@ -169,13 +180,13 @@ azure_object_key_format %{path}/events/ts=%{time_slice}/events_%{index}-%{hostna
 
 ### file_extension
 
-File extension for the uploaded files. Only uses if `store_as` is not set, or set as `none`
+File extension for the uploaded files. Only used if `store_as` is not set, or set as `none`
 
 ### store_as
 
 Archive format on Azure Storage. You can use following types:
 
-- none (default - no tmp file creation for log processing)
+- none (default - no tmp file creation for log processing, use with json or single value format)
 - gzip
 - json
 - text

data/VERSION

@@ -1 +1 @@
-0.1.4
+0.2.2

data/lib/fluent/plugin/azurestorage_gen2_compressor_gzip_command.rb

@@ -24,6 +24,7 @@ module Fluent
                  chunk.path
                else
                  w = Tempfile.new("chunk-gzip-tmp")
+                 w.binmode
                  chunk.write_to(w)
                  w.close
                  w.path

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzma2.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-xz-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/azurestorage_gen2_compressor_lzo.rb

@@ -20,6 +20,7 @@ module Fluent
 
       def compress(chunk, tmp)
         w = Tempfile.new("chunk-tmp")
+        w.binmode
         chunk.write_to(w)
         w.close
 

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -27,16 +27,17 @@ module Fluent::Plugin
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
         config_param :azure_oauth_use_azure_cli, :bool, :default => false
-        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 60 # one hour
+        config_param :azure_oauth_refresh_interval, :integer, :default => 60 * 59 # 59 minutes, do not wait one hour
         config_param :azure_container, :string, :default => nil
         config_param :azure_object_key_format, :string, :default => "%{path}%{time_slice}_%{index}.%{file_extension}"
         config_param :file_extension, :string, :default => "log"
         config_param :store_as, :string, :default => "none"
         config_param :auto_create_container, :bool, :default => false
+        config_param :skip_container_check, :bool, :default => false
+        config_param :enable_retry, :bool, :default => false
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :command_parameter, :string, :default => nil
-        config_param :message_field, :string, :default => nil
 
         DEFAULT_FORMAT_TYPE = "out_file"
         URL_DOMAIN_SUFFIX = '.dfs.core.windows.net'
@@ -57,13 +58,17 @@ module Fluent::Plugin
             compat_parameters_convert(conf, :buffer, :formatter, :inject)
             super
 
-            begin
-                @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
-            rescue => e
-                log.warn "#{@store_as} not found. Use 'text' instead"
-                @compressor = TextCompressor.new
+            if @store_as.nil? || @store_as == "none"
+                log.info "azurestorage_gen2: Compression is disabled (store_as: #{@store_as})"
+            else
+                begin
+                    @compressor = COMPRESSOR_REGISTRY.lookup(@store_as).new(:buffer_type => @buffer_type, :log => log)
+                rescue => e
+                    log.warn "#{@store_as} not found. Use 'text' instead"
+                    @compressor = TextCompressor.new
+                end
+                @compressor.configure(conf)
             end
-            @compressor.configure(conf)
 
             @formatter = formatter_create
       
@@ -84,11 +89,10 @@ module Fluent::Plugin
             @azure_storage_path = ''
             @last_azure_storage_path = ''
             @current_index = 0
+
             if @store_as.nil? || @store_as == "none"
-                @blob_content_type = "text/plain"
                 @final_file_extension = @file_extension
             else
-                @blob_content_type = @compressor.content_type
                 @final_file_extension = @compressor.ext
             end
 
@@ -100,48 +104,49 @@ module Fluent::Plugin
 
         def start
             setup_access_token
-            ensure_container
+            if !@skip_container_check
+                ensure_container
+            end
             super
         end
 
+        def format(tag, time, record)
+            r = inject_values_to_record(tag, time, record)
+            @formatter.format(tag, time, r)
+        end
+
         def write(chunk)
             metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                raw_data=''
                 generate_log_name(metadata, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
                     generate_log_name(metadata, @current_index)
                 end
-                chunk.each do |emit_time, record|
-                    if @message_field.nil? || @message_field.empty?
-                        raw_data << "#{Yajl.dump(record)}\n"
-                    elsif record.key?(@message_field)
-                        line = record[@message_field].chomp
-                        raw_data << "#{line}\n"
-                    end
-                end
-                raw_data = raw_data.chomp
+                raw_data = chunk.read
                 unless raw_data.empty?
+                    log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
                     upload_blob(raw_data, metadata)
                 end
+                chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
             else
                 tmp = Tempfile.new("azure-")
+                tmp.binmode
                 begin
                     @compressor.compress(chunk, tmp)
-                    tmp.close
+                    tmp.rewind
                     generate_log_name(metadata, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
                         generate_log_name(metadata, @current_index)
                     end
-                    log.debug "Start uploading temp file: #{tmp.path}"
+                    log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
                     upload_blob(content, metadata)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
-                    tmp.unlink
+                    tmp.close(true) rescue nil
                 end
             end
 
@@ -149,7 +154,7 @@ module Fluent::Plugin
 
         private
         def upload_blob(content, metadata)
-            log.debug "azurestorage_gen2:  Uploading blob: #{@azure_storage_path}"
+            log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
             existing_content_length = get_blob_properties(@azure_storage_path)
             if existing_content_length == 0
                 create_blob(@azure_storage_path)
@@ -157,11 +162,6 @@ module Fluent::Plugin
             append_blob(content, metadata, existing_content_length)
         end
 
-        def format(tag, time, record)
-            r = inject_values_to_record(tag, time, record)
-            @formatter.format(tag, time, r)
-        end
-
         private
         def generate_log_name(metadata, index)
             time_slice = if metadata.timekey.nil?
@@ -229,7 +229,7 @@ module Fluent::Plugin
                 if response.success?
                   data = JSON.parse(response.body)
                   log.debug "azurestorage_gen2: Token response: #{data}"
-                  @azure_access_token = data["access_token"]
+                  @azure_access_token = data["access_token"].chomp
                 else
                     raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
                 end
@@ -242,12 +242,12 @@ module Fluent::Plugin
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
             content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers, verbose: true)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
                   log.debug "azurestorage_gen2: Token response: #{data}"
-                  @azure_access_token = data["access_token"]
+                  @azure_access_token = data["access_token"].chomp
                 else
                     raise Fluent::UnrecoverableError, "Failed to acquire access token. #{response.code}: #{response.body}"
                 end
@@ -259,7 +259,7 @@ module Fluent::Plugin
         def acquire_access_token_by_az
             access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
             log.debug "azurestorage_gen2: Token response: #{access_token}"
-            @azure_access_token = access_token
+            @azure_access_token = access_token.chomp
         end
 
         private
@@ -301,7 +301,7 @@ module Fluent::Plugin
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
                 elsif response.timed_out?
-                    raise Fluent::UnrecoverableError,  "Creating container '#{@azure_container}' request timed out."
+                    raise Fluent::UnrecoverableError, "Creating container '#{@azure_container}' request timed out."
                 else
                     raise Fluent::UnrecoverableError, "Creating container request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
@@ -312,7 +312,7 @@ module Fluent::Plugin
         private
         def create_blob(blob_path)
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "application/json"}
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION, :"x-ms-date" => datestamp,:"Content-Length" => "0", :"Content-Type" => "text/plain"}
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
@@ -333,13 +333,13 @@ module Fluent::Plugin
 
         private
         def append_blob_block(blob_path, content, position)
-            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position}"
+            log.debug "azurestorage_gen2: append_blob_block.start: Append blob ('#{blob_path}') called with position #{position} (content length: #{content.length}, end position: #{position + content.length})"
             datestamp = create_request_date
-            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"x-ms-content-type" => "#{@blob_content_type}", :"Content-Length" => content.length}
+            headers = {:"x-ms-version" =>  ABFS_API_VERSION,  :"x-ms-date" => datestamp, :"Content-Length" => content.length}
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :body => content, :params => params, :headers=> headers)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{URL_DOMAIN_SUFFIX}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -371,7 +371,7 @@ module Fluent::Plugin
                 elsif response.timed_out?
                     raise Fluent::UnrecoverableError,  "Bloub '#{blob_path}' flush request timed out."
                 else
-                    raise Fluent::UnrecoverableError, "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
+                    raise_error "Blob flush request failed - code: #{response.code}, body: #{response.body}, headers: #{response.headers}"
                 end
             end
             request.run
@@ -509,6 +509,15 @@ module Fluent::Plugin
             Time.now.strftime('%a, %e %b %y %H:%M:%S %Z')
         end
 
+        private
+        def raise_error(error_message)
+            if @enable_retry
+                raise BlobOperationError, error_message
+            else
+                raise Fluent::UnrecoverableError,  error_message
+            end
+        end
+
         def uuid_random
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
@@ -625,4 +634,10 @@ module Fluent::Plugin
           super(message)
         end
     end
+
+    class BlobOperationError < StandardError
+        def initialize(message="Default message")
+            super(message)
+        end
+    end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.2
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-23 00:00:00.000000000 Z
+date: 2020-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd