fluent-plugin-azurestorage-gen2 0.2.8 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 46b6cf60479daf8f64e5196ad36332a11591220e
-  data.tar.gz: 09aec8214be51d7be9ce4b2e7350e145d4b0fb51
+  metadata.gz: c9dfe2426009f9375ce47d7285a6030af693c148
+  data.tar.gz: 9e76fe1f92164e6e3807482f04d7f2cf6fecc19b
 SHA512:
-  metadata.gz: a3f4d5c6245c11e5f8b4b33d26825a3d9f93796d553a1ac1d55c931e891762467c22abc228f546d7c2efdf107c3cc471d47dc6bc0114bd1f5c1dced3a6b30bae
-  data.tar.gz: 67b751890b5aee2086d6d617b41d4da6883d8cfa9e77acc2ac7de54f58e457725cb718606a65ec385972fed1a24518cc9d981f15a454393b1e08b31b662735b9
+  metadata.gz: dbdac55653f0e83acde54dc2ace5e58e43f6954b84bb0179526a4230adf59619be469f313508ecbeb772dc447473cd62e9d3fbaff9cb3af5d45b3d1ba6fdab45
+  data.tar.gz: 95475a64f3447e017da418c4faa87dabc932c12205b2050e9e4dd1382128f919415e466a94c68a90bd22be6be7cc3aaf4a426a9ffb1d4efff2bbb4a87925ef81

data/README.md

@@ -31,6 +31,7 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_storage_account            mystorageabfs
   azure_container                  mycontainer
   azure_instance_msi               /subscriptions/mysubscriptionid
+  azure_client_id                  <msi client id>
   azure_object_key_format          %{path}-%{index}.%{file_extension}
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
@@ -92,6 +93,14 @@ Your Azure Storage Access Key(Primary or Secondary). This also can be got from A
 
 Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
 
+### azure_client_id
+
+Azure AD client id is a specific explicit identity to use when authenticating to Azure AD. Mutually exclusive with azure_object_id and azure_instance_msi. (for now, you need to define next to `azure_instance_msi`)
+
+### azure_object_id
+
+Azure AD object id is a specific explicit identity to use when authenticating to Azure AD. Mutually exclusive with azure_client_id and azure_instance_msi. (for now, you need to define next to `azure_instance_msi`)
+
 ### azure_oauth_tenant_id (Preview)
 
 Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
@@ -140,6 +149,10 @@ If that setting is disabled, the worker won't fail on initialization (getting fi
 
 The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
 
+### url_storage_resource
+
+The url that is used during accessing a resource. Default value: `https://storage.azure.com/`
+
 ### azure_object_key_format
 
 The format of Azure Storage object keys. You can use several built-in variables:
@@ -148,6 +161,7 @@ The format of Azure Storage object keys. You can use several built-in variables:
 - %{time_slice}
 - %{index}
 - %{file_extension}
+- %{upload_timestamp}
 
 to decide keys dynamically.
 
@@ -155,6 +169,7 @@ to decide keys dynamically.
 %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
 %{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
 %{file_extention} is always "gz" for now.
+%{upload_timestamp} is an upload timestamp in text that are formatted with *upload_timestamp_format*. Difference between time_slice and upload_timestamp is that the second one is the actual system timestamp (other one is from the metadata)
 
 The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
 
@@ -279,10 +294,30 @@ Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to
 
 The time to wait old logs. Default is 10 minutes.
 
+### upload_timestamp_format
+
+Format of the upload timestamp used as the file name. Can be used instead of index in case of `write_only` option is enabled. Default value is '%H%M%S%L'.
+
 ### utc
 
 Use UTC instead of local time.
 
+### write_only
+
+If that option is enabled, HEAD calls are skipped during blob operations. (so make sure to set the chunk limit to 4MB in order to avoid HEAD operation because of the append operation needs the last position of the uploaded blobs).
+
+### proxy_url
+
+Proxy URL for Azure endpoint.
+
+### proxy_username
+
+Proxy username for Azure proxy endpoint (used only if `proxy_url` is filled)
+
+### proxy_password
+
+Proxy password for Azure `proxy_username` (used only if `proxy_url` is filled)
+
 ## TODOs
 
 - add storage key support

data/VERSION

@@ -1 +1 @@
-0.2.8
+0.3.2

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -24,6 +24,8 @@ module Fluent::Plugin
         config_param :azure_storage_account, :string, :default => nil
         config_param :azure_storage_access_key, :string, :default => nil, :secret => true
         config_param :azure_instance_msi, :string, :default => nil
+        config_param :azure_client_id, :string, :default => nil
+        config_param :azure_object_id, :string, :default => nil
         config_param :azure_oauth_app_id, :string, :default => nil, :secret => true
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
@@ -39,9 +41,16 @@ module Fluent::Plugin
         config_param :enable_retry, :bool, :default => false
         config_param :startup_fail_on_error, :bool, :default => true
         config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
+        config_param :url_storage_resource, :string, :default => 'https://storage.azure.com/'
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
+        config_param :hex_random_length, :integer, default: 4
         config_param :command_parameter, :string, :default => nil
+        config_param :proxy_url, :string, :default => nil
+        config_param :proxy_username, :string, :default => nil
+        config_param :proxy_password, :string, :default => nil, :secret => true
+        config_param :write_only, :bool, :default => false
+        config_param :upload_timestamp_format, :string, :default => '%H%M%S%L'
 
         DEFAULT_FORMAT_TYPE = "out_file"
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
@@ -51,7 +60,7 @@ module Fluent::Plugin
         config_section :format do
             config_set_default :@type, DEFAULT_FORMAT_TYPE
         end
-  
+
         config_section :buffer do
             config_set_default :chunk_keys, ['time']
             config_set_default :timekey, (60 * 60 * 24)
@@ -74,17 +83,7 @@ module Fluent::Plugin
             end
 
             @formatter = formatter_create
-      
-            if @localtime
-              @path_slicer = Proc.new {|path|
-                Time.now.strftime(path)
-              }
-            else
-              @path_slicer = Proc.new {|path|
-                Time.now.utc.strftime(path)
-              }
-            end
-      
+
             if @azure_container.nil?
               raise Fluent::ConfigError, "azure_container is needed"
             end
@@ -98,7 +97,7 @@ module Fluent::Plugin
             else
                 @final_file_extension = @compressor.ext
             end
-
+            @values_for_object_chunk = {}
         end
 
         def multi_workers_ready?
@@ -110,12 +109,20 @@ module Fluent::Plugin
             if !@skip_container_check
                 if @failsafe_container_check
                     begin
-                        ensure_container
+                        if @write_only && @auto_create_container
+                            create_container
+                        else
+                            ensure_container
+                        end
                     rescue Exception => e
                         log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
                     end
                 else
-                    ensure_container
+                    if @write_only && @auto_create_container
+                        create_container
+                    else
+                        ensure_container
+                    end
                 end
             end
             super
@@ -127,17 +134,16 @@ module Fluent::Plugin
         end
 
         def write(chunk)
-            metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                generate_log_name(metadata, @current_index)
+                generate_log_name(chunk, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                 end
                 raw_data = chunk.read
                 unless raw_data.empty?
                     log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
-                    upload_blob(raw_data, metadata)
+                    upload_blob(raw_data, chunk)
                 end
                 chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
@@ -147,51 +153,76 @@ module Fluent::Plugin
                 begin
                     @compressor.compress(chunk, tmp)
                     tmp.rewind
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
-                        generate_log_name(metadata, @current_index)
+                        generate_log_name(chunk, @current_index)
                     end
                     log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
-                    upload_blob(content, metadata)
+                    upload_blob(content, chunk)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
                     tmp.close(true) rescue nil
                 end
+                @values_for_object_chunk.delete(chunk.unique_id)
             end
 
         end
 
         private
-        def upload_blob(content, metadata)
+        def upload_blob(content, chunk)
             log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
-            existing_content_length = get_blob_properties(@azure_storage_path)
-            if existing_content_length == 0
+            if @write_only
                 create_blob(@azure_storage_path)
+                append_blob(content, chunk, 0)
+            else
+                existing_content_length = get_blob_properties(@azure_storage_path)
+                if existing_content_length == 0
+                    create_blob(@azure_storage_path)
+                end
+                append_blob(content, chunk, existing_content_length)
             end
-            append_blob(content, metadata, existing_content_length)
         end
 
         private
-        def generate_log_name(metadata, index)
+        def generate_log_name(chunk, index)
+            metadata = chunk.metadata
             time_slice = if metadata.timekey.nil?
                        ''.freeze
                      else
                        Time.at(metadata.timekey).utc.strftime(@time_slice_format)
                      end
-            path = @path_slicer.call(@path)
-            values_for_object_key = {
-                "%{path}" => path,
-                "%{time_slice}" => time_slice,
+            if @localtime
+                hms_slicer = Time.now.strftime("%H%M%S")
+                upload_timestamp = Time.now.strftime(@upload_timestamp_format)
+            else
+                hms_slicer = Time.now.utc.strftime("%H%M%S")
+                upload_timestamp = Time.now.utc.strftime(@upload_timestamp_format)
+            end
+
+            @values_for_object_chunk[chunk.unique_id] ||= {
+                "%{hex_random}" => hex_random(chunk),
+            }
+            values_for_object_key_pre = {
+                "%{path}" => @path,
                 "%{index}" => index,
                 "%{uuid_flush}" => uuid_random,
-                "%{file_extension}" => @final_file_extension
+                "%{file_extension}" => @final_file_extension,
+                "%{upload_timestamp}" => upload_timestamp,
             }
-            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
-            extracted_path = extract_placeholders(storage_path, metadata)
-            extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
-            @azure_storage_path = extracted_path
+            values_for_object_key_post = {
+                "%{date_slice}" => time_slice,
+                "%{time_slice}" => time_slice,
+                "%{hms_slice}" => hms_slicer,
+            }.merge!(@values_for_object_chunk[chunk.unique_id])
+            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
+                values_for_object_key_pre.fetch(matched_key, matched_key)
+            end
+            storage_path = extract_placeholders(storage_path, chunk)
+            storage_path = storage_path.gsub(%r(%{[^}]+}), values_for_object_key_post)
+            storage_path = "/" + storage_path unless storage_path.start_with?("/")
+            @azure_storage_path = storage_path
         end
 
         def setup_access_token
@@ -243,11 +274,22 @@ module Fluent::Plugin
         # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
         private
         def acquire_access_token_msi
-            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}" }
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
             end
-            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
+            unless @azure_client_id.nil?
+                params[:client_id] = @azure_client_id
+            end
+            unless @azure_object_id.nil?
+                params[:object_id] = @azure_object_id
+            end
+            req_opts = {
+                :params => params,
+                :headers => { Metadata: "true" }
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -262,10 +304,16 @@ module Fluent::Plugin
 
         private
         def acquire_access_token_oauth_app
-            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
-            content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
+            content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=#{@url_storage_resource}"
+            req_opts = {
+                :params => params,
+                :body => content, 
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -280,7 +328,7 @@ module Fluent::Plugin
 
         private
         def acquire_access_token_by_az
-            access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
+            access_token=`az account get-access-token --resource #{@url_storage_resource} --query accessToken -o tsv`
             log.debug "azurestorage_gen2: Token response: #{access_token}"
             @azure_access_token = access_token.chomp
         end
@@ -292,7 +340,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -319,7 +373,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
@@ -339,7 +399,13 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
@@ -362,7 +428,14 @@ module Fluent::Plugin
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers,
+                :body => content
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -387,7 +460,13 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
@@ -408,7 +487,13 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
@@ -427,7 +512,7 @@ module Fluent::Plugin
         end
 
         private
-        def append_blob(content, metadata, existing_content_length)
+        def append_blob(content, chunk, existing_content_length)
           position = 0
           log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
           loop do
@@ -476,6 +561,16 @@ module Fluent::Plugin
                 "SharedKey #{@azure_storage_account}:#{signed(method, datestamp, resource, headers, params)}"
             end
         end
+        
+        private
+        def add_proxy_options(req_opts = {})
+            unless @proxy_url.nil?
+                req_opts[:proxy] = @proxy_url
+                unless @proxy_username.nil? || @proxy_password.nil?
+                    req_opts[:proxyuserpwd] = "#{@proxy_username}:#{@proxy_password}"
+                end
+            end
+        end
 
         private
         def signed(method, datestamp, resource, headers, params)
@@ -545,6 +640,12 @@ module Fluent::Plugin
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
         end
+
+        def hex_random(chunk)
+            unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
+            unique_hex.reverse!
+            unique_hex[0...@hex_random_length]
+        end
         
         def timekey_to_timeformat(timekey)
             case timekey

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.3.2
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-21 00:00:00.000000000 Z
+date: 2021-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd