fluent-plugin-azurestorage-gen2 0.2.7 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 968d3c5f271eaabe28496c3c1a7a145fc4223068
-  data.tar.gz: d451b9a7fee0eeef3d2dbbced493ac49495a43cd
+  metadata.gz: cabf8cf518b936b57ea4674553cfeb10126f0740
+  data.tar.gz: baba4ab34b169b1316a02d9b8a70e7bde1c373bd
 SHA512:
-  metadata.gz: f91296dd56d58faad94aede59b47afa740c746dc5207154ec31e627790428597d08518bd679f87d6ab9f540600017f725428218297731280b476dadb1f993dac
-  data.tar.gz: 452f1de528f620b44fa317918558218d5a757c3018d67ec6dbbf43dcd86f817c86757f1bc6eea11047c68efacebfd80b187895023a480111499ede945ceba196
+  metadata.gz: e7821f9a44d3520ebe7f596f7770d1fc236c5e5e40c7bb077eefefee29bca471954701e64b8b1a6b20c3aba7db2bd1013eda38c5b71a8b3ea259dd9e6b10987f
+  data.tar.gz: d3f03f3bae956b1ccd4566e54a563e4e5d43cea1478bc9833aec44917947fe7f5a12a6f0e918b50459b8593a4e2663d8772241be66b3845bffd13f3473eca361

data/README.md

@@ -140,6 +140,10 @@ If that setting is disabled, the worker won't fail on initialization (getting fi
 
 The defaultt `url_domain_suffix` is `.dfs.core.windows.net`, you can override this in case of private endpoints.
 
+### url_storage_resource
+
+The url that is used during accessing a resource. Default value: `https://storage.azure.com/`
+
 ### azure_object_key_format
 
 The format of Azure Storage object keys. You can use several built-in variables:
@@ -148,6 +152,7 @@ The format of Azure Storage object keys. You can use several built-in variables:
 - %{time_slice}
 - %{index}
 - %{file_extension}
+- %{upload_timestamp}
 
 to decide keys dynamically.
 
@@ -155,6 +160,7 @@ to decide keys dynamically.
 %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
 %{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
 %{file_extention} is always "gz" for now.
+%{upload_timestamp} is an upload timestamp in text that are formatted with *upload_timestamp_format*. Difference between time_slice and upload_timestamp is that the second one is the actual system timestamp (other one is from the metadata)
 
 The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
 
@@ -279,10 +285,30 @@ Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to
 
 The time to wait old logs. Default is 10 minutes.
 
+### upload_timestamp_format
+
+Format of the upload timestamp used as the file name. Can be used instead of index in case of `write_only` option is enabled. Default value is '%H%M%S%L'.
+
 ### utc
 
 Use UTC instead of local time.
 
+### write_only
+
+If that option is enabled, HEAD calls are skipped during blob operations. (so make sure to set the chunk limit to 4MB in order to avoid HEAD operation because of the append operation needs the last position of the uploaded blobs).
+
+### proxy_url
+
+Proxy URL for Azure endpoint.
+
+### proxy_username
+
+Proxy username for Azure proxy endpoint (used only if `proxy_url` is filled)
+
+### proxy_password
+
+Proxy password for Azure `proxy_username` (used only if `proxy_url` is filled)
+
 ## TODOs
 
 - add storage key support

data/VERSION

@@ -1 +1 @@
-0.2.7
+0.3.1

data/fluent-plugin-azurestorage-gen2.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency 'typhoeus', '~> 1.0', '>= 1.0.1'
   gem.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
   gem.add_runtime_dependency "yajl-ruby", '~> 1.4'
+  gem.add_runtime_dependency 'concurrent-ruby', '~> 1.1', '>= 1.1.5'
   gem.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
   gem.add_development_dependency 'test-unit', '~> 3.3', '>= 3.3.3'
   gem.add_development_dependency 'test-unit-rr', '~> 1.0', '>= 1.0.5'

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -6,6 +6,7 @@ require 'tempfile'
 require 'time'
 require 'typhoeus'
 require 'fluent/plugin/output'
+require 'concurrent'
 require 'zlib'
 
 module Fluent::Plugin
@@ -38,9 +39,16 @@ module Fluent::Plugin
         config_param :enable_retry, :bool, :default => false
         config_param :startup_fail_on_error, :bool, :default => true
         config_param :url_domain_suffix, :string, :default => '.dfs.core.windows.net'
+        config_param :url_storage_resource, :string, :default => 'https://storage.azure.com/'
         config_param :format, :string, :default => "out_file"
         config_param :time_slice_format, :string, :default => '%Y%m%d'
+        config_param :hex_random_length, :integer, default: 4
         config_param :command_parameter, :string, :default => nil
+        config_param :proxy_url, :string, :default => nil
+        config_param :proxy_username, :string, :default => nil
+        config_param :proxy_password, :string, :default => nil, :secret => true
+        config_param :write_only, :bool, :default => false
+        config_param :upload_timestamp_format, :string, :default => '%H%M%S%L'
 
         DEFAULT_FORMAT_TYPE = "out_file"
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
@@ -74,16 +82,6 @@ module Fluent::Plugin
 
             @formatter = formatter_create
       
-            if @localtime
-              @path_slicer = Proc.new {|path|
-                Time.now.strftime(path)
-              }
-            else
-              @path_slicer = Proc.new {|path|
-                Time.now.utc.strftime(path)
-              }
-            end
-      
             if @azure_container.nil?
               raise Fluent::ConfigError, "azure_container is needed"
             end
@@ -97,7 +95,7 @@ module Fluent::Plugin
             else
                 @final_file_extension = @compressor.ext
             end
-
+            @values_for_object_chunk = {}
         end
 
         def multi_workers_ready?
@@ -109,12 +107,20 @@ module Fluent::Plugin
             if !@skip_container_check
                 if @failsafe_container_check
                     begin
-                        ensure_container
+                        if @write_only && @auto_create_container
+                            create_container
+                        else
+                            ensure_container
+                        end
                     rescue Exception => e
                         log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
                     end
                 else
-                    ensure_container
+                    if @write_only && @auto_create_container
+                        create_container
+                    else
+                        ensure_container
+                    end
                 end
             end
             super
@@ -126,17 +132,16 @@ module Fluent::Plugin
         end
 
         def write(chunk)
-            metadata = chunk.metadata
             if @store_as.nil? || @store_as == "none"
-                generate_log_name(metadata, @current_index)
+                generate_log_name(chunk, @current_index)
                 if @last_azure_storage_path != @azure_storage_path
                     @current_index = 0
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                 end
                 raw_data = chunk.read
                 unless raw_data.empty?
                     log.debug "azurestorage_gen2: processing raw data", chunk_id: dump_unique_id_hex(chunk.unique_id)
-                    upload_blob(raw_data, metadata)
+                    upload_blob(raw_data, chunk)
                 end
                 chunk.close rescue nil
                 @last_azure_storage_path = @azure_storage_path
@@ -146,51 +151,76 @@ module Fluent::Plugin
                 begin
                     @compressor.compress(chunk, tmp)
                     tmp.rewind
-                    generate_log_name(metadata, @current_index)
+                    generate_log_name(chunk, @current_index)
                     if @last_azure_storage_path != @azure_storage_path
                         @current_index = 0
-                        generate_log_name(metadata, @current_index)
+                        generate_log_name(chunk, @current_index)
                     end
                     log.debug "azurestorage_gen2: Start uploading temp file: #{tmp.path}"
                     content = File.open(tmp.path, 'rb') { |file| file.read }
-                    upload_blob(content, metadata)
+                    upload_blob(content, chunk)
                     @last_azure_storage_path = @azure_storage_path
                 ensure
                     tmp.close(true) rescue nil
                 end
+                @values_for_object_chunk.delete(chunk.unique_id)
             end
 
         end
 
         private
-        def upload_blob(content, metadata)
+        def upload_blob(content, chunk)
             log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
-            existing_content_length = get_blob_properties(@azure_storage_path)
-            if existing_content_length == 0
+            if @write_only
                 create_blob(@azure_storage_path)
+                append_blob(content, chunk, 0)
+            else
+                existing_content_length = get_blob_properties(@azure_storage_path)
+                if existing_content_length == 0
+                    create_blob(@azure_storage_path)
+                end
+                append_blob(content, chunk, existing_content_length)
             end
-            append_blob(content, metadata, existing_content_length)
         end
 
         private
-        def generate_log_name(metadata, index)
+        def generate_log_name(chunk, index)
+            metadata = chunk.metadata
             time_slice = if metadata.timekey.nil?
                        ''.freeze
                      else
                        Time.at(metadata.timekey).utc.strftime(@time_slice_format)
                      end
-            path = @path_slicer.call(@path)
-            values_for_object_key = {
-                "%{path}" => path,
-                "%{time_slice}" => time_slice,
+            if @localtime
+                hms_slicer = Time.now.strftime("%H%M%S")
+                upload_timestamp = Time.now.strftime(@upload_timestamp_format)
+            else
+                hms_slicer = Time.now.utc.strftime("%H%M%S")
+                upload_timestamp = Time.now.utc.strftime(@upload_timestamp_format)
+            end
+
+            @values_for_object_chunk[chunk.unique_id] ||= {
+                "%{hex_random}" => hex_random(chunk),
+            }
+            values_for_object_key_pre = {
+                "%{path}" => @path,
                 "%{index}" => index,
                 "%{uuid_flush}" => uuid_random,
-                "%{file_extension}" => @final_file_extension
+                "%{file_extension}" => @final_file_extension,
+                "%{upload_timestamp}" => upload_timestamp,
             }
-            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+}), values_for_object_key)
-            extracted_path = extract_placeholders(storage_path, metadata)
-            extracted_path = "/" + extracted_path unless extracted_path.start_with?("/")
-            @azure_storage_path = extracted_path
+            values_for_object_key_post = {
+                "%{date_slice}" => time_slice,
+                "%{time_slice}" => time_slice,
+                "%{hms_slice}" => hms_slicer,
+            }.merge!(@values_for_object_chunk[chunk.unique_id])
+            storage_path = @azure_object_key_format.gsub(%r(%{[^}]+})) do |matched_key|
+                values_for_object_key_pre.fetch(matched_key, matched_key)
+            end
+            storage_path = extract_placeholders(storage_path, chunk)
+            storage_path = storage_path.gsub(%r(%{[^}]+}), values_for_object_key_post)
+            storage_path = "/" + storage_path unless storage_path.start_with?("/")
+            @azure_storage_path = storage_path
         end
 
         def setup_access_token
@@ -242,11 +272,16 @@ module Fluent::Plugin
         # https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/tutorial-linux-vm-access-storage#get-an-access-token-and-use-it-to-call-azure-storage
         private
         def acquire_access_token_msi
-            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/" }
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}" }
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
             end
-            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
+            req_opts = {
+                :params => params,
+                :headers => { Metadata: "true" }
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -261,10 +296,16 @@ module Fluent::Plugin
 
         private
         def acquire_access_token_oauth_app
-            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "https://storage.azure.com/"}
+            params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
-            content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=https://storage.azure.com/"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
+            content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=#{@url_storage_resource}"
+            req_opts = {
+                :params => params,
+                :body => content, 
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -279,7 +320,7 @@ module Fluent::Plugin
 
         private
         def acquire_access_token_by_az
-            access_token=`az account get-access-token --resource https://storage.azure.com/ --query accessToken -o tsv`
+            access_token=`az account get-access-token --resource #{@url_storage_resource} --query accessToken -o tsv`
             log.debug "azurestorage_gen2: Token response: #{access_token}"
             @azure_access_token = access_token.chomp
         end
@@ -291,7 +332,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -318,7 +365,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
@@ -338,7 +391,13 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
@@ -361,7 +420,14 @@ module Fluent::Plugin
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers,
+                :body => content
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -386,7 +452,13 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
@@ -407,7 +479,13 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
@@ -426,7 +504,7 @@ module Fluent::Plugin
         end
 
         private
-        def append_blob(content, metadata, existing_content_length)
+        def append_blob(content, chunk, existing_content_length)
           position = 0
           log.debug "azurestorage_gen2: append_blob.start: Content size: #{content.length}"
           loop do
@@ -475,6 +553,16 @@ module Fluent::Plugin
                 "SharedKey #{@azure_storage_account}:#{signed(method, datestamp, resource, headers, params)}"
             end
         end
+        
+        private
+        def add_proxy_options(req_opts = {})
+            unless @proxy_url.nil?
+                req_opts[:proxy] = @proxy_url
+                unless @proxy_username.nil? || @proxy_password.nil?
+                    req_opts[:proxyuserpwd] = "#{@proxy_username}:#{@proxy_password}"
+                end
+            end
+        end
 
         private
         def signed(method, datestamp, resource, headers, params)
@@ -544,6 +632,12 @@ module Fluent::Plugin
             require 'uuidtools'
             ::UUIDTools::UUID.random_create.to_s
         end
+
+        def hex_random(chunk)
+            unique_hex = Fluent::UniqueId.hex(chunk.unique_id)
+            unique_hex.reverse!
+            unique_hex[0...@hex_random_length]
+        end
         
         def timekey_to_timeformat(timekey)
             case timekey

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.3.1
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-20 00:00:00.000000000 Z
+date: 2020-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -104,6 +104,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement