fluent-plugin-azurestorage-gen2 0.2.10 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b42f7d710c0d096c722704a8737dc9ef17acc34c
-  data.tar.gz: 820a5d16c3523f721fe4c9b60dcd04f4e7369a03
+  metadata.gz: f2ae08b739ba4ac4992427d2f87e8c5dc923d39b
+  data.tar.gz: 658ff0d35909232a1741d43bfa95bfd00cef3957
 SHA512:
-  metadata.gz: 9dd02068916c98e40bd0cd59524ce4a6fed211c0d4b912dd4057ff3e7012f7f423c5c00f9fdc72ce5398911ef0a9990a307b2091315adca95ef7ac28a5ae35ab
-  data.tar.gz: a7bff81b96aea91b67d3cca7cec85fa27edc5e701dd47db993e8ae9dd6d6d1ff980f62c4b6755959935acbe0d5ea52d8908eee4ed682001e0ef65af7013f05f6
+  metadata.gz: 5a122fb0024c716cfbe9a6aa45f733ec938a28a0059bbb66f2d49ae885533ddb529247b757927abcc4784bfc9b78ab5f3593146cad5fbd562ac972cff15503bc
+  data.tar.gz: 99103e68b7c4995a7c495069eb302cfd006378c5477642bb03d76d5b9ea2462e3820a11ebc21a38c5383dc26ba5d5f3a4151f24ebf1788ddccf5f93eb3ef52e4

data/README.md

@@ -152,6 +152,7 @@ The format of Azure Storage object keys. You can use several built-in variables:
 - %{time_slice}
 - %{index}
 - %{file_extension}
+- %{upload_timestamp}
 
 to decide keys dynamically.
 
@@ -159,6 +160,7 @@ to decide keys dynamically.
 %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
 %{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
 %{file_extention} is always "gz" for now.
+%{upload_timestamp} is an upload timestamp in text that are formatted with *upload_timestamp_format*. Difference between time_slice and upload_timestamp is that the second one is the actual system timestamp (other one is from the metadata)
 
 The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
 
@@ -283,10 +285,30 @@ Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to
 
 The time to wait old logs. Default is 10 minutes.
 
+### upload_timestamp_format
+
+Format of the upload timestamp used as the file name. Can be used instead of index in case of `write_only` option is enabled. Default value is '%H%M%S%L'.
+
 ### utc
 
 Use UTC instead of local time.
 
+### write_only
+
+If that option is enabled, HEAD calls are skipped during blob operations. (so make sure to set the chunk limit to 4MB in order to avoid HEAD operation because of the append operation needs the last position of the uploaded blobs).
+
+### proxy_url
+
+Proxy URL for Azure endpoint.
+
+### proxy_username
+
+Proxy username for Azure proxy endpoint (used only if `proxy_url` is filled)
+
+### proxy_password
+
+Proxy password for Azure `proxy_username` (used only if `proxy_url` is filled)
+
 ## TODOs
 
 - add storage key support

data/VERSION

@@ -1 +1 @@
-0.2.10
+0.3.0

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -44,6 +44,11 @@ module Fluent::Plugin
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :hex_random_length, :integer, default: 4
         config_param :command_parameter, :string, :default => nil
+        config_param :proxy_url, :string, :default => nil
+        config_param :proxy_username, :string, :default => nil
+        config_param :proxy_password, :string, :default => nil, :secret => true
+        config_param :write_only, :bool, :default => false
+        config_param :upload_timestamp_format, :string, :default => '%H%M%S%L'
 
         DEFAULT_FORMAT_TYPE = "out_file"
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
@@ -158,11 +163,16 @@ module Fluent::Plugin
         private
         def upload_blob(content, chunk)
             log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
-            existing_content_length = get_blob_properties(@azure_storage_path)
-            if existing_content_length == 0
+            if @write_only
                 create_blob(@azure_storage_path)
+                append_blob(content, chunk, 0)
+            else
+                existing_content_length = get_blob_properties(@azure_storage_path)
+                if existing_content_length == 0
+                    create_blob(@azure_storage_path)
+                end
+                append_blob(content, chunk, existing_content_length)
             end
-            append_blob(content, chunk, existing_content_length)
         end
 
         private
@@ -175,8 +185,10 @@ module Fluent::Plugin
                      end
             if @localtime
                 hms_slicer = Time.now.strftime("%H%M%S")
+                upload_timestamp = Time.now.strftime(@upload_timestamp_format)
             else
                 hms_slicer = Time.now.utc.strftime("%H%M%S")
+                upload_timestamp = Time.now.utc.strftime(@upload_timestamp_format)
             end
 
             @values_for_object_chunk[chunk.unique_id] ||= {
@@ -186,7 +198,8 @@ module Fluent::Plugin
                 "%{path}" => @path,
                 "%{index}" => index,
                 "%{uuid_flush}" => uuid_random,
-                "%{file_extension}" => @final_file_extension
+                "%{file_extension}" => @final_file_extension,
+                "%{upload_timestamp}" => upload_timestamp,
             }
             values_for_object_key_post = {
                 "%{date_slice}" => time_slice,
@@ -255,7 +268,12 @@ module Fluent::Plugin
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
             end
-            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
+            req_opts = {
+                :params => params,
+                :headers => { Metadata: "true" }
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -273,7 +291,13 @@ module Fluent::Plugin
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
             content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=#{@url_storage_resource}"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
+            req_opts = {
+                :params => params,
+                :body => content, 
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -300,7 +324,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -327,7 +357,13 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
@@ -347,7 +383,13 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
@@ -370,7 +412,14 @@ module Fluent::Plugin
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers,
+                :body => content
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -395,7 +444,13 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
@@ -416,7 +471,13 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
@@ -484,6 +545,16 @@ module Fluent::Plugin
                 "SharedKey #{@azure_storage_account}:#{signed(method, datestamp, resource, headers, params)}"
             end
         end
+        
+        private
+        def add_proxy_options(req_opts = {})
+            unless @proxy_url.nil?
+                req_opts[:proxy] = @proxy_url
+                unless @proxy_username.nil? || @proxy_password.nil?
+                    req_opts[:proxyuserpwd] = "#{@proxy_username}:#{@proxy_password}"
+                end
+            end
+        end
 
         private
         def signed(method, datestamp, resource, headers, params)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.3.0
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-28 00:00:00.000000000 Z
+date: 2020-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd