fluent-plugin-azurestorage-gen2 0.2.10 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b42f7d710c0d096c722704a8737dc9ef17acc34c
-  data.tar.gz: 820a5d16c3523f721fe4c9b60dcd04f4e7369a03
+SHA256:
+  metadata.gz: eeea9ef451a8996e41202f49defb227905aeec16aab55f6f208617517365b929
+  data.tar.gz: 0c2027d20716c613c35ee0a325694225d138f0712fc4ca638962beef9397edea
 SHA512:
-  metadata.gz: 9dd02068916c98e40bd0cd59524ce4a6fed211c0d4b912dd4057ff3e7012f7f423c5c00f9fdc72ce5398911ef0a9990a307b2091315adca95ef7ac28a5ae35ab
-  data.tar.gz: a7bff81b96aea91b67d3cca7cec85fa27edc5e701dd47db993e8ae9dd6d6d1ff980f62c4b6755959935acbe0d5ea52d8908eee4ed682001e0ef65af7013f05f6
+  metadata.gz: 65a5ffaea781a9e74f860f489e91f2fc0f0264a8998ffd694808688e48a826a5f3af5d02c1b07091565324c70ad7d6dcfbdfcbb7278d108eacd01089964cdf6c
+  data.tar.gz: 40f629211e74147a59dadf047809f1b743ef01437542b9c880b0a2fe51907dd182a2db49a90ba9c80c1aa3741a30b739d59452b31a195ffb839529559b6c3c70

data/README.md

@@ -31,6 +31,7 @@ $ gem install fluent-plugin-azurestorage-gen2
   azure_storage_account            mystorageabfs
   azure_container                  mycontainer
   azure_instance_msi               /subscriptions/mysubscriptionid
+  azure_client_id                  <msi client id>
   azure_object_key_format          %{path}-%{index}.%{file_extension}
   azure_oauth_refresh_interval     3600
   time_slice_format                %Y%m%d-%H
@@ -92,6 +93,14 @@ Your Azure Storage Access Key(Primary or Secondary). This also can be got from A
 
 Your Azure Managed Service Identity ID. When storage key authentication is not used, the plugin uses OAuth2 to authenticate as given MSI. This authentication method only works on Azure VM. If the VM has only one MSI assigned, this parameter becomes optional and the only MSI will be used. Otherwise this parameter is required.
 
+### azure_client_id
+
+Azure AD client id is a specific explicit identity to use when authenticating to Azure AD. Mutually exclusive with azure_object_id and azure_instance_msi. (for now, you need to define next to `azure_instance_msi`)
+
+### azure_object_id
+
+Azure AD object id is a specific explicit identity to use when authenticating to Azure AD. Mutually exclusive with azure_client_id and azure_instance_msi. (for now, you need to define next to `azure_instance_msi`)
+
 ### azure_oauth_tenant_id (Preview)
 
 Azure account tenant id from your Azure Directory. Required if OAuth based credential mechanism is used.
@@ -152,6 +161,7 @@ The format of Azure Storage object keys. You can use several built-in variables:
 - %{time_slice}
 - %{index}
 - %{file_extension}
+- %{upload_timestamp}
 
 to decide keys dynamically.
 
@@ -159,6 +169,7 @@ to decide keys dynamically.
 %{time_slice} is the time-slice in text that are formatted with *time_slice_format*.
 %{index} is the sequential number starts from 0, increments when multiple files are uploaded to Azure Storage in the same time slice.
 %{file_extention} is always "gz" for now.
+%{upload_timestamp} is an upload timestamp in text that are formatted with *upload_timestamp_format*. Difference between time_slice and upload_timestamp is that the second one is the actual system timestamp (other one is from the metadata)
 
 The default format is "%{path}%{time_slice}_%{index}.%{file_extension}".
 
@@ -283,10 +294,34 @@ Format of the time used as the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to
 
 The time to wait old logs. Default is 10 minutes.
 
+### upload_timestamp_format
+
+Format of the upload timestamp used as the file name. Can be used instead of index in case of `write_only` option is enabled. Default value is '%H%M%S%L'.
+
 ### utc
 
 Use UTC instead of local time.
 
+### write_only
+
+If that option is enabled, HEAD calls are skipped during blob operations. (so make sure to set the chunk limit to 4MB in order to avoid HEAD operation because of the append operation needs the last position of the uploaded blobs).
+
+### proxy_url
+
+Proxy URL for Azure endpoint.
+
+### proxy_username
+
+Proxy username for Azure proxy endpoint (used only if `proxy_url` is filled)
+
+### proxy_password
+
+Proxy password for Azure `proxy_username` (used only if `proxy_url` is filled)
+
+### http_timeout_seconds
+
+The time limit for HTTP request in seconds. Default: 120
+
 ## TODOs
 
 - add storage key support

data/VERSION

@@ -1 +1 @@
-0.2.10
+0.3.3

data/lib/fluent/plugin/out_azurestorage_gen2.rb

@@ -24,6 +24,8 @@ module Fluent::Plugin
         config_param :azure_storage_account, :string, :default => nil
         config_param :azure_storage_access_key, :string, :default => nil, :secret => true
         config_param :azure_instance_msi, :string, :default => nil
+        config_param :azure_client_id, :string, :default => nil
+        config_param :azure_object_id, :string, :default => nil
         config_param :azure_oauth_app_id, :string, :default => nil, :secret => true
         config_param :azure_oauth_secret, :string, :default => nil, :secret => true
         config_param :azure_oauth_tenant_id, :string, :default => nil
@@ -44,6 +46,12 @@ module Fluent::Plugin
         config_param :time_slice_format, :string, :default => '%Y%m%d'
         config_param :hex_random_length, :integer, default: 4
         config_param :command_parameter, :string, :default => nil
+        config_param :proxy_url, :string, :default => nil
+        config_param :proxy_username, :string, :default => nil
+        config_param :proxy_password, :string, :default => nil, :secret => true
+        config_param :write_only, :bool, :default => false
+        config_param :upload_timestamp_format, :string, :default => '%H%M%S%L'
+        config_param :http_timeout_seconds, :integer, :default => 120
 
         DEFAULT_FORMAT_TYPE = "out_file"
         ACCESS_TOKEN_API_VERSION = "2018-02-01"
@@ -53,7 +61,7 @@ module Fluent::Plugin
         config_section :format do
             config_set_default :@type, DEFAULT_FORMAT_TYPE
         end
-  
+
         config_section :buffer do
             config_set_default :chunk_keys, ['time']
             config_set_default :timekey, (60 * 60 * 24)
@@ -76,7 +84,7 @@ module Fluent::Plugin
             end
 
             @formatter = formatter_create
-      
+
             if @azure_container.nil?
               raise Fluent::ConfigError, "azure_container is needed"
             end
@@ -102,12 +110,20 @@ module Fluent::Plugin
             if !@skip_container_check
                 if @failsafe_container_check
                     begin
-                        ensure_container
+                        if @write_only && @auto_create_container
+                            create_container
+                        else
+                            ensure_container
+                        end
                     rescue Exception => e
                         log.warn("#{e.message}, container list/create failsafe is enabled. Continue without those operations.")
                     end
                 else
-                    ensure_container
+                    if @write_only && @auto_create_container
+                        create_container
+                    else
+                        ensure_container
+                    end
                 end
             end
             super
@@ -158,11 +174,16 @@ module Fluent::Plugin
         private
         def upload_blob(content, chunk)
             log.debug "azurestorage_gen2: Uploading blob: #{@azure_storage_path}"
-            existing_content_length = get_blob_properties(@azure_storage_path)
-            if existing_content_length == 0
+            if @write_only
                 create_blob(@azure_storage_path)
+                append_blob(content, chunk, 0)
+            else
+                existing_content_length = get_blob_properties(@azure_storage_path)
+                if existing_content_length == 0
+                    create_blob(@azure_storage_path)
+                end
+                append_blob(content, chunk, existing_content_length)
             end
-            append_blob(content, chunk, existing_content_length)
         end
 
         private
@@ -175,8 +196,10 @@ module Fluent::Plugin
                      end
             if @localtime
                 hms_slicer = Time.now.strftime("%H%M%S")
+                upload_timestamp = Time.now.strftime(@upload_timestamp_format)
             else
                 hms_slicer = Time.now.utc.strftime("%H%M%S")
+                upload_timestamp = Time.now.utc.strftime(@upload_timestamp_format)
             end
 
             @values_for_object_chunk[chunk.unique_id] ||= {
@@ -186,7 +209,8 @@ module Fluent::Plugin
                 "%{path}" => @path,
                 "%{index}" => index,
                 "%{uuid_flush}" => uuid_random,
-                "%{file_extension}" => @final_file_extension
+                "%{file_extension}" => @final_file_extension,
+                "%{upload_timestamp}" => upload_timestamp,
             }
             values_for_object_key_post = {
                 "%{date_slice}" => time_slice,
@@ -255,7 +279,19 @@ module Fluent::Plugin
             unless @azure_instance_msi.nil?
                 params[:msi_res_id] = @azure_instance_msi
             end
-            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", params: params, headers: { Metadata: "true"})
+            unless @azure_client_id.nil?
+                params[:client_id] = @azure_client_id
+            end
+            unless @azure_object_id.nil?
+                params[:object_id] = @azure_object_id
+            end
+            req_opts = {
+                :params => params,
+                :headers => { Metadata: "true" },
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("http://169.254.169.254/metadata/identity/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -273,7 +309,14 @@ module Fluent::Plugin
             params = { :"api-version" => ACCESS_TOKEN_API_VERSION, :resource => "#{@url_storage_resource}"}
             headers = {:"Content-Type" => "application/x-www-form-urlencoded"}
             content = "grant_type=client_credentials&client_id=#{@azure_oauth_app_id}&client_secret=#{@azure_oauth_secret}&resource=#{@url_storage_resource}"
-            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", :body => content, :headers => headers)
+            req_opts = {
+                :params => params,
+                :body => content, 
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://login.microsoftonline.com/#{@azure_oauth_tenant_id}/oauth2/token", req_opts)
             request.on_complete do |response|
                 if response.success?
                   data = JSON.parse(response.body)
@@ -300,7 +343,14 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.info "azurestorage_gen2: Container '#{@azure_container}' exists."
@@ -327,7 +377,14 @@ module Fluent::Plugin
             params = {:resource => "filesystem" }
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Container '#{@azure_container}' created, response code: #{response.code}"
@@ -347,7 +404,14 @@ module Fluent::Plugin
             params = {:resource => "file", :recursive => "false"}
             auth_header = create_auth_header("put", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :put, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :put,
+                :params => params,
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been created, response code: #{response.code}"
@@ -370,7 +434,15 @@ module Fluent::Plugin
             params = {:action => "append", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :headers=> headers, :params => params, :body => content)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers,
+                :body => content,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' has been appended, response code: #{response.code}"
@@ -395,7 +467,14 @@ module Fluent::Plugin
             params = {:action => "flush", :position => "#{position}"}
             auth_header = create_auth_header("patch", datestamp, "#{@azure_container}#{blob_path}",headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :patch, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :patch,
+                :params => params,
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                     log.debug "azurestorage_gen2: Blob '#{blob_path}' flush was successful, response code: #{response.code}"
@@ -416,7 +495,14 @@ module Fluent::Plugin
             content_length = -1
             auth_header = create_auth_header("head", datestamp, "#{@azure_container}#{blob_path}", headers, params)
             headers[:Authorization] = auth_header
-            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", :method => :head, :params => params, :headers=> headers)
+            req_opts = {
+                :method => :head,
+                :params => params,
+                :headers => headers,
+                :timeout => @http_timeout_seconds
+            }
+            add_proxy_options(req_opts)
+            request = Typhoeus::Request.new("https://#{azure_storage_account}#{@url_domain_suffix}/#{@azure_container}#{blob_path}", req_opts)
             request.on_complete do |response|
                 if response.success?
                   log.debug "azurestorage_gen2: Get blob properties for '#{blob_path}', response headers: #{response.headers}"
@@ -484,6 +570,16 @@ module Fluent::Plugin
                 "SharedKey #{@azure_storage_account}:#{signed(method, datestamp, resource, headers, params)}"
             end
         end
+        
+        private
+        def add_proxy_options(req_opts = {})
+            unless @proxy_url.nil?
+                req_opts[:proxy] = @proxy_url
+                unless @proxy_username.nil? || @proxy_password.nil?
+                    req_opts[:proxyuserpwd] = "#{@proxy_username}:#{@proxy_password}"
+                end
+            end
+        end
 
         private
         def signed(method, datestamp, resource, headers, params)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azurestorage-gen2
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.3.3
 platform: ruby
 authors:
 - Oliver Szabo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-28 00:00:00.000000000 Z
+date: 2021-12-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -225,8 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Azure Storage output plugin for Fluentd event collector