fluent-plugin-azure-storage-append-blob-lts 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1dca8c31f6ec0f0383d24637d3c0b6225d92e8505387a5c1a422d85057f0340
-  data.tar.gz: 240d435415a53386a245d3449d092eb2d71a85c2e08e989b841ec71b7a20a9f5
+  metadata.gz: 0d55d827bbb8aa8906536f7593257e42abf9c6b24e5aa485176cd5aa15284732
+  data.tar.gz: ff055bb2b5585ee3a38b2ed18b75ac1aa8ba6a20654be97b13a920a6566472e5
 SHA512:
-  metadata.gz: 7744655dfad7169b43df0db53a20c781276d8dba8a83c2858e1809cb281008e9b401b6755ec6fb0efa4eff0311c2e76fc2d2e60164ffd373ef0087b81fabc9cf
-  data.tar.gz: 046e1e69571d6502d6c06d52d49c3e97234f4b2a44e15210122941d6787bac825769b40f2c1e70cff926f8d5a6ca9cf916ee2aac42b06a82367c7b9c0f410279
+  metadata.gz: ea451b6605f9126c9af9e697cacccac03145631926684ac7d001eb6c18ca22755ec13cd647eb3b016cbed48678b890a61ff829644cfd041ef4c88a7495cf215c
+  data.tar.gz: e50e9381970ce666d3bc6337526be2e5620d6eced418b9df02a26d6f80f13ce0fb31a45e0cdef3a55090afc6276c328a660b66137005d17b1756948d77ac7491

data/Dockerfile

@@ -1,8 +1,8 @@
-FROM ruby:latest
+FROM ruby:2.7
 
 WORKDIR /plugin
 
-ADD . /plugin
+COPY . /plugin
 
 RUN gem install bundler && \
     gem install fluentd --no-doc && \

data/README.md

@@ -24,6 +24,7 @@ And then execute:
     <match pattern>
       type azure-storage-append-blob
 
+      azure_cloud                       <azure cloud environment>
       azure_storage_account             <your azure storage account>
       azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
       azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
@@ -36,6 +37,7 @@ And then execute:
       path                              logs/
       azure_object_key_format           %{path}%{time_slice}_%{index}.log
       time_slice_format                 %Y%m%d-%H
+      calculate_checksums               true
       # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
       # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
       <buffer tag,time>
@@ -47,6 +49,12 @@ And then execute:
       </buffer>
     </match>
 
+### `azure_cloud` (Optional)
+
+Default: `AZUREPUBLICCLOUD`
+
+Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
+
 ### `azure_storage_account` (Required)
 
 Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
@@ -60,7 +68,7 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt
 
 ### `azure_imds_api_version` (Optional, only for MSI)
 
-Default: 2019-08-15
+Default: `2019-08-15`
 
 The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
 
@@ -68,7 +76,7 @@ See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instanc
 
 ### `azure_token_refresh_interval` (Optional, only for MSI)
 
-Default: 60 (1 hour)
+Default: `60` (1 hour)
 
 When using MSI, the initial access token needs to be refreshed periodically.
 
@@ -82,8 +90,9 @@ Azure Identity Client ID to use for accessing Azure Blob service.
 
 ### `auto_create_container`
 
+Default: `true`
+
 This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
-The default value is `true`
 
 ### `azure_object_key_format`
 
@@ -127,6 +136,14 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
 
 Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
 
+### `calculate_checksums`
+
+Default: `true`
+
+Whether to calculate MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
+
+You want to set it to `false` in FIPS-enabled environments.
+
 ### Run tests
 
     gem install bundler

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
-  spec.version = '0.5.0'
+  spec.version = '0.6.0'
   spec.authors = ['Jonas-Taha El Sesiy']
   spec.email = ['github@elsesiy.com']
 

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -11,7 +11,7 @@ require 'json'
 
 module Fluent
   module Plugin
-    class AzureStorageAppendBlobOut < Fluent::Plugin::Output
+    class AzureStorageAppendBlobOut < Output
       Fluent::Plugin.register_output('azure-storage-append-blob', self)
 
       helpers :formatter, :inject
@@ -19,18 +19,32 @@ module Fluent
       DEFAULT_FORMAT_TYPE = 'out_file'.freeze
       AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
 
+      def initialize
+        super
+        @use_msi = false
+
+        # Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
+        @storage_endpoint_mapping = {
+          'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
+          'AZUREGERMANCLOUD' => 'core.cloudapi.de',
+          'AZUREPUBLICCLOUD' => 'core.windows.net',
+          'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
+        }
+      end
+
       config_param :path, :string, default: ''
       config_param :azure_storage_account, :string, default: nil
       config_param :azure_storage_access_key, :string, default: nil, secret: true
-      config_param :azure_storage_sas_token, :string, default: nil, secret: true
       config_param :azure_storage_connection_string, :string, default: nil, secret: true
+      config_param :azure_storage_sas_token, :string, default: nil, secret: true
+      config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
       config_param :azure_msi_client_id, :string, default: nil
       config_param :azure_container, :string, default: nil
       config_param :azure_imds_api_version, :string, default: '2019-08-15'
       config_param :azure_token_refresh_interval, :integer, default: 60
-      config_param :use_msi, :bool, default: false
       config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
       config_param :auto_create_container, :bool, default: true
+      config_param :calculate_checksums, :bool, default: true
       config_param :format, :string, default: DEFAULT_FORMAT_TYPE
       config_param :time_slice_format, :string, default: '%Y%m%d'
       config_param :localtime, :bool, default: false
@@ -61,6 +75,11 @@ module Fluent
                          end
                        end
 
+        @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
+        if @azure_storage_dns_suffix.nil?
+          raise ConfigError 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD'
+        end
+
         if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
            (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
            (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
@@ -70,6 +89,12 @@ module Fluent
           raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
           raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
         end
+
+        @blob_options = {}
+
+        if !@calculate_checksums
+          @blob_options[:content_md5] = ''
+        end
       end
 
       def multi_workers_ready?
@@ -80,13 +105,13 @@ module Fluent
         access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
                                          "api-version=#{@azure_imds_api_version}" \
                                          '&resource=https://storage.azure.com/' \
-                                         "#{! azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
+                                         "#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
                                          headers: { 'Metadata' => 'true' }).get
 
         if access_key_request.status == 200
           JSON.parse(access_key_request.body)['access_token']
         else
-          raise "Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter."
+          raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
         end
       end
 
@@ -95,7 +120,11 @@ module Fluent
         if @use_msi
           token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
           token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
-          @bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
+          @bs = Azure::Storage::Blob::BlobService.new(
+            storage_account_name: @azure_storage_account,
+            storage_dns_suffix: @azure_storage_dns_suffix,
+            signer: token_signer
+          )
 
           refresh_interval = @azure_token_refresh_interval * 60
           cancelled = false
@@ -112,7 +141,7 @@ module Fluent
         elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
           @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
         else
-          @bs_params = { storage_account_name: @azure_storage_account }
+          @bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
 
           if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
             @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
@@ -152,7 +181,11 @@ module Fluent
           append_blob(content, metadata)
           @last_azure_storage_path = @azure_storage_path
         ensure
-          tmp.close(true) rescue nil
+          begin
+            tmp.close(true)
+          rescue StandardError
+            nil
+          end
         end
       end
 
@@ -202,34 +235,32 @@ module Fluent
         position = 0
         log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
         loop do
-          begin
-            size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
-            log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
-            @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1])
-            position += size
-            break if position >= content.length
-          rescue Azure::Core::Http::HTTPError => e
-            status_code = e.status_code
-
-            if status_code == 409 # exceeds azure block limit
-              @current_index += 1
-              old_azure_storage_path = @azure_storage_path
-              generate_log_name(metadata, @current_index)
-
-              # If index is not a part of format, rethrow exception.
-              if old_azure_storage_path == @azure_storage_path
-                log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
-                raise
-              end
-
-              log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
-              @bs.create_append_blob(@azure_container, @azure_storage_path)
-            elsif status_code == 404 # blob not found
-              log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
-              @bs.create_append_blob(@azure_container, @azure_storage_path)
-            else
+          size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+          log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
+          @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
+          position += size
+          break if position >= content.length
+        rescue Azure::Core::Http::HTTPError => e
+          status_code = e.status_code
+
+          if status_code == 409 # exceeds azure block limit
+            @current_index += 1
+            old_azure_storage_path = @azure_storage_path
+            generate_log_name(metadata, @current_index)
+
+            # If index is not a part of format, rethrow exception.
+            if old_azure_storage_path == @azure_storage_path
+              log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
               raise
             end
+
+            log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          elsif status_code == 404 # blob not found
+            log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          else
+            raise
           end
         end
         log.debug 'azure_storage_append_blob: append_blob.complete'

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -11,6 +11,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
   end
 
   CONFIG = %(
+    azure_cloud AZUREGERMANCLOUD
     azure_storage_account test_storage_account
     azure_storage_access_key MY_FAKE_SECRET
     azure_container test_container
@@ -55,6 +56,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
 
     test 'config with access key should set instance variables' do
       d = create_driver
+      assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
       assert_equal 'test_storage_account', d.instance.azure_storage_account
       assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
       assert_equal 'test_container', d.instance.azure_container
@@ -66,7 +68,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
       d = create_driver conf: MSI_CONFIG
       assert_equal 'test_storage_account', d.instance.azure_storage_account
       assert_equal 'test_container', d.instance.azure_container
-      assert_equal true, d.instance.use_msi
+      assert_equal true, d.instance.instance_variable_get(:@use_msi)
       assert_equal true, d.instance.auto_create_container
       assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
       assert_equal 120, d.instance.azure_token_refresh_interval
@@ -76,7 +78,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     test 'config with connection string should set instance variables' do
       d = create_driver conf: CONNSTR_CONFIG
       assert_equal 'https://test', d.instance.azure_storage_connection_string
-      assert_equal false, d.instance.use_msi
+      assert_equal false, d.instance.instance_variable_get(:@use_msi)
       assert_equal true, d.instance.auto_create_container
     end
   end
@@ -122,11 +124,11 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     end
     attr_reader :blocks
 
-    def append_blob_block(container, path, data)
+    def append_blob_block(_container, _path, data, options={})
       @blocks.append(data)
     end
 
-    def get_container_properties(container)
+    def get_container_properties(_container)
       unless @response.status_code == 200
         raise Azure::Core::Http::HTTPError.new(@response)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-storage-append-blob-lts
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jonas-Taha El Sesiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-31 00:00:00.000000000 Z
+date: 2021-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler