fluent-plugin-azure-storage-append-blob-lts 0.3.0 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 335e03eeafd0648547d44b98ce7bbbc3dd8a780dc03d3e39a17005a8305c0ba1
-  data.tar.gz: e1361ff0a6cc0d95eec72e2b330861de638afd17601a3ba2d6f81d0a18d7ee9a
+  metadata.gz: 24eb19ac68cd42369ff7f147be24635945aad7c003f4ed789bcb4189d5f9a96e
+  data.tar.gz: f293437b177cf4f7fc7a02694150f03f479c70749a3973d2ffc45a19377b7784
 SHA512:
-  metadata.gz: b2c40956a6eb51ea8a0d19058527fe8c470d48dbf59130671322c99c073353ffb4b9a0f931afcf7707dbc3deda7d44974174214e3045f7d566da876b7f434e93
-  data.tar.gz: 31258dfaf599e2bcfdda8c268aef6295b9720407c93a8b8997376ef192b7024fe15639da751be765c43e562738ad7f50daf479055f9668ef3cd7748e41173dd2
+  metadata.gz: 7be9b8cf563a7c2cf5b10667a8d037ea5a17d84a78ab3a782a937f5c06e0f850804985fef3b5b6eff795e965c4eefa0d690ddd737485929fa4882df416e420c0
+  data.tar.gz: d4bb668e69155459e1b69ce954f9b8434a9dfa5e92dbf5e1324e9a1d50514c2e7b6b34e513f5fcb3621f5ba7b2b2dee5d7dea1cf37788ffe198869f5d7a3a0a4

data/.github/CODEOWNERS

@@ -0,0 +1 @@
+* @elsesiy

data/.github/workflows/publish.yaml

@@ -11,14 +11,12 @@ jobs:
   build:
     name: Build + Publish
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Ruby 2.6
       uses: actions/setup-ruby@v1
       with:
         ruby-version: 2.6.x
-
     - name: Publish to RubyGems
       run: |
         mkdir -p $HOME/.gem

data/.github/workflows/test.yaml

@@ -8,9 +8,7 @@ on:
 
 jobs:
   test:
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Ruby

data/Dockerfile

@@ -1,8 +1,8 @@
-FROM ruby:latest
+FROM ruby:2.7
 
 WORKDIR /plugin
 
-ADD . /plugin
+COPY . /plugin
 
 RUN gem install bundler && \
     gem install fluentd --no-doc && \

data/README.md

@@ -1,5 +1,9 @@
 # fluent-plugin-azure-storage-append-blob-lts
 
+![Tests](https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts/workflows/Test/badge.svg?branch=master)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts.svg)](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts)
+[![Twitter](https://img.shields.io/badge/twitter-@elsesiy-blue.svg)](http://twitter.com/elsesiy)
+
 Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
 This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
 
@@ -24,6 +28,7 @@ And then execute:
     <match pattern>
       type azure-storage-append-blob
 
+      azure_cloud                       <azure cloud environment>
       azure_storage_account             <your azure storage account>
       azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
       azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
@@ -31,10 +36,12 @@ And then execute:
       azure_imds_api_version            <Azure Instance Metadata Service API Version> # only used for MSI
       azure_token_refresh_interval      <refresh interval in min> # only used for MSI
       azure_container                   <your azure storage container>
+      azure_msi_client_id               <Azure Managed Identity Client ID> # only used for MSI
       auto_create_container             true
       path                              logs/
       azure_object_key_format           %{path}%{time_slice}_%{index}.log
       time_slice_format                 %Y%m%d-%H
+      compute_checksums                 true
       # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
       # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
       <buffer tag,time>
@@ -46,6 +53,12 @@ And then execute:
       </buffer>
     </match>
 
+### `azure_cloud` (Optional)
+
+Default: `AZUREPUBLICCLOUD`
+
+Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
+
 ### `azure_storage_account` (Required)
 
 Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
@@ -59,7 +72,7 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt
 
 ### `azure_imds_api_version` (Optional, only for MSI)
 
-Default: 2019-08-15
+Default: `2019-08-15`
 
 The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
 
@@ -67,7 +80,7 @@ See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instanc
 
 ### `azure_token_refresh_interval` (Optional, only for MSI)
 
-Default: 60 (1 hour)
+Default: `60` (1 hour)
 
 When using MSI, the initial access token needs to be refreshed periodically.
 
@@ -75,10 +88,15 @@ When using MSI, the initial access token needs to be refreshed periodically.
 
 Azure Storage Container name
 
+### `azure_msi_client_id` (Optional, only for MSI)
+
+Azure Identity Client ID to use for accessing Azure Blob service.
+
 ### `auto_create_container`
 
+Default: `true`
+
 This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
-The default value is `true`
 
 ### `azure_object_key_format`
 
@@ -122,6 +140,14 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
 
 Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
 
+### `compute_checksums`
+
+Default: `true`
+
+Whether to compute MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
+
+You want to set it to `false` in FIPS-enabled environments.
+
 ### Run tests
 
     gem install bundler

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
-  spec.version = '0.3.0'
+  spec.version = '0.6.2'
   spec.authors = ['Jonas-Taha El Sesiy']
-  spec.email = ['']
+  spec.email = ['github@elsesiy.com']
 
   spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
-  spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob'
+  spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/microsoft/fluent-plugin-azure-storage-append-blob'
   spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
   spec.license = 'MIT'
 

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -11,7 +11,7 @@ require 'json'
 
 module Fluent
   module Plugin
-    class AzureStorageAppendBlobOut < Fluent::Plugin::Output
+    class AzureStorageAppendBlobOut < Output
       Fluent::Plugin.register_output('azure-storage-append-blob', self)
 
       helpers :formatter, :inject
@@ -19,17 +19,32 @@ module Fluent
       DEFAULT_FORMAT_TYPE = 'out_file'.freeze
       AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
 
+      def initialize
+        super
+        @use_msi = false
+
+        # Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
+        @storage_endpoint_mapping = {
+          'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
+          'AZUREGERMANCLOUD' => 'core.cloudapi.de',
+          'AZUREPUBLICCLOUD' => 'core.windows.net',
+          'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
+        }
+      end
+
       config_param :path, :string, default: ''
       config_param :azure_storage_account, :string, default: nil
       config_param :azure_storage_access_key, :string, default: nil, secret: true
-      config_param :azure_storage_sas_token, :string, default: nil, secret: true
       config_param :azure_storage_connection_string, :string, default: nil, secret: true
+      config_param :azure_storage_sas_token, :string, default: nil, secret: true
+      config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
+      config_param :azure_msi_client_id, :string, default: nil
       config_param :azure_container, :string, default: nil
       config_param :azure_imds_api_version, :string, default: '2019-08-15'
       config_param :azure_token_refresh_interval, :integer, default: 60
-      config_param :use_msi, :bool, default: false
       config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
       config_param :auto_create_container, :bool, default: true
+      config_param :compute_checksums, :bool, default: true
       config_param :format, :string, default: DEFAULT_FORMAT_TYPE
       config_param :time_slice_format, :string, default: '%Y%m%d'
       config_param :localtime, :bool, default: false
@@ -60,15 +75,25 @@ module Fluent
                          end
                        end
 
-        raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
-
-        raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
+        @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
+        if @azure_storage_dns_suffix.nil?
+          raise ConfigError 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD'
+        end
 
         if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
            (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
            (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
-          log.info 'Using MSI since neither azure_storage_access_key nor azure_storage_sas_token was provided.'
+          log.info 'Using MSI since neither of azure_storage_access_key, azure_storage_sas_token, azure_storage_connection_string was provided.'
           @use_msi = true
+        elsif @azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?
+          raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
+          raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
+        end
+
+        @blob_options = {}
+
+        if !@compute_checksums
+          @blob_options[:content_md5] = ''
         end
       end
 
@@ -79,11 +104,15 @@ module Fluent
       def get_access_token
         access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
                                          "api-version=#{@azure_imds_api_version}" \
-                                         '&resource=https://storage.azure.com/',
-                                         headers: { 'Metadata' => 'true' })
-                                    .get
-                                    .body
-        JSON.parse(access_key_request)['access_token']
+                                         '&resource=https://storage.azure.com/' \
+                                         "#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
+                                         headers: { 'Metadata' => 'true' }).get
+
+        if access_key_request.status == 200
+          JSON.parse(access_key_request.body)['access_token']
+        else
+          raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
+        end
       end
 
       def start
@@ -91,7 +120,11 @@ module Fluent
         if @use_msi
           token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
           token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
-          @bs = Azure::Storage::Blob::BlobService.new(storage_account_name: @azure_storage_account, signer: token_signer)
+          @bs = Azure::Storage::Blob::BlobService.new(
+            storage_account_name: @azure_storage_account,
+            storage_dns_suffix: @azure_storage_dns_suffix,
+            signer: token_signer
+          )
 
           refresh_interval = @azure_token_refresh_interval * 60
           cancelled = false
@@ -108,7 +141,7 @@ module Fluent
         elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
           @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
         else
-          @bs_params = { storage_account_name: @azure_storage_account }
+          @bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
 
           if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
             @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
@@ -132,23 +165,26 @@ module Fluent
       end
 
       def write(chunk)
-        metadata = chunk.metadata
         tmp = Tempfile.new('azure-')
         begin
           chunk.write_to(tmp)
 
-          generate_log_name(metadata, @current_index)
+          generate_log_name(chunk, @current_index)
           if @last_azure_storage_path != @azure_storage_path
             @current_index = 0
-            generate_log_name(metadata, @current_index)
+            generate_log_name(chunk, @current_index)
           end
 
           content = File.open(tmp.path, 'rb', &:read)
 
-          append_blob(content, metadata)
+          append_blob(content, chunk)
           @last_azure_storage_path = @azure_storage_path
         ensure
-          tmp.close(true) rescue nil
+          begin
+            tmp.close(true)
+          rescue StandardError
+            nil
+          end
         end
       end
 
@@ -177,7 +213,8 @@ module Fluent
         end
       end
 
-      def generate_log_name(metadata, index)
+      def generate_log_name(chunk, index)
+        metadata = chunk.metadata
         time_slice = if metadata.timekey.nil?
                        ''.freeze
                      else
@@ -191,41 +228,39 @@ module Fluent
           '%{index}' => index
         }
         storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
-        @azure_storage_path = extract_placeholders(storage_path, metadata)
+        @azure_storage_path = extract_placeholders(storage_path, chunk)
       end
 
-      def append_blob(content, metadata)
+      def append_blob(content, chunk)
         position = 0
         log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
         loop do
-          begin
-            size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
-            log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
-            @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1])
-            position += size
-            break if position >= content.length
-          rescue Azure::Core::Http::HTTPError => e
-            status_code = e.status_code
-
-            if status_code == 409 # exceeds azure block limit
-              @current_index += 1
-              old_azure_storage_path = @azure_storage_path
-              generate_log_name(metadata, @current_index)
-
-              # If index is not a part of format, rethrow exception.
-              if old_azure_storage_path == @azure_storage_path
-                log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
-                raise
-              end
-
-              log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
-              @bs.create_append_blob(@azure_container, @azure_storage_path)
-            elsif status_code == 404 # blob not found
-              log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
-              @bs.create_append_blob(@azure_container, @azure_storage_path)
-            else
+          size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
+          log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
+          @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
+          position += size
+          break if position >= content.length
+        rescue Azure::Core::Http::HTTPError => e
+          status_code = e.status_code
+
+          if status_code == 409 # exceeds azure block limit
+            @current_index += 1
+            old_azure_storage_path = @azure_storage_path
+            generate_log_name(chunk, @current_index)
+
+            # If index is not a part of format, rethrow exception.
+            if old_azure_storage_path == @azure_storage_path
+              log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
               raise
             end
+
+            log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          elsif status_code == 404 # blob not found
+            log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
+            @bs.create_append_blob(@azure_container, @azure_storage_path)
+          else
+            raise
           end
         end
         log.debug 'azure_storage_append_blob: append_blob.complete'

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -11,6 +11,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
   end
 
   CONFIG = %(
+    azure_cloud AZUREGERMANCLOUD
     azure_storage_account test_storage_account
     azure_storage_access_key MY_FAKE_SECRET
     azure_container test_container
@@ -18,6 +19,12 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     path log
   ).freeze
 
+  CONNSTR_CONFIG = %(
+    azure_storage_connection_string https://test
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
   MSI_CONFIG = %(
     azure_storage_account test_storage_account
     azure_container test_container
@@ -49,6 +56,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
 
     test 'config with access key should set instance variables' do
       d = create_driver
+      assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
       assert_equal 'test_storage_account', d.instance.azure_storage_account
       assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
       assert_equal 'test_container', d.instance.azure_container
@@ -60,12 +68,19 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
       d = create_driver conf: MSI_CONFIG
       assert_equal 'test_storage_account', d.instance.azure_storage_account
       assert_equal 'test_container', d.instance.azure_container
-      assert_equal true, d.instance.use_msi
+      assert_equal true, d.instance.instance_variable_get(:@use_msi)
       assert_equal true, d.instance.auto_create_container
       assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
       assert_equal 120, d.instance.azure_token_refresh_interval
       assert_equal '1970-01-01', d.instance.azure_imds_api_version
     end
+
+    test 'config with connection string should set instance variables' do
+      d = create_driver conf: CONNSTR_CONFIG
+      assert_equal 'https://test', d.instance.azure_storage_connection_string
+      assert_equal false, d.instance.instance_variable_get(:@use_msi)
+      assert_equal true, d.instance.auto_create_container
+    end
   end
 
   sub_test_case 'test path slicing' do
@@ -92,15 +107,13 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
   # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
   # a response object when it is created.
   class FakeResponse
-    def initialize(status=404)
+    def initialize(status = 404)
       @status = status
-      @body = "body"
+      @body = 'body'
       @headers = {}
     end
 
-    attr_reader :status
-    attr_reader :body
-    attr_reader :headers
+    attr_reader :status, :body, :headers
   end
 
   # This class is used to test plugin functions which interact with the blob service
@@ -111,11 +124,11 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     end
     attr_reader :blocks
 
-    def append_blob_block(container, path, data)
+    def append_blob_block(_container, _path, data, options={})
       @blocks.append(data)
     end
 
-    def get_container_properties(container)
+    def get_container_properties(_container)
       unless @response.status_code == 200
         raise Azure::Core::Http::HTTPError.new(@response)
       end
@@ -125,18 +138,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
   sub_test_case 'test container_exists' do
     test 'container 404 returns false' do
       d = create_driver service: FakeBlobService.new(404)
-      assert_false d.instance.container_exists? "anything"
+      assert_false d.instance.container_exists? 'anything'
     end
 
     test 'existing container returns true' do
       d = create_driver service: FakeBlobService.new(200)
-      assert_true d.instance.container_exists? "anything"
+      assert_true d.instance.container_exists? 'anything'
     end
 
     test 'unexpected exception raises' do
       d = create_driver service: FakeBlobService.new(500)
       assert_raise_kind_of Azure::Core::Http::HTTPError do
-        d.instance.container_exists? "anything"
+        d.instance.container_exists? 'anything'
       end
     end
   end
@@ -175,10 +188,10 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
 
     test 'long buffer appends multiple times' do
       limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
-      buf_1 = 'a' * limit
-      buf_2 = 'a' * 3
-      blocks = fake_appended_blocks buf_1 + buf_2
-      assert_equal [buf_1, buf_2], blocks
+      buf1 = 'a' * limit
+      buf2 = 'a' * 3
+      blocks = fake_appended_blocks buf1 + buf2
+      assert_equal [buf1, buf2], blocks
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-storage-append-blob-lts
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.2
 platform: ruby
 authors:
 - Jonas-Taha El Sesiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-10 00:00:00.000000000 Z
+date: 2021-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -87,13 +87,14 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
-  https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob
+  https://github.com/microsoft/fluent-plugin-azure-storage-append-blob
 email:
-- ''
+- github@elsesiy.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/CODEOWNERS"
 - ".github/workflows/publish.yaml"
 - ".github/workflows/test.yaml"
 - ".gitignore"