fluent-plugin-azure-storage-append-blob-lts 0.4.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/CODEOWNERS +1 -0
- data/.github/workflows/publish.yaml +0 -2
- data/.github/workflows/test.yaml +0 -2
- data/Dockerfile +2 -2
- data/README.md +32 -4
- data/fluent-plugin-azure-storage-append-blob-lts.gemspec +3 -3
- data/lib/fluent/plugin/out_azure-storage-append-blob.rb +83 -46
- data/test/plugin/test_out_azure_storage_append_blob.rb +65 -16
- metadata +5 -4
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA256:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: f9e6de27ebbf624d51968e415d502111523d3fd670d2345558381b03895934d6
         | 
| 4 | 
            +
              data.tar.gz: 80fbe42cee9d82adb02afbaf2cfed1fd92b0d4fbbbfb1e67b50e328795d90b23
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: d3c2e0296f88ae875f19043e322678aa9a7ea3a26e9bb41f9f4292991ff09a6a50022d03c1da48b36e0c724c2e9e83f70ef431e1b9a3fc628507774a8de32943
         | 
| 7 | 
            +
              data.tar.gz: a86038ca1acc515dd05df9f86d014cef337c5cc8a70847a15dd94b8f86fe2233c5b0a49be988f26a3da86f298d1aa27465aab4fb904ddd8eeef9e16535291d99
         | 
    
        data/.github/CODEOWNERS
    ADDED
    
    | @@ -0,0 +1 @@ | |
| 1 | 
            +
            * @elsesiy
         | 
| @@ -11,14 +11,12 @@ jobs: | |
| 11 11 | 
             
              build:
         | 
| 12 12 | 
             
                name: Build + Publish
         | 
| 13 13 | 
             
                runs-on: ubuntu-latest
         | 
| 14 | 
            -
             | 
| 15 14 | 
             
                steps:
         | 
| 16 15 | 
             
                - uses: actions/checkout@v2
         | 
| 17 16 | 
             
                - name: Set up Ruby 2.6
         | 
| 18 17 | 
             
                  uses: actions/setup-ruby@v1
         | 
| 19 18 | 
             
                  with:
         | 
| 20 19 | 
             
                    ruby-version: 2.6.x
         | 
| 21 | 
            -
             | 
| 22 20 | 
             
                - name: Publish to RubyGems
         | 
| 23 21 | 
             
                  run: |
         | 
| 24 22 | 
             
                    mkdir -p $HOME/.gem
         | 
    
        data/.github/workflows/test.yaml
    CHANGED
    
    
    
        data/Dockerfile
    CHANGED
    
    
    
        data/README.md
    CHANGED
    
    | @@ -1,5 +1,9 @@ | |
| 1 1 | 
             
            # fluent-plugin-azure-storage-append-blob-lts
         | 
| 2 2 |  | 
| 3 | 
            +
            
         | 
| 4 | 
            +
            [](https://badge.fury.io/rb/fluent-plugin-azure-storage-append-blob-lts)
         | 
| 5 | 
            +
            [](http://twitter.com/elsesiy)
         | 
| 6 | 
            +
             | 
| 3 7 | 
             
            Azure Storage Append Blob output plugin buffers logs in local file and uploads them to Azure Storage Append Blob periodically.
         | 
| 4 8 | 
             
            This plugin is a fork of [microsoft/fluent-plugin-azure-storage-append-blob](https://github.com/microsoft/fluent-plugin-azure-storage-append-blob) which is not actively maintained.
         | 
| 5 9 |  | 
| @@ -24,6 +28,8 @@ And then execute: | |
| 24 28 | 
             
                <match pattern>
         | 
| 25 29 | 
             
                  type azure-storage-append-blob
         | 
| 26 30 |  | 
| 31 | 
            +
                  azure_cloud                       <azure cloud environment>
         | 
| 32 | 
            +
                  azure_storage_dns_suffix          <your azure storage dns suffix> # only used for Azure Stack Cloud
         | 
| 27 33 | 
             
                  azure_storage_account             <your azure storage account>
         | 
| 28 34 | 
             
                  azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
         | 
| 29 35 | 
             
                  azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
         | 
| @@ -36,6 +42,7 @@ And then execute: | |
| 36 42 | 
             
                  path                              logs/
         | 
| 37 43 | 
             
                  azure_object_key_format           %{path}%{time_slice}_%{index}.log
         | 
| 38 44 | 
             
                  time_slice_format                 %Y%m%d-%H
         | 
| 45 | 
            +
                  compute_checksums                 true
         | 
| 39 46 | 
             
                  # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
         | 
| 40 47 | 
             
                  # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
         | 
| 41 48 | 
             
                  <buffer tag,time>
         | 
| @@ -47,6 +54,18 @@ And then execute: | |
| 47 54 | 
             
                  </buffer>
         | 
| 48 55 | 
             
                </match>
         | 
| 49 56 |  | 
| 57 | 
            +
            ### `azure_cloud` (Optional)
         | 
| 58 | 
            +
             | 
| 59 | 
            +
            Default: `AZUREPUBLICCLOUD`
         | 
| 60 | 
            +
             | 
| 61 | 
            +
            Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
         | 
| 62 | 
            +
             | 
| 63 | 
            +
            Use `AZURESTACKCLOUD` for Azure Stack Cloud.
         | 
| 64 | 
            +
             | 
| 65 | 
            +
            ### `azure_storage_dns_suffix` (Required only for Azure Stack Cloud)
         | 
| 66 | 
            +
             | 
| 67 | 
            +
            Your Azure Storage endpoint suffix. This can be retrieved from Azure Storage connection string, `EndpointSuffix` section.
         | 
| 68 | 
            +
             | 
| 50 69 | 
             
            ### `azure_storage_account` (Required)
         | 
| 51 70 |  | 
| 52 71 | 
             
            Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
         | 
| @@ -60,15 +79,15 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt | |
| 60 79 |  | 
| 61 80 | 
             
            ### `azure_imds_api_version` (Optional, only for MSI)
         | 
| 62 81 |  | 
| 63 | 
            -
            Default:  | 
| 82 | 
            +
            Default: `2020-12-01`
         | 
| 64 83 |  | 
| 65 84 | 
             
            The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
         | 
| 66 85 |  | 
| 67 | 
            -
            See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
         | 
| 86 | 
            +
            See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service?tabs=windows#versioning) for more details.
         | 
| 68 87 |  | 
| 69 88 | 
             
            ### `azure_token_refresh_interval` (Optional, only for MSI)
         | 
| 70 89 |  | 
| 71 | 
            -
            Default: 60 (1 hour)
         | 
| 90 | 
            +
            Default: `60` (1 hour)
         | 
| 72 91 |  | 
| 73 92 | 
             
            When using MSI, the initial access token needs to be refreshed periodically.
         | 
| 74 93 |  | 
| @@ -82,8 +101,9 @@ Azure Identity Client ID to use for accessing Azure Blob service. | |
| 82 101 |  | 
| 83 102 | 
             
            ### `auto_create_container`
         | 
| 84 103 |  | 
| 104 | 
            +
            Default: `true`
         | 
| 105 | 
            +
             | 
| 85 106 | 
             
            This plugin creates the Azure container if it does not already exist exist when you set 'auto_create_container' to true.
         | 
| 86 | 
            -
            The default value is `true`
         | 
| 87 107 |  | 
| 88 108 | 
             
            ### `azure_object_key_format`
         | 
| 89 109 |  | 
| @@ -127,6 +147,14 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin | |
| 127 147 |  | 
| 128 148 | 
             
            Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
         | 
| 129 149 |  | 
| 150 | 
            +
            ### `compute_checksums`
         | 
| 151 | 
            +
             | 
| 152 | 
            +
            Default: `true`
         | 
| 153 | 
            +
             | 
| 154 | 
            +
            Whether to compute MD5 checksum of the blob contents during append operation and provide it in a header for the blob service.
         | 
| 155 | 
            +
             | 
| 156 | 
            +
            You want to set it to `false` in FIPS-enabled environments.
         | 
| 157 | 
            +
             | 
| 130 158 | 
             
            ### Run tests
         | 
| 131 159 |  | 
| 132 160 | 
             
                gem install bundler
         | 
| @@ -3,12 +3,12 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) | |
| 3 3 |  | 
| 4 4 | 
             
            Gem::Specification.new do |spec|
         | 
| 5 5 | 
             
              spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
         | 
| 6 | 
            -
              spec.version = '0. | 
| 6 | 
            +
              spec.version = '0.6.3'
         | 
| 7 7 | 
             
              spec.authors = ['Jonas-Taha El Sesiy']
         | 
| 8 | 
            -
              spec.email = ['']
         | 
| 8 | 
            +
              spec.email = ['github@elsesiy.com']
         | 
| 9 9 |  | 
| 10 10 | 
             
              spec.summary = 'Azure Storage Append Blob output plugin for Fluentd event collector'
         | 
| 11 | 
            -
              spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/ | 
| 11 | 
            +
              spec.description = 'Fluentd plugin to upload logs to Azure Storage append blobs. Fork of https://github.com/microsoft/fluent-plugin-azure-storage-append-blob'
         | 
| 12 12 | 
             
              spec.homepage = 'https://github.com/elsesiy/fluent-plugin-azure-storage-append-blob-lts'
         | 
| 13 13 | 
             
              spec.license = 'MIT'
         | 
| 14 14 |  | 
| @@ -11,7 +11,7 @@ require 'json' | |
| 11 11 |  | 
| 12 12 | 
             
            module Fluent
         | 
| 13 13 | 
             
              module Plugin
         | 
| 14 | 
            -
                class AzureStorageAppendBlobOut <  | 
| 14 | 
            +
                class AzureStorageAppendBlobOut < Output
         | 
| 15 15 | 
             
                  Fluent::Plugin.register_output('azure-storage-append-blob', self)
         | 
| 16 16 |  | 
| 17 17 | 
             
                  helpers :formatter, :inject
         | 
| @@ -19,18 +19,33 @@ module Fluent | |
| 19 19 | 
             
                  DEFAULT_FORMAT_TYPE = 'out_file'.freeze
         | 
| 20 20 | 
             
                  AZURE_BLOCK_SIZE_LIMIT = 4 * 1024 * 1024 - 1
         | 
| 21 21 |  | 
| 22 | 
            +
                  def initialize
         | 
| 23 | 
            +
                    super
         | 
| 24 | 
            +
                    @use_msi = false
         | 
| 25 | 
            +
             | 
| 26 | 
            +
                    # Storage endpoint suffixes for various environments, see https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go
         | 
| 27 | 
            +
                    @storage_endpoint_mapping = {
         | 
| 28 | 
            +
                      'AZURECHINACLOUD' => 'core.chinacloudapi.cn',
         | 
| 29 | 
            +
                      'AZUREGERMANCLOUD' => 'core.cloudapi.de',
         | 
| 30 | 
            +
                      'AZUREPUBLICCLOUD' => 'core.windows.net',
         | 
| 31 | 
            +
                      'AZUREUSGOVERNMENTCLOUD' => 'core.usgovcloudapi.net'
         | 
| 32 | 
            +
                    }
         | 
| 33 | 
            +
                  end
         | 
| 34 | 
            +
             | 
| 22 35 | 
             
                  config_param :path, :string, default: ''
         | 
| 23 36 | 
             
                  config_param :azure_storage_account, :string, default: nil
         | 
| 24 37 | 
             
                  config_param :azure_storage_access_key, :string, default: nil, secret: true
         | 
| 25 | 
            -
                  config_param :azure_storage_sas_token, :string, default: nil, secret: true
         | 
| 26 38 | 
             
                  config_param :azure_storage_connection_string, :string, default: nil, secret: true
         | 
| 39 | 
            +
                  config_param :azure_storage_dns_suffix, :string, default: nil
         | 
| 40 | 
            +
                  config_param :azure_storage_sas_token, :string, default: nil, secret: true
         | 
| 41 | 
            +
                  config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
         | 
| 27 42 | 
             
                  config_param :azure_msi_client_id, :string, default: nil
         | 
| 28 43 | 
             
                  config_param :azure_container, :string, default: nil
         | 
| 29 | 
            -
                  config_param :azure_imds_api_version, :string, default: ' | 
| 44 | 
            +
                  config_param :azure_imds_api_version, :string, default: '2020-12-01'
         | 
| 30 45 | 
             
                  config_param :azure_token_refresh_interval, :integer, default: 60
         | 
| 31 | 
            -
                  config_param :use_msi, :bool, default: false
         | 
| 32 46 | 
             
                  config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
         | 
| 33 47 | 
             
                  config_param :auto_create_container, :bool, default: true
         | 
| 48 | 
            +
                  config_param :compute_checksums, :bool, default: true
         | 
| 34 49 | 
             
                  config_param :format, :string, default: DEFAULT_FORMAT_TYPE
         | 
| 35 50 | 
             
                  config_param :time_slice_format, :string, default: '%Y%m%d'
         | 
| 36 51 | 
             
                  config_param :localtime, :bool, default: false
         | 
| @@ -61,15 +76,31 @@ module Fluent | |
| 61 76 | 
             
                                     end
         | 
| 62 77 | 
             
                                   end
         | 
| 63 78 |  | 
| 64 | 
            -
                     | 
| 65 | 
            -
             | 
| 66 | 
            -
             | 
| 79 | 
            +
                    if @azure_cloud == 'AZURESTACKCLOUD'
         | 
| 80 | 
            +
                      if @azure_storage_dns_suffix.nil?
         | 
| 81 | 
            +
                        raise ConfigError, 'azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD'
         | 
| 82 | 
            +
                      end
         | 
| 83 | 
            +
                    else
         | 
| 84 | 
            +
                      @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
         | 
| 85 | 
            +
                      if @azure_storage_dns_suffix.nil?
         | 
| 86 | 
            +
                        raise ConfigError, 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD'
         | 
| 87 | 
            +
                      end
         | 
| 88 | 
            +
                    end
         | 
| 67 89 |  | 
| 68 90 | 
             
                    if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
         | 
| 69 91 | 
             
                       (@azure_storage_sas_token.nil? || @azure_storage_sas_token.empty?) &&
         | 
| 70 92 | 
             
                       (@azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?)
         | 
| 71 | 
            -
                      log.info 'Using MSI since neither azure_storage_access_key  | 
| 93 | 
            +
                      log.info 'Using MSI since neither of azure_storage_access_key, azure_storage_sas_token, azure_storage_connection_string was provided.'
         | 
| 72 94 | 
             
                      @use_msi = true
         | 
| 95 | 
            +
                    elsif @azure_storage_connection_string.nil? || @azure_storage_connection_string.empty?
         | 
| 96 | 
            +
                      raise ConfigError, 'azure_storage_account needs to be specified' if @azure_storage_account.nil?
         | 
| 97 | 
            +
                      raise ConfigError, 'azure_container needs to be specified' if @azure_container.nil?
         | 
| 98 | 
            +
                    end
         | 
| 99 | 
            +
             | 
| 100 | 
            +
                    @blob_options = {}
         | 
| 101 | 
            +
             | 
| 102 | 
            +
                    if !@compute_checksums
         | 
| 103 | 
            +
                      @blob_options[:content_md5] = ''
         | 
| 73 104 | 
             
                    end
         | 
| 74 105 | 
             
                  end
         | 
| 75 106 |  | 
| @@ -81,13 +112,13 @@ module Fluent | |
| 81 112 | 
             
                    access_key_request = Faraday.new('http://169.254.169.254/metadata/identity/oauth2/token?' \
         | 
| 82 113 | 
             
                                                     "api-version=#{@azure_imds_api_version}" \
         | 
| 83 114 | 
             
                                                     '&resource=https://storage.azure.com/' \
         | 
| 84 | 
            -
                                                     "#{! | 
| 115 | 
            +
                                                     "#{!azure_msi_client_id.nil? ? "&client_id=#{azure_msi_client_id}" : ''}",
         | 
| 85 116 | 
             
                                                     headers: { 'Metadata' => 'true' }).get
         | 
| 86 117 |  | 
| 87 118 | 
             
                    if access_key_request.status == 200
         | 
| 88 119 | 
             
                      JSON.parse(access_key_request.body)['access_token']
         | 
| 89 120 | 
             
                    else
         | 
| 90 | 
            -
                      raise  | 
| 121 | 
            +
                      raise 'Access token request was not sucssessful. Possibly due to missing azure_msi_client_id config parameter.'
         | 
| 91 122 | 
             
                    end
         | 
| 92 123 | 
             
                  end
         | 
| 93 124 |  | 
| @@ -96,7 +127,11 @@ module Fluent | |
| 96 127 | 
             
                    if @use_msi
         | 
| 97 128 | 
             
                      token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
         | 
| 98 129 | 
             
                      token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
         | 
| 99 | 
            -
                      @bs = Azure::Storage::Blob::BlobService.new( | 
| 130 | 
            +
                      @bs = Azure::Storage::Blob::BlobService.new(
         | 
| 131 | 
            +
                        storage_account_name: @azure_storage_account,
         | 
| 132 | 
            +
                        storage_dns_suffix: @azure_storage_dns_suffix,
         | 
| 133 | 
            +
                        signer: token_signer
         | 
| 134 | 
            +
                      )
         | 
| 100 135 |  | 
| 101 136 | 
             
                      refresh_interval = @azure_token_refresh_interval * 60
         | 
| 102 137 | 
             
                      cancelled = false
         | 
| @@ -113,7 +148,7 @@ module Fluent | |
| 113 148 | 
             
                    elsif !@azure_storage_connection_string.nil? && !@azure_storage_connection_string.empty?
         | 
| 114 149 | 
             
                      @bs = Azure::Storage::Blob::BlobService.create_from_connection_string(@azure_storage_connection_string)
         | 
| 115 150 | 
             
                    else
         | 
| 116 | 
            -
                      @bs_params = { storage_account_name: @azure_storage_account }
         | 
| 151 | 
            +
                      @bs_params = { storage_account_name: @azure_storage_account, storage_dns_suffix: @azure_storage_dns_suffix }
         | 
| 117 152 |  | 
| 118 153 | 
             
                      if !@azure_storage_access_key.nil? && !@azure_storage_access_key.empty?
         | 
| 119 154 | 
             
                        @bs_params.merge!({ storage_access_key: @azure_storage_access_key })
         | 
| @@ -137,23 +172,26 @@ module Fluent | |
| 137 172 | 
             
                  end
         | 
| 138 173 |  | 
| 139 174 | 
             
                  def write(chunk)
         | 
| 140 | 
            -
                    metadata = chunk.metadata
         | 
| 141 175 | 
             
                    tmp = Tempfile.new('azure-')
         | 
| 142 176 | 
             
                    begin
         | 
| 143 177 | 
             
                      chunk.write_to(tmp)
         | 
| 144 178 |  | 
| 145 | 
            -
                      generate_log_name( | 
| 179 | 
            +
                      generate_log_name(chunk, @current_index)
         | 
| 146 180 | 
             
                      if @last_azure_storage_path != @azure_storage_path
         | 
| 147 181 | 
             
                        @current_index = 0
         | 
| 148 | 
            -
                        generate_log_name( | 
| 182 | 
            +
                        generate_log_name(chunk, @current_index)
         | 
| 149 183 | 
             
                      end
         | 
| 150 184 |  | 
| 151 185 | 
             
                      content = File.open(tmp.path, 'rb', &:read)
         | 
| 152 186 |  | 
| 153 | 
            -
                      append_blob(content,  | 
| 187 | 
            +
                      append_blob(content, chunk)
         | 
| 154 188 | 
             
                      @last_azure_storage_path = @azure_storage_path
         | 
| 155 189 | 
             
                    ensure
         | 
| 156 | 
            -
                       | 
| 190 | 
            +
                      begin
         | 
| 191 | 
            +
                        tmp.close(true)
         | 
| 192 | 
            +
                      rescue StandardError
         | 
| 193 | 
            +
                        nil
         | 
| 194 | 
            +
                      end
         | 
| 157 195 | 
             
                    end
         | 
| 158 196 | 
             
                  end
         | 
| 159 197 |  | 
| @@ -182,7 +220,8 @@ module Fluent | |
| 182 220 | 
             
                    end
         | 
| 183 221 | 
             
                  end
         | 
| 184 222 |  | 
| 185 | 
            -
                  def generate_log_name( | 
| 223 | 
            +
                  def generate_log_name(chunk, index)
         | 
| 224 | 
            +
                    metadata = chunk.metadata
         | 
| 186 225 | 
             
                    time_slice = if metadata.timekey.nil?
         | 
| 187 226 | 
             
                                   ''.freeze
         | 
| 188 227 | 
             
                                 else
         | 
| @@ -196,41 +235,39 @@ module Fluent | |
| 196 235 | 
             
                      '%{index}' => index
         | 
| 197 236 | 
             
                    }
         | 
| 198 237 | 
             
                    storage_path = @azure_object_key_format.gsub(/%{[^}]+}/, values_for_object_key)
         | 
| 199 | 
            -
                    @azure_storage_path = extract_placeholders(storage_path,  | 
| 238 | 
            +
                    @azure_storage_path = extract_placeholders(storage_path, chunk)
         | 
| 200 239 | 
             
                  end
         | 
| 201 240 |  | 
| 202 | 
            -
                  def append_blob(content,  | 
| 241 | 
            +
                  def append_blob(content, chunk)
         | 
| 203 242 | 
             
                    position = 0
         | 
| 204 243 | 
             
                    log.debug "azure_storage_append_blob: append_blob.start: Content size: #{content.length}"
         | 
| 205 244 | 
             
                    loop do
         | 
| 206 | 
            -
                       | 
| 207 | 
            -
             | 
| 208 | 
            -
             | 
| 209 | 
            -
             | 
| 210 | 
            -
             | 
| 211 | 
            -
             | 
| 212 | 
            -
                       | 
| 213 | 
            -
             | 
| 214 | 
            -
             | 
| 215 | 
            -
                         | 
| 216 | 
            -
             | 
| 217 | 
            -
             | 
| 218 | 
            -
             | 
| 219 | 
            -
             | 
| 220 | 
            -
             | 
| 221 | 
            -
                           | 
| 222 | 
            -
                            log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
         | 
| 223 | 
            -
                            raise
         | 
| 224 | 
            -
                          end
         | 
| 225 | 
            -
             | 
| 226 | 
            -
                          log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
         | 
| 227 | 
            -
                          @bs.create_append_blob(@azure_container, @azure_storage_path)
         | 
| 228 | 
            -
                        elsif status_code == 404 # blob not found
         | 
| 229 | 
            -
                          log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
         | 
| 230 | 
            -
                          @bs.create_append_blob(@azure_container, @azure_storage_path)
         | 
| 231 | 
            -
                        else
         | 
| 245 | 
            +
                      size = [content.length - position, AZURE_BLOCK_SIZE_LIMIT].min
         | 
| 246 | 
            +
                      log.debug "azure_storage_append_blob: append_blob.chunk: content[#{position}..#{position + size}]"
         | 
| 247 | 
            +
                      @bs.append_blob_block(@azure_container, @azure_storage_path, content[position..position + size - 1], options=@blob_options)
         | 
| 248 | 
            +
                      position += size
         | 
| 249 | 
            +
                      break if position >= content.length
         | 
| 250 | 
            +
                    rescue Azure::Core::Http::HTTPError => e
         | 
| 251 | 
            +
                      status_code = e.status_code
         | 
| 252 | 
            +
             | 
| 253 | 
            +
                      if status_code == 409 # exceeds azure block limit
         | 
| 254 | 
            +
                        @current_index += 1
         | 
| 255 | 
            +
                        old_azure_storage_path = @azure_storage_path
         | 
| 256 | 
            +
                        generate_log_name(chunk, @current_index)
         | 
| 257 | 
            +
             | 
| 258 | 
            +
                        # If index is not a part of format, rethrow exception.
         | 
| 259 | 
            +
                        if old_azure_storage_path == @azure_storage_path
         | 
| 260 | 
            +
                          log.warn 'azure_storage_append_blob: append_blob: blocks limit reached, you need to use %{index} for the format.'
         | 
| 232 261 | 
             
                          raise
         | 
| 233 262 | 
             
                        end
         | 
| 263 | 
            +
             | 
| 264 | 
            +
                        log.debug "azure_storage_append_blob: append_blob: blocks limit reached, creating new blob #{@azure_storage_path}."
         | 
| 265 | 
            +
                        @bs.create_append_blob(@azure_container, @azure_storage_path)
         | 
| 266 | 
            +
                      elsif status_code == 404 # blob not found
         | 
| 267 | 
            +
                        log.debug "azure_storage_append_blob: append_blob: #{@azure_storage_path} blob doesn't exist, creating new blob."
         | 
| 268 | 
            +
                        @bs.create_append_blob(@azure_container, @azure_storage_path)
         | 
| 269 | 
            +
                      else
         | 
| 270 | 
            +
                        raise
         | 
| 234 271 | 
             
                      end
         | 
| 235 272 | 
             
                    end
         | 
| 236 273 | 
             
                    log.debug 'azure_storage_append_blob: append_blob.complete'
         | 
| @@ -11,6 +11,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 11 11 | 
             
              end
         | 
| 12 12 |  | 
| 13 13 | 
             
              CONFIG = %(
         | 
| 14 | 
            +
                azure_cloud AZUREGERMANCLOUD
         | 
| 14 15 | 
             
                azure_storage_account test_storage_account
         | 
| 15 16 | 
             
                azure_storage_access_key MY_FAKE_SECRET
         | 
| 16 17 | 
             
                azure_container test_container
         | 
| @@ -18,6 +19,12 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 18 19 | 
             
                path log
         | 
| 19 20 | 
             
              ).freeze
         | 
| 20 21 |  | 
| 22 | 
            +
              CONNSTR_CONFIG = %(
         | 
| 23 | 
            +
                azure_storage_connection_string https://test
         | 
| 24 | 
            +
                time_slice_format        %Y%m%d-%H
         | 
| 25 | 
            +
                path log
         | 
| 26 | 
            +
              ).freeze
         | 
| 27 | 
            +
             | 
| 21 28 | 
             
              MSI_CONFIG = %(
         | 
| 22 29 | 
             
                azure_storage_account test_storage_account
         | 
| 23 30 | 
             
                azure_container test_container
         | 
| @@ -27,6 +34,16 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 27 34 | 
             
                path log
         | 
| 28 35 | 
             
              ).freeze
         | 
| 29 36 |  | 
| 37 | 
            +
              AZURESTACKCLOUD_CONFIG = %(
         | 
| 38 | 
            +
                azure_cloud AZURESTACKCLOUD
         | 
| 39 | 
            +
                azure_storage_dns_suffix test.storage.dns.suffix
         | 
| 40 | 
            +
                azure_storage_account test_storage_account
         | 
| 41 | 
            +
                azure_storage_access_key MY_FAKE_SECRET
         | 
| 42 | 
            +
                azure_container test_container
         | 
| 43 | 
            +
                time_slice_format        %Y%m%d-%H
         | 
| 44 | 
            +
                path log
         | 
| 45 | 
            +
              ).freeze
         | 
| 46 | 
            +
             | 
| 30 47 | 
             
              def create_driver(conf: CONFIG, service: nil)
         | 
| 31 48 | 
             
                d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
         | 
| 32 49 | 
             
                d.instance.instance_variable_set(:@bs, service)
         | 
| @@ -36,7 +53,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 36 53 |  | 
| 37 54 | 
             
              sub_test_case 'test config' do
         | 
| 38 55 | 
             
                test 'config should reject with no azure container' do
         | 
| 39 | 
            -
                  assert_raise Fluent::ConfigError do
         | 
| 56 | 
            +
                  assert_raise Fluent::ConfigError.new('azure_container needs to be specified') do
         | 
| 40 57 | 
             
                    create_driver conf: %(
         | 
| 41 58 | 
             
                      azure_storage_account test_storage_account
         | 
| 42 59 | 
             
                      azure_storage_access_key MY_FAKE_SECRET
         | 
| @@ -47,8 +64,25 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 47 64 | 
             
                  end
         | 
| 48 65 | 
             
                end
         | 
| 49 66 |  | 
| 67 | 
            +
                test 'config should reject for invalid cloud ' do
         | 
| 68 | 
            +
                  assert_raise Fluent::ConfigError.new('azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD') do
         | 
| 69 | 
            +
                    create_driver conf: %(
         | 
| 70 | 
            +
                      azure_cloud INVALIDCLOUD
         | 
| 71 | 
            +
                    )
         | 
| 72 | 
            +
                  end
         | 
| 73 | 
            +
                end
         | 
| 74 | 
            +
             | 
| 75 | 
            +
                test 'config should reject for Azure Stack Cloud with no azure storage dns suffix' do
         | 
| 76 | 
            +
                  assert_raise Fluent::ConfigError.new('azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD') do
         | 
| 77 | 
            +
                    create_driver conf: %(
         | 
| 78 | 
            +
                      azure_cloud AZURESTACKCLOUD
         | 
| 79 | 
            +
                    )
         | 
| 80 | 
            +
                  end
         | 
| 81 | 
            +
                end
         | 
| 82 | 
            +
             | 
| 50 83 | 
             
                test 'config with access key should set instance variables' do
         | 
| 51 84 | 
             
                  d = create_driver
         | 
| 85 | 
            +
                  assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
         | 
| 52 86 | 
             
                  assert_equal 'test_storage_account', d.instance.azure_storage_account
         | 
| 53 87 | 
             
                  assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
         | 
| 54 88 | 
             
                  assert_equal 'test_container', d.instance.azure_container
         | 
| @@ -60,12 +94,29 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 60 94 | 
             
                  d = create_driver conf: MSI_CONFIG
         | 
| 61 95 | 
             
                  assert_equal 'test_storage_account', d.instance.azure_storage_account
         | 
| 62 96 | 
             
                  assert_equal 'test_container', d.instance.azure_container
         | 
| 63 | 
            -
                  assert_equal true, d.instance.use_msi
         | 
| 97 | 
            +
                  assert_equal true, d.instance.instance_variable_get(:@use_msi)
         | 
| 64 98 | 
             
                  assert_equal true, d.instance.auto_create_container
         | 
| 65 99 | 
             
                  assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
         | 
| 66 100 | 
             
                  assert_equal 120, d.instance.azure_token_refresh_interval
         | 
| 67 101 | 
             
                  assert_equal '1970-01-01', d.instance.azure_imds_api_version
         | 
| 68 102 | 
             
                end
         | 
| 103 | 
            +
             | 
| 104 | 
            +
                test 'config with connection string should set instance variables' do
         | 
| 105 | 
            +
                  d = create_driver conf: CONNSTR_CONFIG
         | 
| 106 | 
            +
                  assert_equal 'https://test', d.instance.azure_storage_connection_string
         | 
| 107 | 
            +
                  assert_equal false, d.instance.instance_variable_get(:@use_msi)
         | 
| 108 | 
            +
                  assert_equal true, d.instance.auto_create_container
         | 
| 109 | 
            +
                end
         | 
| 110 | 
            +
              
         | 
| 111 | 
            +
                test 'config for Azure Stack Cloud should set instance variables' do
         | 
| 112 | 
            +
                  d = create_driver conf: AZURESTACKCLOUD_CONFIG
         | 
| 113 | 
            +
                  assert_equal 'test.storage.dns.suffix', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
         | 
| 114 | 
            +
                  assert_equal 'test_storage_account', d.instance.azure_storage_account
         | 
| 115 | 
            +
                  assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
         | 
| 116 | 
            +
                  assert_equal 'test_container', d.instance.azure_container
         | 
| 117 | 
            +
                  assert_equal true, d.instance.auto_create_container
         | 
| 118 | 
            +
                  assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
         | 
| 119 | 
            +
                end
         | 
| 69 120 | 
             
              end
         | 
| 70 121 |  | 
| 71 122 | 
             
              sub_test_case 'test path slicing' do
         | 
| @@ -92,15 +143,13 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 92 143 | 
             
              # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
         | 
| 93 144 | 
             
              # a response object when it is created.
         | 
| 94 145 | 
             
              class FakeResponse
         | 
| 95 | 
            -
                def initialize(status=404)
         | 
| 146 | 
            +
                def initialize(status = 404)
         | 
| 96 147 | 
             
                  @status = status
         | 
| 97 | 
            -
                  @body =  | 
| 148 | 
            +
                  @body = 'body'
         | 
| 98 149 | 
             
                  @headers = {}
         | 
| 99 150 | 
             
                end
         | 
| 100 151 |  | 
| 101 | 
            -
                attr_reader :status
         | 
| 102 | 
            -
                attr_reader :body
         | 
| 103 | 
            -
                attr_reader :headers
         | 
| 152 | 
            +
                attr_reader :status, :body, :headers
         | 
| 104 153 | 
             
              end
         | 
| 105 154 |  | 
| 106 155 | 
             
              # This class is used to test plugin functions which interact with the blob service
         | 
| @@ -111,11 +160,11 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 111 160 | 
             
                end
         | 
| 112 161 | 
             
                attr_reader :blocks
         | 
| 113 162 |  | 
| 114 | 
            -
                def append_blob_block( | 
| 163 | 
            +
                def append_blob_block(_container, _path, data, options={})
         | 
| 115 164 | 
             
                  @blocks.append(data)
         | 
| 116 165 | 
             
                end
         | 
| 117 166 |  | 
| 118 | 
            -
                def get_container_properties( | 
| 167 | 
            +
                def get_container_properties(_container)
         | 
| 119 168 | 
             
                  unless @response.status_code == 200
         | 
| 120 169 | 
             
                    raise Azure::Core::Http::HTTPError.new(@response)
         | 
| 121 170 | 
             
                  end
         | 
| @@ -125,18 +174,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 125 174 | 
             
              sub_test_case 'test container_exists' do
         | 
| 126 175 | 
             
                test 'container 404 returns false' do
         | 
| 127 176 | 
             
                  d = create_driver service: FakeBlobService.new(404)
         | 
| 128 | 
            -
                  assert_false d.instance.container_exists?  | 
| 177 | 
            +
                  assert_false d.instance.container_exists? 'anything'
         | 
| 129 178 | 
             
                end
         | 
| 130 179 |  | 
| 131 180 | 
             
                test 'existing container returns true' do
         | 
| 132 181 | 
             
                  d = create_driver service: FakeBlobService.new(200)
         | 
| 133 | 
            -
                  assert_true d.instance.container_exists?  | 
| 182 | 
            +
                  assert_true d.instance.container_exists? 'anything'
         | 
| 134 183 | 
             
                end
         | 
| 135 184 |  | 
| 136 185 | 
             
                test 'unexpected exception raises' do
         | 
| 137 186 | 
             
                  d = create_driver service: FakeBlobService.new(500)
         | 
| 138 187 | 
             
                  assert_raise_kind_of Azure::Core::Http::HTTPError do
         | 
| 139 | 
            -
                    d.instance.container_exists?  | 
| 188 | 
            +
                    d.instance.container_exists? 'anything'
         | 
| 140 189 | 
             
                  end
         | 
| 141 190 | 
             
                end
         | 
| 142 191 | 
             
              end
         | 
| @@ -175,10 +224,10 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase | |
| 175 224 |  | 
| 176 225 | 
             
                test 'long buffer appends multiple times' do
         | 
| 177 226 | 
             
                  limit = Fluent::Plugin::AzureStorageAppendBlobOut::AZURE_BLOCK_SIZE_LIMIT
         | 
| 178 | 
            -
                   | 
| 179 | 
            -
                   | 
| 180 | 
            -
                  blocks = fake_appended_blocks  | 
| 181 | 
            -
                  assert_equal [ | 
| 227 | 
            +
                  buf1 = 'a' * limit
         | 
| 228 | 
            +
                  buf2 = 'a' * 3
         | 
| 229 | 
            +
                  blocks = fake_appended_blocks buf1 + buf2
         | 
| 230 | 
            +
                  assert_equal [buf1, buf2], blocks
         | 
| 182 231 | 
             
                end
         | 
| 183 232 | 
             
              end
         | 
| 184 233 | 
             
            end
         | 
    
        metadata
    CHANGED
    
    | @@ -1,14 +1,14 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: fluent-plugin-azure-storage-append-blob-lts
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0. | 
| 4 | 
            +
              version: 0.6.3
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Jonas-Taha El Sesiy
         | 
| 8 8 | 
             
            autorequire: 
         | 
| 9 9 | 
             
            bindir: bin
         | 
| 10 10 | 
             
            cert_chain: []
         | 
| 11 | 
            -
            date: 2021- | 
| 11 | 
            +
            date: 2021-04-03 00:00:00.000000000 Z
         | 
| 12 12 | 
             
            dependencies:
         | 
| 13 13 | 
             
            - !ruby/object:Gem::Dependency
         | 
| 14 14 | 
             
              name: bundler
         | 
| @@ -87,13 +87,14 @@ dependencies: | |
| 87 87 | 
             
                  - !ruby/object:Gem::Version
         | 
| 88 88 | 
             
                    version: '2'
         | 
| 89 89 | 
             
            description: Fluentd plugin to upload logs to Azure Storage append blobs. Fork of
         | 
| 90 | 
            -
              https://github.com/ | 
| 90 | 
            +
              https://github.com/microsoft/fluent-plugin-azure-storage-append-blob
         | 
| 91 91 | 
             
            email:
         | 
| 92 | 
            -
            -  | 
| 92 | 
            +
            - github@elsesiy.com
         | 
| 93 93 | 
             
            executables: []
         | 
| 94 94 | 
             
            extensions: []
         | 
| 95 95 | 
             
            extra_rdoc_files: []
         | 
| 96 96 | 
             
            files:
         | 
| 97 | 
            +
            - ".github/CODEOWNERS"
         | 
| 97 98 | 
             
            - ".github/workflows/publish.yaml"
         | 
| 98 99 | 
             
            - ".github/workflows/test.yaml"
         | 
| 99 100 | 
             
            - ".gitignore"
         |