fluent-plugin-azure-storage-append-blob-lts 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24eb19ac68cd42369ff7f147be24635945aad7c003f4ed789bcb4189d5f9a96e
-  data.tar.gz: f293437b177cf4f7fc7a02694150f03f479c70749a3973d2ffc45a19377b7784
+  metadata.gz: 4e900def94ba82e3cb4e29b5ce37beed5b3801b951e5f68b1a1bc1e5a84ebfc1
+  data.tar.gz: 64780e024de8bb93023c7c42c5aff1d26ae461f51a769aae3ef74697fb607264
 SHA512:
-  metadata.gz: 7be9b8cf563a7c2cf5b10667a8d037ea5a17d84a78ab3a782a937f5c06e0f850804985fef3b5b6eff795e965c4eefa0d690ddd737485929fa4882df416e420c0
-  data.tar.gz: d4bb668e69155459e1b69ce954f9b8434a9dfa5e92dbf5e1324e9a1d50514c2e7b6b34e513f5fcb3621f5ba7b2b2dee5d7dea1cf37788ffe198869f5d7a3a0a4
+  metadata.gz: d7da2a55760e6221520198ccb54632892bfc3293d68bd78f0b088260f2d6ca2ff49b014caaff7427fa99de6731a52f642167be9df96e930323ffce7eb54d70e0
+  data.tar.gz: bc0cacd9e1bc4fb6a2e425a0556d8c9737fc2f35cd4530399bb3e95f62fa1b55804b118f4ec7550dadc549e29a3c7767eb2c61a3f7d84fca1bb24969084ea3dd

data/.github/workflows/publish.yaml

@@ -1,29 +1,28 @@
+---
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: "3.1"
+      - env:
+          RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem --otp ${{ github.event.inputs.otp }}
 name: Publish Ruby Gem
-
 on:
   workflow_dispatch:
     inputs:
       otp:
-        description: 'One Time Password'
+        description: "One Time Password"
         required: true
-
-jobs:
-  build:
-    name: Build + Publish
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.6.x
-    - name: Publish to RubyGems
-      run: |
-        mkdir -p $HOME/.gem
-        touch $HOME/.gem/credentials
-        chmod 0600 $HOME/.gem/credentials
-        printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
-        gem build *.gemspec
-        gem push *.gem --otp ${{ github.event.inputs.otp }}
-      env:
-        RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/test.yaml

@@ -1,21 +1,20 @@
-name: Test
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
+---
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.6
-    - name: Install dependencies
-      run: bundle install
-    - name: Run tests
-      run: bundle exec rake test
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.1"
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test
+name: Test
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.7
+FROM ruby:3.1.4
 
 WORKDIR /plugin
 

data/README.md

@@ -29,6 +29,7 @@ And then execute:
       type azure-storage-append-blob
 
       azure_cloud                       <azure cloud environment>
+      azure_storage_dns_suffix          <your azure storage dns suffix> # only used for Azure Stack Cloud
       azure_storage_account             <your azure storage account>
       azure_storage_access_key          <your azure storage access key> # leave empty to use MSI
       azure_storage_connection_string   <your azure storage connection string> # leave empty to use MSI
@@ -41,6 +42,7 @@ And then execute:
       path                              logs/
       azure_object_key_format           %{path}%{time_slice}_%{index}.log
       time_slice_format                 %Y%m%d-%H
+      compress                          false
       compute_checksums                 true
       # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
       # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
@@ -59,6 +61,12 @@ Default: `AZUREPUBLICCLOUD`
 
 Cloud environment used to determine the storage endpoint suffix to use, see [here](https://github.com/Azure/go-autorest/blob/master/autorest/azure/environments.go).
 
+Use `AZURESTACKCLOUD` for Azure Stack Cloud.
+
+### `azure_storage_dns_suffix` (Required only for Azure Stack Cloud)
+
+Your Azure Storage endpoint suffix. This can be retrieved from Azure Storage connection string, `EndpointSuffix` section.
+
 ### `azure_storage_account` (Required)
 
 Your Azure Storage Account Name. This can be retrieved from Azure Management portal.
@@ -72,11 +80,11 @@ If all are empty, the plugin will use the local Managed Identity endpoint to obt
 
 ### `azure_imds_api_version` (Optional, only for MSI)
 
-Default: `2019-08-15`
+Default: `2020-12-01`
 
 The Instance Metadata Service is used during the OAuth flow to obtain an access token. This API is versioned and specifying the version is mandatory.
 
-See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service#versioning) for more details.
+See [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/instance-metadata-service?tabs=windows#versioning) for more details.
 
 ### `azure_token_refresh_interval` (Optional, only for MSI)
 
@@ -140,6 +148,20 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
 
 Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
 
+### `compress`
+
+Default: `false`
+
+If `true`, compress (gzip) the file prior to uploading it.
+
+Note: If desired, set `.gz` suffix via `azure_object_key_format`.
+
+Example:
+
+```
+azure_object_key_format %{path}%{time_slice}-%{index}.log.gz
+```
+
 ### `compute_checksums`
 
 Default: `true`

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
-  spec.version = '0.6.2'
+  spec.version = '0.7.0'
   spec.authors = ['Jonas-Taha El Sesiy']
   spec.email = ['github@elsesiy.com']
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files = test_files
   spec.require_paths = ['lib']
-  spec.required_ruby_version = '>= 2.6'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 13.0'

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -8,6 +8,7 @@ require 'azure/storage/blob'
 require 'faraday'
 require 'fluent/plugin/output'
 require 'json'
+require 'zlib'
 
 module Fluent
   module Plugin
@@ -36,15 +37,17 @@ module Fluent
       config_param :azure_storage_account, :string, default: nil
       config_param :azure_storage_access_key, :string, default: nil, secret: true
       config_param :azure_storage_connection_string, :string, default: nil, secret: true
+      config_param :azure_storage_dns_suffix, :string, default: nil
       config_param :azure_storage_sas_token, :string, default: nil, secret: true
       config_param :azure_cloud, :string, default: 'AZUREPUBLICCLOUD'
       config_param :azure_msi_client_id, :string, default: nil
       config_param :azure_container, :string, default: nil
-      config_param :azure_imds_api_version, :string, default: '2019-08-15'
+      config_param :azure_imds_api_version, :string, default: '2020-12-01'
       config_param :azure_token_refresh_interval, :integer, default: 60
       config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
       config_param :auto_create_container, :bool, default: true
       config_param :compute_checksums, :bool, default: true
+      config_param :compress, :bool, default: false
       config_param :format, :string, default: DEFAULT_FORMAT_TYPE
       config_param :time_slice_format, :string, default: '%Y%m%d'
       config_param :localtime, :bool, default: false
@@ -75,9 +78,15 @@ module Fluent
                          end
                        end
 
-        @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
-        if @azure_storage_dns_suffix.nil?
-          raise ConfigError 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD'
+        if @azure_cloud == 'AZURESTACKCLOUD'
+          if @azure_storage_dns_suffix.nil?
+            raise ConfigError, 'azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD'
+          end
+        else
+          @azure_storage_dns_suffix = @storage_endpoint_mapping[@azure_cloud]
+          if @azure_storage_dns_suffix.nil?
+            raise ConfigError, 'azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD'
+          end
         end
 
         if (@azure_storage_access_key.nil? || @azure_storage_access_key.empty?) &&
@@ -117,7 +126,9 @@ module Fluent
 
       def start
         super
-        if @use_msi
+        if @bs
+          # nop
+        elsif @use_msi
           token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
           token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
           @bs = Azure::Storage::Blob::BlobService.new(
@@ -164,10 +175,49 @@ module Fluent
         @formatter.format(tag, time, r)
       end
 
+      def write_compress(chunk, tmp)
+        tmp.binmode
+        write_compress_helper(chunk, tmp)
+        tmp.rewind
+      end
+
+      # ref: https://github.com/fluent/fluent-plugin-s3/blob/master/lib/fluent/plugin/s3_compressor_gzip_command.rb
+      def write_compress_helper(chunk, tmp)
+        chunk_is_file = @buffer_type == 'file'
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-gzip-tmp")
+                 w.binmode
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+
+        res = system "gzip -c #{path} > #{tmp.path}"
+        unless res
+          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+          begin
+            tmp.truncate(0)
+            gw = Zlib::GzipWriter.new(tmp)
+            chunk.write_to(gw)
+            gw.close
+          ensure
+            gw.close rescue nil
+          end
+        end
+      ensure
+        w.close(true) rescue nil
+      end
+
       def write(chunk)
         tmp = Tempfile.new('azure-')
         begin
-          chunk.write_to(tmp)
+          if @compress
+            write_compress(chunk, tmp)
+          else
+            chunk.write_to(tmp)
+          end
 
           generate_log_name(chunk, @current_index)
           if @last_azure_storage_path != @azure_storage_path

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -2,6 +2,8 @@ require 'helper'
 require 'fluent/plugin/out_azure-storage-append-blob.rb'
 require 'azure/core/http/http_response'
 require 'azure/core/http/http_error'
+require 'stringio'
+require 'zlib'
 
 include Fluent::Test::Helpers
 
@@ -34,6 +36,16 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     path log
   ).freeze
 
+  AZURESTACKCLOUD_CONFIG = %(
+    azure_cloud AZURESTACKCLOUD
+    azure_storage_dns_suffix test.storage.dns.suffix
+    azure_storage_account test_storage_account
+    azure_storage_access_key MY_FAKE_SECRET
+    azure_container test_container
+    time_slice_format        %Y%m%d-%H
+    path log
+  ).freeze
+
   def create_driver(conf: CONFIG, service: nil)
     d = Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureStorageAppendBlobOut).configure(conf)
     d.instance.instance_variable_set(:@bs, service)
@@ -43,7 +55,7 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
 
   sub_test_case 'test config' do
     test 'config should reject with no azure container' do
-      assert_raise Fluent::ConfigError do
+      assert_raise Fluent::ConfigError.new('azure_container needs to be specified') do
         create_driver conf: %(
           azure_storage_account test_storage_account
           azure_storage_access_key MY_FAKE_SECRET
@@ -54,6 +66,22 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
       end
     end
 
+    test 'config should reject for invalid cloud ' do
+      assert_raise Fluent::ConfigError.new('azure_cloud invalid, must be either of AZURECHINACLOUD, AZUREGERMANCLOUD, AZUREPUBLICCLOUD, AZUREUSGOVERNMENTCLOUD, AZURESTACKCLOUD') do
+        create_driver conf: %(
+          azure_cloud INVALIDCLOUD
+        )
+      end
+    end
+
+    test 'config should reject for Azure Stack Cloud with no azure storage dns suffix' do
+      assert_raise Fluent::ConfigError.new('azure_storage_dns_suffix invalid, must not be empty for AZURESTACKCLOUD') do
+        create_driver conf: %(
+          azure_cloud AZURESTACKCLOUD
+        )
+      end
+    end
+
     test 'config with access key should set instance variables' do
       d = create_driver
       assert_equal 'core.cloudapi.de', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
@@ -81,6 +109,16 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
       assert_equal false, d.instance.instance_variable_get(:@use_msi)
       assert_equal true, d.instance.auto_create_container
     end
+  
+    test 'config for Azure Stack Cloud should set instance variables' do
+      d = create_driver conf: AZURESTACKCLOUD_CONFIG
+      assert_equal 'test.storage.dns.suffix', d.instance.instance_variable_get(:@azure_storage_dns_suffix)
+      assert_equal 'test_storage_account', d.instance.azure_storage_account
+      assert_equal 'MY_FAKE_SECRET', d.instance.azure_storage_access_key
+      assert_equal 'test_container', d.instance.azure_container
+      assert_equal true, d.instance.auto_create_container
+      assert_equal '%{path}%{time_slice}-%{index}.log', d.instance.azure_object_key_format
+    end
   end
 
   sub_test_case 'test path slicing' do
@@ -104,6 +142,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'compress options' do
+    test 'compress default value' do
+      d = create_driver
+      assert_equal false, d.instance.instance_variable_get(:@compress)
+    end
+    test 'compress set true' do
+      config = CONFIG.clone + "\ncompress true\n"
+      d = create_driver conf: config
+      assert_equal true, d.instance.instance_variable_get(:@compress)
+    end
+  end
+
   # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
   # a response object when it is created.
   class FakeResponse
@@ -116,6 +166,70 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     attr_reader :status, :body, :headers
   end
 
+  def uncompress_blocks(blocks)
+      gzip_data = blocks.join
+      uncompressed_data = ""
+      while gzip_data do
+        Zlib::GzipReader.wrap(StringIO.new(gzip_data.b)) do |gz|
+          uncompressed_data << gz.read
+          gzip_data = gz.unused
+        end
+      end
+      uncompressed_data
+  end
+
+  sub_test_case 'test append blob for compress' do
+    test 'compress 2 events 1 chunk' do
+      config = CONFIG.clone + %(
+        compress true
+        <buffer time>
+          timekey      5
+          timekey_wait 0
+        </buffer>
+      )
+
+      svc = FakeBlobService.new(200)
+      d = create_driver conf:config, service: svc
+
+      d.run(default_tag: 'test') do
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 1 })
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 2 })
+      end
+
+      uncompressed_data = uncompress_blocks(svc.blocks)
+
+      expected =  "2011-01-02T13:14:15+00:00\ttest\t{\"a\":1}\n" +
+                  "2011-01-02T13:14:15+00:00\ttest\t{\"a\":2}\n"
+      assert_equal(expected, uncompressed_data)
+      assert_equal(10, svc.blocks.size)
+    end
+
+    test 'compress 2 events 2 chunk' do
+      config = CONFIG.clone + %(
+        compress true
+        <buffer time>
+          timekey      5
+          timekey_wait 0
+        </buffer>
+      )
+
+      svc = FakeBlobService.new(200)
+      d = create_driver conf:config, service: svc
+
+      d.run(default_tag: 'test') do
+        d.feed(event_time("2011-01-02 13:14:00 UTC"), { :a => 1 })
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 2 }) # after 15 sec
+      end
+
+      uncompressed_data = uncompress_blocks(svc.blocks)
+
+      expected = "2011-01-02T13:14:00+00:00\ttest\t{\"a\":1}\n" +
+                 "2011-01-02T13:14:15+00:00\ttest\t{\"a\":2}\n"
+      assert_equal(expected, uncompressed_data)
+      assert_equal(20, svc.blocks.size)
+    end
+  end
+
   # This class is used to test plugin functions which interact with the blob service
   class FakeBlobService
     def initialize(status)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-storage-append-blob-lts
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Jonas-Taha El Sesiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-04 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -120,14 +120,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Azure Storage Append Blob output plugin for Fluentd event collector