fluent-plugin-azure-storage-append-blob-lts 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9e6de27ebbf624d51968e415d502111523d3fd670d2345558381b03895934d6
-  data.tar.gz: 80fbe42cee9d82adb02afbaf2cfed1fd92b0d4fbbbfb1e67b50e328795d90b23
+  metadata.gz: 4e900def94ba82e3cb4e29b5ce37beed5b3801b951e5f68b1a1bc1e5a84ebfc1
+  data.tar.gz: 64780e024de8bb93023c7c42c5aff1d26ae461f51a769aae3ef74697fb607264
 SHA512:
-  metadata.gz: d3c2e0296f88ae875f19043e322678aa9a7ea3a26e9bb41f9f4292991ff09a6a50022d03c1da48b36e0c724c2e9e83f70ef431e1b9a3fc628507774a8de32943
-  data.tar.gz: a86038ca1acc515dd05df9f86d014cef337c5cc8a70847a15dd94b8f86fe2233c5b0a49be988f26a3da86f298d1aa27465aab4fb904ddd8eeef9e16535291d99
+  metadata.gz: d7da2a55760e6221520198ccb54632892bfc3293d68bd78f0b088260f2d6ca2ff49b014caaff7427fa99de6731a52f642167be9df96e930323ffce7eb54d70e0
+  data.tar.gz: bc0cacd9e1bc4fb6a2e425a0556d8c9737fc2f35cd4530399bb3e95f62fa1b55804b118f4ec7550dadc549e29a3c7767eb2c61a3f7d84fca1bb24969084ea3dd

data/.github/workflows/publish.yaml

@@ -1,29 +1,28 @@
+---
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: "3.1"
+      - env:
+          RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+        name: Publish to RubyGems
+        run: |
+          mkdir -p $HOME/.gem
+          touch $HOME/.gem/credentials
+          chmod 0600 $HOME/.gem/credentials
+          printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
+          gem build *.gemspec
+          gem push *.gem --otp ${{ github.event.inputs.otp }}
 name: Publish Ruby Gem
-
 on:
   workflow_dispatch:
     inputs:
       otp:
-        description: 'One Time Password'
+        description: "One Time Password"
         required: true
-
-jobs:
-  build:
-    name: Build + Publish
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby 2.6
-      uses: actions/setup-ruby@v1
-      with:
-        ruby-version: 2.6.x
-    - name: Publish to RubyGems
-      run: |
-        mkdir -p $HOME/.gem
-        touch $HOME/.gem/credentials
-        chmod 0600 $HOME/.gem/credentials
-        printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
-        gem build *.gemspec
-        gem push *.gem --otp ${{ github.event.inputs.otp }}
-      env:
-        RUBYGEMS_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/.github/workflows/test.yaml

@@ -1,21 +1,20 @@
-name: Test
-
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
-
+---
 jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.6
-    - name: Install dependencies
-      run: bundle install
-    - name: Run tests
-      run: bundle exec rake test
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.1"
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test
+name: Test
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]

data/Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.7
+FROM ruby:3.1.4
 
 WORKDIR /plugin
 

data/README.md

@@ -42,6 +42,7 @@ And then execute:
       path                              logs/
       azure_object_key_format           %{path}%{time_slice}_%{index}.log
       time_slice_format                 %Y%m%d-%H
+      compress                          false
       compute_checksums                 true
       # if you want to use %{tag} or %Y/%m/%d/ like syntax in path / azure_blob_name_format,
       # need to specify tag for %{tag} and time for %Y/%m/%d in <buffer> argument.
@@ -147,6 +148,20 @@ The [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin
 
 Format of the time used in the file name. Default is '%Y%m%d'. Use '%Y%m%d%H' to split files hourly.
 
+### `compress`
+
+Default: `false`
+
+If `true`, compress (gzip) the file prior to uploading it.
+
+Note: If desired, set `.gz` suffix via `azure_object_key_format`.
+
+Example:
+
+```
+azure_object_key_format %{path}%{time_slice}-%{index}.log.gz
+```
+
 ### `compute_checksums`
 
 Default: `true`

data/fluent-plugin-azure-storage-append-blob-lts.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name = 'fluent-plugin-azure-storage-append-blob-lts'
-  spec.version = '0.6.3'
+  spec.version = '0.7.0'
   spec.authors = ['Jonas-Taha El Sesiy']
   spec.email = ['github@elsesiy.com']
 
@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables = files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files = test_files
   spec.require_paths = ['lib']
-  spec.required_ruby_version = '>= 2.6'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 13.0'

data/lib/fluent/plugin/out_azure-storage-append-blob.rb

@@ -8,6 +8,7 @@ require 'azure/storage/blob'
 require 'faraday'
 require 'fluent/plugin/output'
 require 'json'
+require 'zlib'
 
 module Fluent
   module Plugin
@@ -46,6 +47,7 @@ module Fluent
       config_param :azure_object_key_format, :string, default: '%{path}%{time_slice}-%{index}.log'
       config_param :auto_create_container, :bool, default: true
       config_param :compute_checksums, :bool, default: true
+      config_param :compress, :bool, default: false
       config_param :format, :string, default: DEFAULT_FORMAT_TYPE
       config_param :time_slice_format, :string, default: '%Y%m%d'
       config_param :localtime, :bool, default: false
@@ -124,7 +126,9 @@ module Fluent
 
       def start
         super
-        if @use_msi
+        if @bs
+          # nop
+        elsif @use_msi
           token_credential = Azure::Storage::Common::Core::TokenCredential.new get_access_token
           token_signer = Azure::Storage::Common::Core::Auth::TokenSigner.new token_credential
           @bs = Azure::Storage::Blob::BlobService.new(
@@ -171,10 +175,49 @@ module Fluent
         @formatter.format(tag, time, r)
       end
 
+      def write_compress(chunk, tmp)
+        tmp.binmode
+        write_compress_helper(chunk, tmp)
+        tmp.rewind
+      end
+
+      # ref: https://github.com/fluent/fluent-plugin-s3/blob/master/lib/fluent/plugin/s3_compressor_gzip_command.rb
+      def write_compress_helper(chunk, tmp)
+        chunk_is_file = @buffer_type == 'file'
+        path = if chunk_is_file
+                 chunk.path
+               else
+                 w = Tempfile.new("chunk-gzip-tmp")
+                 w.binmode
+                 chunk.write_to(w)
+                 w.close
+                 w.path
+               end
+
+        res = system "gzip -c #{path} > #{tmp.path}"
+        unless res
+          log.warn "failed to execute gzip command. Fallback to GzipWriter. status = #{$?}"
+          begin
+            tmp.truncate(0)
+            gw = Zlib::GzipWriter.new(tmp)
+            chunk.write_to(gw)
+            gw.close
+          ensure
+            gw.close rescue nil
+          end
+        end
+      ensure
+        w.close(true) rescue nil
+      end
+
       def write(chunk)
         tmp = Tempfile.new('azure-')
         begin
-          chunk.write_to(tmp)
+          if @compress
+            write_compress(chunk, tmp)
+          else
+            chunk.write_to(tmp)
+          end
 
           generate_log_name(chunk, @current_index)
           if @last_azure_storage_path != @azure_storage_path

data/test/plugin/test_out_azure_storage_append_blob.rb

@@ -2,6 +2,8 @@ require 'helper'
 require 'fluent/plugin/out_azure-storage-append-blob.rb'
 require 'azure/core/http/http_response'
 require 'azure/core/http/http_error'
+require 'stringio'
+require 'zlib'
 
 include Fluent::Test::Helpers
 
@@ -140,6 +142,18 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'compress options' do
+    test 'compress default value' do
+      d = create_driver
+      assert_equal false, d.instance.instance_variable_get(:@compress)
+    end
+    test 'compress set true' do
+      config = CONFIG.clone + "\ncompress true\n"
+      d = create_driver conf: config
+      assert_equal true, d.instance.instance_variable_get(:@compress)
+    end
+  end
+
   # This class is used to create an Azure::Core::Http::HTTPError. HTTPError parses
   # a response object when it is created.
   class FakeResponse
@@ -152,6 +166,70 @@ class AzureStorageAppendBlobOutTest < Test::Unit::TestCase
     attr_reader :status, :body, :headers
   end
 
+  def uncompress_blocks(blocks)
+      gzip_data = blocks.join
+      uncompressed_data = ""
+      while gzip_data do
+        Zlib::GzipReader.wrap(StringIO.new(gzip_data.b)) do |gz|
+          uncompressed_data << gz.read
+          gzip_data = gz.unused
+        end
+      end
+      uncompressed_data
+  end
+
+  sub_test_case 'test append blob for compress' do
+    test 'compress 2 events 1 chunk' do
+      config = CONFIG.clone + %(
+        compress true
+        <buffer time>
+          timekey      5
+          timekey_wait 0
+        </buffer>
+      )
+
+      svc = FakeBlobService.new(200)
+      d = create_driver conf:config, service: svc
+
+      d.run(default_tag: 'test') do
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 1 })
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 2 })
+      end
+
+      uncompressed_data = uncompress_blocks(svc.blocks)
+
+      expected =  "2011-01-02T13:14:15+00:00\ttest\t{\"a\":1}\n" +
+                  "2011-01-02T13:14:15+00:00\ttest\t{\"a\":2}\n"
+      assert_equal(expected, uncompressed_data)
+      assert_equal(10, svc.blocks.size)
+    end
+
+    test 'compress 2 events 2 chunk' do
+      config = CONFIG.clone + %(
+        compress true
+        <buffer time>
+          timekey      5
+          timekey_wait 0
+        </buffer>
+      )
+
+      svc = FakeBlobService.new(200)
+      d = create_driver conf:config, service: svc
+
+      d.run(default_tag: 'test') do
+        d.feed(event_time("2011-01-02 13:14:00 UTC"), { :a => 1 })
+        d.feed(event_time("2011-01-02 13:14:15 UTC"), { :a => 2 }) # after 15 sec
+      end
+
+      uncompressed_data = uncompress_blocks(svc.blocks)
+
+      expected = "2011-01-02T13:14:00+00:00\ttest\t{\"a\":1}\n" +
+                 "2011-01-02T13:14:15+00:00\ttest\t{\"a\":2}\n"
+      assert_equal(expected, uncompressed_data)
+      assert_equal(20, svc.blocks.size)
+    end
+  end
+
   # This class is used to test plugin functions which interact with the blob service
   class FakeBlobService
     def initialize(status)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-azure-storage-append-blob-lts
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Jonas-Taha El Sesiy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-03 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -120,14 +120,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Azure Storage Append Blob output plugin for Fluentd event collector