fluent-plugin-out-http 1.3.4 → 1.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc6b6a32252a6bd023ff50a7785ad1c761f0e0de88fd791cf80a03b4e03313c3
-  data.tar.gz: 35f04c05e359cc78d9a2305a8b157439c930254d23ad032c5a57496506d21a84
+  metadata.gz: d3d0fe7b452afa815821161d819bdf70c394e10068d43ad9d7962f9e59506585
+  data.tar.gz: e6e6b6b72bda444045c274cdf1b4edc5d61da372f330b2697b09d16134072a92
 SHA512:
-  metadata.gz: 14be9f2277907f3cf48a5388568b80a4a91f5daa6a0af8e0b24fd19510fdf0c0bad5c51dcc6338a2a61058907bffa9c5c95bf333a15d6a7ef5bc99a955c7e051
-  data.tar.gz: 27ae799e130b2469947cee7f603477fba5fdd80cffa5041929cb7bfaa4948ce656d90cd349ccdf8884c8616758f79b2263e158b00dc8831a926462e8ce5757b2
+  metadata.gz: 7e3e2600427b82ec559a99411423cd161e7fd99146cb2beffb1467fc0f997297f59d3cb352b29cae717b5fc012b1122edc80f9a29087d26e0c9400a56ce9a37e
+  data.tar.gz: 2dd79aa067059bcd8626bc0d65230283f850996ea861fd792c53e8ea92c47cdf12a11c4823bc81309fecf28241390354a7dff8ce46d348c1fcfbb48a43c5b8da

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'

data/.github/workflows/test.yml

@@ -8,14 +8,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1' ]
+        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4', '4.0' ]
         os:
           - ubuntu-latest
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
@@ -27,14 +27,14 @@ jobs:
     name: Gem test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: ruby/setup-ruby@v1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
         with:
           ruby-version: 3.1
       - name: Install
         run: |
           rake install
-          gem install test-unit webrick
+          gem install test-unit test-unit-rr webrick
       - name: Test
         run: |
           mkdir -p tmp

data/CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## 1.3.5
+* Mask authentication bearer token in logs
+* Avoid unnecessary gzip decompression and recompression
+
 ## 1.3.4
 * Add debug log about chunk writing
 * Tweak CI

data/fluent-plugin-out-http.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-out-http"
-  gem.version       = "1.3.4"
+  gem.version       = "1.3.5"
   gem.authors       = ["Marica Odagaki"]
   gem.email         = ["ento.entotto@gmail.com"]
   gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint}
@@ -22,5 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit", ">= 3.1.0"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0"
   gem.add_development_dependency "webrick"
 end

data/lib/fluent/plugin/out_http.rb

@@ -61,7 +61,7 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
   config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer],  :default => :none
   config_param :username, :string, :default => ''
   config_param :password, :string, :default => '', :secret => true
-  config_param :token, :string, :default => ''
+  config_param :token, :string, :default => '', :secret => true
   # Switch non-buffered/buffered plugin
   config_param :buffered, :bool, :default => false
   config_param :bulk_request, :bool, :default => false
@@ -146,12 +146,21 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
     end
   end
 
+  def buffer_compressed?
+    @buffered && @buffer.compress == :gzip
+  end
+
   def compress_body(req, data)
     return unless @compress_request
-    gz = Zlib::GzipWriter.new(StringIO.new)
-    gz << data
 
     req['Content-Encoding'] = "gzip"
+    if buffer_compressed?
+      req.body = data
+      return
+    end
+
+    gz = Zlib::GzipWriter.new(StringIO.new)
+    gz << data
     req.body = gz.close.string
   end
 
@@ -261,7 +270,8 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
   end
 
   def handle_records(tag, time, chunk)
-    req, uri = create_request(tag, time, chunk.read)
+    record = buffer_compressed? ? chunk.read(compressed: @buffer.compress) : chunk.read
+    req, uri = create_request(tag, time, record)
     send_request(req, uri)
   end
 

data/test/plugin/test_out_http.rb

@@ -8,6 +8,8 @@ require 'fluent/test/driver/output'
 require 'fluent/test/helpers'
 require_relative "./script/plugin/formatter_test"
 
+require 'test/unit/rr'
+
 module OS
   # ref. http://stackoverflow.com/questions/170956/how-can-i-find-which-operating-system-my-ruby-program-is-running-on
   def OS.windows?
@@ -100,9 +102,18 @@ class HTTPOutputTestBase < Test::Unit::TestCase
 
           expander = -> (req) {
             if req["Content-Encoding"] == "gzip"
-              StringIO.open(req.body, 'rb'){|sio|
-                Zlib::GzipReader.wrap(sio).read
-              }
+              sio = StringIO.new(req.body)
+              output = ""
+
+              until sio.eof?
+                gz = Zlib::GzipReader.new(sio)
+                output << gz.read
+                unused = gz.unused
+                if unused
+                  sio.pos -= unused.length
+                end
+              end
+              output
             else
               req.body
             end
@@ -523,6 +534,50 @@ class HTTPOutputTest < HTTPOutputTestBase
       assert_equal expected, record[:x_ndjson]
       assert_nil record[:auth]
     end
+
+    def test_emit_x_ndjson_with_compression_with_compressed_buffer
+      binary_string = "\xe3\x81\x82"
+      d = create_driver CONFIG_JSON + %[
+        buffered true
+        bulk_request true
+        compress_request true
+        <buffer>
+          @type memory
+          compress gzip
+        </buffer>
+      ]
+
+      # Should not recompress
+      dont_allow(Zlib::GzipWriter).new
+
+      d.run(default_tag: 'test.metrics') do
+        d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
+        d.feed({ 'field1' => 70, 'field2' => 30, 'field3' => 20, 'otherfield' => 2, 'binary' => binary_string })
+      end
+
+      assert_equal 1, @posts.size
+      record = @posts[0]
+
+      expected =[
+        {
+          "binary"     => "\u3042",
+          "field1"     => 50,
+          "field2"     => 20,
+          "field3"     => 10,
+          "otherfield" => 1
+        },
+        {
+          "binary"     => "\u3042",
+          "field1"     => 70,
+          "field2"     => 30,
+          "field3"     => 20,
+          "otherfield" => 2
+        }
+      ]
+
+      assert_equal expected, record[:x_ndjson]
+      assert_nil record[:auth]
+    end
   end
 
   def test_emit_form_put
@@ -859,6 +914,8 @@ class HTTPSOutputTest < HTTPOutputTestBase
     ssl_no_verify true
     cacert_file /tmp/ssl.cert
     ]
+    cert, _ = WEBrick::Utils.create_self_signed_cert(2048, [["CN", "localhost"]], "Generated Test Cert")
+    File.write('/tmp/ssl.cert', cert.to_pem)
     d = create_driver config
     d.run(default_tag: 'test.metrics') do
       d.feed({ 'field1' => 50 })

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-http
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.3.5
 platform: ruby
 authors:
 - Marica Odagaki
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-10-24 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
@@ -86,6 +85,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webrick
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +120,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - CHANGELOG.md
@@ -125,7 +139,6 @@ homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-out-http
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -140,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
-signing_key: 
+rubygems_version: 4.0.6
 specification_version: 4
 summary: A generic Fluentd output plugin to send logs to an HTTP endpoint
 test_files: