fluent-plugin-out-http 1.3.3 → 1.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6d4a3bd96989a1de8d1d71e651bcb22fb69c679c29901f8f3f38f9f76ff659fd
-  data.tar.gz: c50fda8ab077e2c14f8bcfed38b083d732a279ac1356cd0e2db106d279bcbe47
+  metadata.gz: d3d0fe7b452afa815821161d819bdf70c394e10068d43ad9d7962f9e59506585
+  data.tar.gz: e6e6b6b72bda444045c274cdf1b4edc5d61da372f330b2697b09d16134072a92
 SHA512:
-  metadata.gz: 69257fe22d883dda33db126e5bb866697277544f3f3142f741c736ab0c57698e280db6a60035dd3c8468adf22090fb7d1bdf14279522b8e4477df4eb47d6a708
-  data.tar.gz: 89aeda125386401d33c02f6b144563c14a737b762a56e79ea60bb08dd7ba7a3009012b12bbd965cf8de240ca8932838939750dacafacda4801a3a6d16f933310
+  metadata.gz: 7e3e2600427b82ec559a99411423cd161e7fd99146cb2beffb1467fc0f997297f59d3cb352b29cae717b5fc012b1122edc80f9a29087d26e0c9400a56ce9a37e
+  data.tar.gz: 2dd79aa067059bcd8626bc0d65230283f850996ea861fd792c53e8ea92c47cdf12a11c4823bc81309fecf28241390354a7dff8ce46d348c1fcfbb48a43c5b8da

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'monthly'

data/.github/workflows/test.yml

@@ -0,0 +1,43 @@
+name: Test
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.7', '3.0', '3.1', '3.2', '3.3', '3.4', '4.0' ]
+        os:
+          - ubuntu-latest
+          - macOS-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: unit testing
+        run: |
+          bundle exec rake test
+
+  gem:
+    name: Gem test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+      - uses: ruby/setup-ruby@6ca151fd1bfcfd6fe0c4eb6837eb0584d0134a0c # v1.290.0
+        with:
+          ruby-version: 3.1
+      - name: Install
+        run: |
+          rake install
+          gem install test-unit test-unit-rr webrick
+      - name: Test
+        run: |
+          mkdir -p tmp
+          cd tmp
+          cp -a ../test/ ./
+          ../run_test.rb

data/CHANGELOG.md

@@ -1,4 +1,12 @@
 # Changelog
+## 1.3.5
+* Mask authentication bearer token in logs
+* Avoid unnecessary gzip decompression and recompression
+
+## 1.3.4
+* Add debug log about chunk writing
+* Tweak CI
+
 ## 1.3.3
 * Revert x-ndjson format payload behavior
 

data/README.md

@@ -2,8 +2,38 @@
 
 A generic [fluentd][1] output plugin for sending logs to an HTTP endpoint.
 
+**NOTE:** This plugin name is overlapped from Fluentd core plugin which name is `out_http`. But core plugin does not provide buffered/non-buffered switch and batch/non-batch request switch. If you want to use these switch features. Please consider to use this plugin.
+
 [![Build Status](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http.svg?branch=master)](https://travis-ci.org/fluent-plugins-nursery/fluent-plugin-out-http)
 
+## Installation
+
+### For bundler
+
+Write the following line in Gemfile:
+
+```gemfile
+gem "fluent-plugin-out-http"
+```
+
+and then,
+
+```console
+$ bundle install
+```
+
+### for fluentd
+
+```console
+$ fluent-gem install fluent-plugin-out-http
+```
+
+### For td-agent
+
+```console
+$ td-agent-gem install fluent-plugin-out-http
+```
+
 ## Configuration options
 
     <match *>
@@ -45,3 +75,11 @@ Heavily based on [fluent-plugin-growthforecast][2]
   [1]: http://fluentd.org/
   [2]: https://github.com/tagomoris/fluent-plugin-growthforecast
   [3]: https://github.com/sabottenda/fluent-plugin-bufferize
+
+## How to release
+
+1. Update `gem.version` of the `gemspec` file.
+1. Update `CHANGELOG.md`.
+1. Commit them.
+1. `$ bundle exec rake release`
+    * It will add and push the tag, and publish the gem.

data/fluent-plugin-out-http.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-out-http"
-  gem.version       = "1.3.3"
+  gem.version       = "1.3.5"
   gem.authors       = ["Marica Odagaki"]
   gem.email         = ["ento.entotto@gmail.com"]
   gem.summary       = %q{A generic Fluentd output plugin to send logs to an HTTP endpoint}
@@ -22,4 +22,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency "bundler"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "test-unit", ">= 3.1.0"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0"
+  gem.add_development_dependency "webrick"
 end

data/lib/fluent/plugin/out_http.rb

@@ -61,7 +61,7 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
   config_param :authentication, :enum, list: [:none, :basic, :jwt, :bearer],  :default => :none
   config_param :username, :string, :default => ''
   config_param :password, :string, :default => '', :secret => true
-  config_param :token, :string, :default => ''
+  config_param :token, :string, :default => '', :secret => true
   # Switch non-buffered/buffered plugin
   config_param :buffered, :bool, :default => false
   config_param :bulk_request, :bool, :default => false
@@ -146,12 +146,21 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
     end
   end
 
+  def buffer_compressed?
+    @buffered && @buffer.compress == :gzip
+  end
+
   def compress_body(req, data)
     return unless @compress_request
-    gz = Zlib::GzipWriter.new(StringIO.new)
-    gz << data
 
     req['Content-Encoding'] = "gzip"
+    if buffer_compressed?
+      req.body = data
+      return
+    end
+
+    gz = Zlib::GzipWriter.new(StringIO.new)
+    gz << data
     req.body = gz.close.string
   end
 
@@ -261,7 +270,8 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
   end
 
   def handle_records(tag, time, chunk)
-    req, uri = create_request(tag, time, chunk.read)
+    record = buffer_compressed? ? chunk.read(compressed: @buffer.compress) : chunk.read
+    req, uri = create_request(tag, time, record)
     send_request(req, uri)
   end
 
@@ -302,6 +312,9 @@ class Fluent::Plugin::HTTPOutput < Fluent::Plugin::Output
   def write(chunk)
     tag = chunk.metadata.tag
     @endpoint_url = extract_placeholders(@endpoint_url, chunk)
+
+    log.debug { "#{@http_method.capitalize} data to #{@endpoint_url} with chunk(#{dump_unique_id_hex(chunk.unique_id)})" }
+
     if @bulk_request
       time = Fluent::Engine.now
       handle_records(tag, time, chunk)

data/run_test.rb

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+Dir.glob("test/**/test_*.rb") do |test_rb|
+  require File.expand_path(test_rb)
+end

data/test/plugin/test_out_http.rb

@@ -8,6 +8,8 @@ require 'fluent/test/driver/output'
 require 'fluent/test/helpers'
 require_relative "./script/plugin/formatter_test"
 
+require 'test/unit/rr'
+
 module OS
   # ref. http://stackoverflow.com/questions/170956/how-can-i-find-which-operating-system-my-ruby-program-is-running-on
   def OS.windows?
@@ -100,9 +102,18 @@ class HTTPOutputTestBase < Test::Unit::TestCase
 
           expander = -> (req) {
             if req["Content-Encoding"] == "gzip"
-              StringIO.open(req.body, 'rb'){|sio|
-                Zlib::GzipReader.wrap(sio).read
-              }
+              sio = StringIO.new(req.body)
+              output = ""
+
+              until sio.eof?
+                gz = Zlib::GzipReader.new(sio)
+                output << gz.read
+                unused = gz.unused
+                if unused
+                  sio.pos -= unused.length
+                end
+              end
+              output
             else
               req.body
             end
@@ -523,6 +534,50 @@ class HTTPOutputTest < HTTPOutputTestBase
       assert_equal expected, record[:x_ndjson]
       assert_nil record[:auth]
     end
+
+    def test_emit_x_ndjson_with_compression_with_compressed_buffer
+      binary_string = "\xe3\x81\x82"
+      d = create_driver CONFIG_JSON + %[
+        buffered true
+        bulk_request true
+        compress_request true
+        <buffer>
+          @type memory
+          compress gzip
+        </buffer>
+      ]
+
+      # Should not recompress
+      dont_allow(Zlib::GzipWriter).new
+
+      d.run(default_tag: 'test.metrics') do
+        d.feed({ 'field1' => 50, 'field2' => 20, 'field3' => 10, 'otherfield' => 1, 'binary' => binary_string })
+        d.feed({ 'field1' => 70, 'field2' => 30, 'field3' => 20, 'otherfield' => 2, 'binary' => binary_string })
+      end
+
+      assert_equal 1, @posts.size
+      record = @posts[0]
+
+      expected =[
+        {
+          "binary"     => "\u3042",
+          "field1"     => 50,
+          "field2"     => 20,
+          "field3"     => 10,
+          "otherfield" => 1
+        },
+        {
+          "binary"     => "\u3042",
+          "field1"     => 70,
+          "field2"     => 30,
+          "field3"     => 20,
+          "otherfield" => 2
+        }
+      ]
+
+      assert_equal expected, record[:x_ndjson]
+      assert_nil record[:auth]
+    end
   end
 
   def test_emit_form_put
@@ -859,6 +914,8 @@ class HTTPSOutputTest < HTTPOutputTestBase
     ssl_no_verify true
     cacert_file /tmp/ssl.cert
     ]
+    cert, _ = WEBrick::Utils.create_self_signed_cert(2048, [["CN", "localhost"]], "Generated Test Cert")
+    File.write('/tmp/ssl.cert', cert.to_pem)
     d = create_driver config
     d.run(default_tag: 'test.metrics') do
       d.feed({ 'field1' => 50 })

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-out-http
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.5
 platform: ruby
 authors:
 - Marica Odagaki
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
@@ -86,6 +85,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A generic Fluentd output plugin to send logs to an HTTP endpoint
 email:
 - ento.entotto@gmail.com
@@ -93,8 +120,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/test.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - ISSUE_TEMPLATE.md
@@ -104,13 +132,13 @@ files:
 - fluent-plugin-out-http.gemspec
 - lib/fluent/plugin/out_http.rb
 - lib/fluent/test/http_output_test.rb
+- run_test.rb
 - test/plugin/script/plugin/formatter_test.rb
 - test/plugin/test_out_http.rb
 homepage: https://github.com/fluent-plugins-nursery/fluent-plugin-out-http
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -125,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 4.0.6
 specification_version: 4
 summary: A generic Fluentd output plugin to send logs to an HTTP endpoint
 test_files:

data/.travis.yml

@@ -1,24 +0,0 @@
-rvm:
-  - 2.4.6
-  - 2.5.6
-  - 2.6.4
-  - ruby-head
-
-os:
-  - linux
-  - osx
-osx_image: xcode11.2
-
-dist: trusty
-
-gemfile:
-  - Gemfile
-
-before_install:
-  - gem update --system=2.7.8
-
-script: bundle exec rake test
-
-matrix:
-  allow_failures:
-    - rvm: ruby-head