fluent-plugin-logzio 0.0.16 → 0.0.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 75acfaf69be145654b96a47df4ebccc3a41ab018
-  data.tar.gz: 9070b8733148fa6366235a50909f55494eae9c27
+SHA256:
+  metadata.gz: a6b850642fbf62e80b5033d03300ea4cbb3e600484836ae411877546785182a3
+  data.tar.gz: 2a40dc72118892e5bdf8bbf2becf44e6e2bf8688470034ae0152fb918b05a481
 SHA512:
-  metadata.gz: 0471525a8263e3e242607e912114727305fe3a2ccdb89c5d854cba6c732e5339c587e3931c0663c6357d18aabe26c224a214d4b8c75d88a4a4c7d06ef02bf124
-  data.tar.gz: 201dcb57eea7ef2687ab84e0f51070855ee26cd344cf59f31d18578843bbde3ad3613ceeecd9319564af9036fe8298ac5d849b14da5d291ff688bcf25050a740
+  metadata.gz: b98047acec6ffededc5c44f22ac3f2b451da2fc5c3b6a9d0b32f7d1e1a5e838f06c5ab1cdb4754f30500817a74bfebcbf0e92d1583b14d48f085ce348a203e7b
+  data.tar.gz: c05ed2eb34bd102938d8b00708ffa6ac0978b5420d46001925dcf0a9042e9b46ec14868b42b6396eb0c0be80663d81579186c58ed1fb2650643bd6ce52f10fa1

data/.whitesource

@@ -0,0 +1,8 @@
+{
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW"
+  }
+}

data/README.md

@@ -68,8 +68,17 @@ This is an **example** only. Your needs in production may vary!
 * **retry_count** How many times to resend failed bulks. Defaults to 4 times.
 * **retry_sleep** How long to sleep initially between retries, exponential step-off. Initial default is 2s.
 * **bulk_limit** Limit to the size of the Logz.io upload bulk. Defaults to 1000000 bytes leaving about 24kB for overhead.
+* **bulk_limit_warning_limit** Limit to the size of the Logz.io warning message when a record exceeds bulk_limit to prevent a recursion when Fluent warnings are sent to the Logz.io output.  Defaults to nil (no truncation).
+* **proxy_uri** Your proxy uri. Default is nil
+* **proxy_cert** Your proxy cert. Default is nil
+* **gzip** should the plugin ship the logs in gzip compression. Default is false
+
 
 ## Release Notes
+- 0.0.21: Update gem `net-http-persistent` to 3.x.
+- 0.0.20: Support gzip compression
+- 0.0.18: Support proxy_uri and proxy_cert in the configuration file. Put logzio output plugin class under Fluent::Plugin module and thus work with multi workers.
+- 0.0.17: Optional truncate log messages when they are exceeding bulk size in warning logs
 - 0.0.16: More fluentD 1.0+ adjustments
 - 0.0.15: Support FluentD 1.0+. Split the chunk into bulk uploads, decoupling `chunk_limit_size`/`buffer_chunk_limit` from Logz.io bulk limit. Tunable `bulk_limit` and initial `retry_sleep`.
 - 0.0.14: Refactor send function to handle more cases, and retry in case of logzio connection failure.

data/fluent-plugin-logzio.gemspec

@@ -4,9 +4,9 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-logzio'
-  s.version     = '0.0.16'
-  s.authors     = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov']
-  s.email       = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz']
+  s.version     = '0.0.21'
+  s.authors     = ['Yury Kotov', 'Roi Rav-Hon', 'Arcadiy Ivanov', 'Miri Ignatiev']
+  s.email       = ['bairkan@gmail.com', 'roi@logz.io', 'arcadiy@ivanov.biz', 'miri.ignatiev@logz.io']
   s.homepage    = 'https://github.com/logzio/fluent-plugin-logzio'
   s.summary     = %q{Fluentd plugin for output to Logz.io}
   s.description = %q{Fluentd pluging (fluent.org) for output to Logz.io (logz.io)}
@@ -19,10 +19,10 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
   s.required_ruby_version = Gem::Requirement.new('>= 2.1.0')
 
-  s.add_dependency 'net-http-persistent', '~> 2.9'
+  s.add_dependency 'net-http-persistent', '~> 3.0'
   s.add_runtime_dependency 'fluentd', ['>= 0.14.0', '< 2']
   s.add_development_dependency 'rake', '~> 12.3'
   s.add_development_dependency 'bundler', '~> 1.16'
   s.add_development_dependency 'rspec', '~> 3.7'
   s.add_development_dependency 'test-unit', '~> 3.2'
-end
+end

data/lib/fluent/plugin/out_logzio_buffered.rb

@@ -1,7 +1,9 @@
 require 'time'
 require 'fluent/plugin/output'
+require 'zlib'
+require 'stringio'
 
-module Fluent
+module Fluent::Plugin
   class LogzioOutputBuffered < Output
     Fluent::Plugin.register_output('logzio_buffered', self)
 
@@ -13,22 +15,40 @@ module Fluent
     config_param :retry_count, :integer, default: 4 # How many times to resend failed bulks. Undocumented because not suppose to be changed
     config_param :retry_sleep, :integer, default: 2 # How long to sleep initially between retries, exponential step-off
     config_param :bulk_limit, :integer, default: 1000000 # Make sure submission to LogzIO does not exceed 1MB limit and leave some overhead
+    config_param :bulk_limit_warning_limit, :integer, default: nil # If fluent warnings are sent to the Logzio output, truncating is necessary to prevent a recursion
     config_param :http_idle_timeout, :integer, default: 5
     config_param :output_tags_fieldname, :string, default: 'fluentd_tags'
+    config_param :proxy_uri, :string, default: nil
+    config_param :proxy_cert, :string, default: nil
+    config_param :gzip, :bool, default: false # False for backward compatibility
 
     def configure(conf)
       super
       compat_parameters_convert(conf, :buffer)
 
       log.debug "Logz.io URL #{@endpoint_url}"
+
+      if conf['proxy_uri']
+        log.debug "Proxy #{@proxy_uri}"
+        ENV['http_proxy'] = @proxy_uri
+      end
+
+      if conf['proxy_cert']
+        log.debug "Proxy #{@proxy_cert}"
+        ENV['SSL_CERT_FILE'] = @proxy_cert
+      end
+
     end
 
     def start
       super
       require 'net/http/persistent'
       @uri = URI @endpoint_url
-      @http = Net::HTTP::Persistent.new 'fluent-plugin-logzio', :ENV
+      @http = Net::HTTP::Persistent.new name: 'fluent-plugin-logzio', proxy: :ENV
       @http.headers['Content-Type'] = 'text/plain'
+      if @gzip
+        @http.headers['Content-Encoding'] = 'gzip'
+      end
       @http.idle_timeout = @http_idle_timeout
       @http.socket_options << [Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1]
 
@@ -43,6 +63,10 @@ module Fluent
       true
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def format(tag, time, record)
       if time.is_a?(Fluent::EventTime)
         sec_frac = time.to_f
@@ -74,7 +98,13 @@ module Fluent
         end
 
         if record_size > @bulk_limit
-          log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
+          if @bulk_limit_warning_limit.is_a?(Integer)
+            log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record starts with (truncated at #{@bulk_limit_warning_limit} characters): #{json_record[0,@bulk_limit_warning_limit]}"
+            # Send the full message to debug facility
+            log.debug "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
+          else
+            log.warn "Record with size #{record_size} exceeds #{@bulk_limit} and can't be sent to Logz.io. Record is: #{json_record}"
+          end
           next
         end
         if bulk_size + record_size > @bulk_limit
@@ -98,7 +128,9 @@ module Fluent
 
       # Logz.io bulk http endpoint expecting log line with \n delimiter
       post.body = bulk_records.join("\n")
-
+      if gzip
+        post.body = compress(post.body)
+      end
       sleep_interval = @retry_sleep
 
       begin
@@ -114,14 +146,14 @@ module Fluent
                 log.info "Got 400 code from Logz.io. This means that some of your logs are too big, or badly formatted. Response: #{response.body}"
                 should_retry = false
               else
-                log.debug "Got HTTP #{response.code} from Logz.io, not giving up just yet (Try #{counter + 1}/#{@retry_count})"
+                log.warn "Got HTTP #{response.code} from Logz.io, not giving up just yet (Try #{counter + 1}/#{@retry_count})"
               end
             else
               log.debug "Successfully sent bulk of #{bulk_records.size} records, size #{bulk_size}B to Logz.io"
               should_retry = false
             end
           rescue StandardError => e
-            log.debug "Error connecting to Logz.io. Got exception: #{e} (Try #{counter + 1}/#{@retry_count})"
+            log.warn "Error connecting to Logz.io. Got exception: #{e} (Try #{counter + 1}/#{@retry_count})"
           end
 
           if should_retry
@@ -139,5 +171,13 @@ module Fluent
         log.error "Got unexpected exception! Here: #{e}"
       end
     end
+
+    def compress(string)
+      wio = StringIO.new("w")
+      w_gz = Zlib::GzipWriter.new(wio)
+      w_gz.write(string)
+      w_gz.close
+      wio.string
+    end
   end
 end

data/spec/lib/fluent/plugin/out_logzio_buffered_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
-describe 'Fluent::LogzioOutputBuffered' do
-  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
+describe 'Fluent::Plugin::LogzioOutputBuffered' do
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
   let(:config) do
     %[
       endpoint_url         https://logz.io?token=123

data/spec/lib/fluent/plugin/out_logzio_multi_bulk_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
-describe 'Fluent::LogzioOutputBuffered' do
-  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
+describe 'Fluent::Plugin::LogzioOutputBuffered' do
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
   let(:config) do
     %[
       endpoint_url         https://logz.io?token=123

data/spec/lib/fluent/plugin/out_logzio_record_too_large_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
-describe 'Fluent::LogzioOutputBuffered' do
-  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::LogzioOutputBuffered).configure(config) }
+describe 'Fluent::Plugin::LogzioOutputBuffered' do
+  let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::LogzioOutputBuffered).configure(config) }
   let(:config) do
     %[
       endpoint_url         https://logz.io?token=123

metadata

@@ -1,16 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-logzio
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.21
 platform: ruby
 authors:
 - Yury Kotov
 - Roi Rav-Hon
 - Arcadiy Ivanov
+- Miri Ignatiev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-06 00:00:00.000000000 Z
+date: 2021-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-http-persistent
@@ -18,14 +19,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -107,12 +108,14 @@ email:
 - bairkan@gmail.com
 - roi@logz.io
 - arcadiy@ivanov.biz
+- miri.ignatiev@logz.io
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- ".whitesource"
 - Gemfile
 - LICENSE
 - README.md
@@ -142,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: fluent-plugin-logzio
-rubygems_version: 2.5.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for output to Logz.io