fluent-plugin-vmware-loginsight 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 198031c3139a6f62c7fff502b82487c3f113cb9e0fc77540c9ac08635b3c9a09
-  data.tar.gz: f4f0b05c508be86e532c2d11a3a77523021b795a02478b68659e9f211ca22e81
+  metadata.gz: b1f7c396ae7d9694669619e530a495192cb3ba56d6eb62838d27c5333d6a8f2a
+  data.tar.gz: ea053415b736253ecae294ca26930dd3408755498ec38e2cbdea68d3c31b698b
 SHA512:
-  metadata.gz: c47fedca8aeb8945d089b0ba347fd5c304ed099f543954caa1def1daf6545eedff1ecf443b30799a75bdd5b5a9577e67fe591c236c65999ac7cd830c29eb51ee
-  data.tar.gz: 6c9184bb6e8092f7995a078424995a94b92a417141dc15be5ececb639e7cd6a3432de1d4fd8fbfc1a6d73343c291f38311c002bd9c30bbd50b331bc65db10b15
+  metadata.gz: fd8dda762a1fd5d72caa1bebbbe7ecea69e79685df2bb3a22bb79730a61197f82f7dcced544a4f01075cfaf197e3eae63a0d739aff5834878285b42612444f39
+  data.tar.gz: 3b5395db3fa3b7125ac6144d23ad789e349de49c20577cd40e87a7501c5e39c057285a22165d115fdf34adc988ab754532c299b7d8fffb0aa908883119f0060b

data/.github/workflows/gem-push.yml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby 2.6
+    - name: Set up Ruby 3.0
       uses: actions/setup-ruby@v1
       with:
-        ruby-version: 2.6.x
+        ruby-version: 3.0
 
     - name: Publish to GPR
       run: |

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v1.4.1 - Apr 25, 2023
+
+* Add support for gzip compression ([#17](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/17),[#18](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/18), [@toddc-vmware](https://github.com/toddc-vmware))
+* Add support for multi-worker ([#27](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/27),[@yajith](https://github.com/yajith))
+* Update default batch size to 4MB from 512KB ([#28](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/28), [@mohitevishal](https://github.com/mohitevishal))
+
 ## v1.4.0 - Mar 07, 2023
 
 * Base Photon image has been updated to photon:4.0-20230227

data/README.md

@@ -131,7 +131,7 @@ request_timeout, :time, :default => 5
 http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
 # Number of bytes per post request
-max_batch_size, :integer, :default => 512000
+max_batch_size, :integer, :default => 4000000
 
 # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
 rate_limit_msec, :integer, :default => 0

data/VERSION

@@ -1 +1 @@
-1.4.0
+1.4.1

data/lib/fluent/plugin/out_vmware_loginsight.rb

@@ -12,6 +12,7 @@
 require 'fluent/plugin/output'
 require 'json'
 require 'net/http'
+require 'zlib'
 require 'uri'
 
 module Fluent::Plugin
@@ -44,13 +45,15 @@ module Fluent::Plugin
     # HTTP method
     # post | put
     config_param :http_method, :string, :default => :post
+    config_param :http_compress, :bool, :default => true
     # form | json
     config_param :serializer, :string, :default => :json
     config_param :request_retries, :integer, :default => 3
     config_param :request_timeout, :time, :default => 5
     config_param :http_conn_debug, :bool, :default => false
-    # in bytes
-    config_param :max_batch_size, :integer, :default => 512000
+    # Log events batched per request.
+    # in bytes | default ~4MB
+    config_param :max_batch_size, :integer, :default => 4000000
 
     # Simple rate limiting: ignore any records within `rate_limit_msec`
     # since the last one.
@@ -114,10 +117,17 @@ module Fluent::Plugin
       url
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def set_header(req)
       if @serializer == 'json'
         set_json_header(req)
       end
+      if @http_compress
+        set_gzip_header(req)
+      end
       req
     end
 
@@ -126,6 +136,11 @@ module Fluent::Plugin
       req
     end
 
+    def set_gzip_header(req)
+      req['Content-Encoding'] = 'gzip'
+      req
+    end
+
     def shorten_key(key)
       # LI doesn't allow some characters in field 'name'
       # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
@@ -219,13 +234,23 @@ module Fluent::Plugin
       ret
     end
 
+    def get_body(req)
+      body = ""
+      if @http_compress
+        gzip_body = Zlib::GzipReader.new(StringIO.new(req.body.to_s))
+        body = gzip_body.read
+      else
+        body = req.body
+      end
+      return body[1..1024]
+    end
+
     def send_request(req, uri)
       is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
       if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
         $log.info('Dropped request due to rate limiting')
         return
       end
-
       if @auth and @auth.to_s.eql? "basic"
         req.basic_auth(@username, @password)
       end
@@ -253,7 +278,7 @@ module Fluent::Plugin
         # log-container logs to LI as well, you may end up in a cycle.
         # TODO handle the cyclic case at plugin level if possible.
         # $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: " \
-        #   "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
+        #   "#{e.class}, '#{e.message}', \n Request: #{get_body(req)}"
         retry unless (retries -= 1).zero?
         raise e if @raise_on_error
       else
@@ -267,17 +292,27 @@ module Fluent::Plugin
                           end
             # ditto cyclic warning
             # $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
-            #   "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
+            #   "Request Size: #{req.body.size} Request Body: #{get_body(req)}"
          end #end unless
       end # end begin
     end # end send_request
 
+    def set_body(req, event_req)
+        if @http_compress
+            gzip_body = Zlib::GzipWriter.new(StringIO.new)
+            gzip_body << event_req.to_json
+            req.body = gzip_body.close.string
+        else
+            req.body = event_req.to_json
+        end
+    end
+
     def send_events(uri, events)
       req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
       event_req = {
         "events" => events
       }
-      req.body = event_req.to_json
+      set_body(req, event_req)
       set_header(req)
       send_request(req, uri)
     end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-vmware-loginsight
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Vishal Mohite
 - Chris Todd
 - Samvel Israelyan
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-16 00:00:00.000000000 Z
+date: 2023-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -106,7 +106,7 @@ homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,9 +121,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: Fluend output plugin to forward logs to VMware Aria Operations For Logs
 test_files: