fluent-plugin-vmware-loginsight 1.4.0 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 198031c3139a6f62c7fff502b82487c3f113cb9e0fc77540c9ac08635b3c9a09
-  data.tar.gz: f4f0b05c508be86e532c2d11a3a77523021b795a02478b68659e9f211ca22e81
+  metadata.gz: e3aff37b5e1d472a2dfdb139d313a6558d021c8000f3bd3e2d5b6cf4a11d87e7
+  data.tar.gz: 2798b507094394c297f7134ff2fed20ba75c586b3693bfd93245b2ddf4c21406
 SHA512:
-  metadata.gz: c47fedca8aeb8945d089b0ba347fd5c304ed099f543954caa1def1daf6545eedff1ecf443b30799a75bdd5b5a9577e67fe591c236c65999ac7cd830c29eb51ee
-  data.tar.gz: 6c9184bb6e8092f7995a078424995a94b92a417141dc15be5ececb639e7cd6a3432de1d4fd8fbfc1a6d73343c291f38311c002bd9c30bbd50b331bc65db10b15
+  metadata.gz: 1d2a2405b82847309bd1852ab87c88781119f0901c26e452628a322714369e40c9b3e65a493c07762a5f142d432acba927ba454a8016ce271cfc5e19df24bf6e
+  data.tar.gz: 03ed49a376b299f5bbe3a0e18986a4e148aeb0425377ae138ea9e915d65abfdeb3a69fef7cf0a9c7dc5fdbaf71005f12fdb30f2b8acec862833dc20da0d3b7a1

data/.github/workflows/gem-push.yml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby 2.6
+    - name: Set up Ruby 3.0
       uses: actions/setup-ruby@v1
       with:
-        ruby-version: 2.6.x
+        ruby-version: 3.0
 
     - name: Publish to GPR
       run: |

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## v1.4.2 - Oct 19, 2023
+
+* Fixed the issue with the timestamp ignoring milliseconds ([#33](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/33))
+
+## v1.4.1 - Apr 25, 2023
+
+* Add support for gzip compression ([#17](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/17),[#18](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/18), [@toddc-vmware](https://github.com/toddc-vmware))
+* Add support for multi-worker ([#27](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/27),[@yajith](https://github.com/yajith))
+* Update default batch size to 4MB from 512KB ([#28](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/28), [@mohitevishal](https://github.com/mohitevishal))
+
 ## v1.4.0 - Mar 07, 2023
 
 * Base Photon image has been updated to photon:4.0-20230227

data/Dockerfile

@@ -45,7 +45,7 @@ RUN buildDeps="\
     && gem install --norc --no-document fluent-plugin-kubernetes_metadata_filter \
     #
     # Install Log Insight plugin
-    && gem install --norc --no-document -v 1.4.0 fluent-plugin-vmware-loginsight \
+    && gem install --norc --no-document -v 1.4.1 fluent-plugin-vmware-loginsight \
     #
     # Install jemalloc 5.3.0
     && curl -L --output /tmp/jemalloc-5.3.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \

data/README.md

@@ -131,7 +131,7 @@ request_timeout, :time, :default => 5
 http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
 # Number of bytes per post request
-max_batch_size, :integer, :default => 512000
+max_batch_size, :integer, :default => 4000000
 
 # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
 rate_limit_msec, :integer, :default => 0

data/VERSION

@@ -1 +1 @@
-1.4.0
+1.4.2

data/lib/fluent/plugin/out_vmware_loginsight.rb

@@ -12,6 +12,7 @@
 require 'fluent/plugin/output'
 require 'json'
 require 'net/http'
+require 'zlib'
 require 'uri'
 
 module Fluent::Plugin
@@ -44,13 +45,15 @@ module Fluent::Plugin
     # HTTP method
     # post | put
     config_param :http_method, :string, :default => :post
+    config_param :http_compress, :bool, :default => true
     # form | json
     config_param :serializer, :string, :default => :json
     config_param :request_retries, :integer, :default => 3
     config_param :request_timeout, :time, :default => 5
     config_param :http_conn_debug, :bool, :default => false
-    # in bytes
-    config_param :max_batch_size, :integer, :default => 512000
+    # Log events batched per request.
+    # in bytes | default ~4MB
+    config_param :max_batch_size, :integer, :default => 4000000
 
     # Simple rate limiting: ignore any records within `rate_limit_msec`
     # since the last one.
@@ -114,10 +117,17 @@ module Fluent::Plugin
       url
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def set_header(req)
       if @serializer == 'json'
         set_json_header(req)
       end
+      if @http_compress
+        set_gzip_header(req)
+      end
       req
     end
 
@@ -126,6 +136,11 @@ module Fluent::Plugin
       req
     end
 
+    def set_gzip_header(req)
+      req['Content-Encoding'] = 'gzip'
+      req
+    end
+
     def shorten_key(key)
       # LI doesn't allow some characters in field 'name'
       # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
@@ -192,7 +207,7 @@ module Fluent::Plugin
       event = {
         "fields" => fields,
         "text" => log.gsub(/^$\n/, ''),
-        "timestamp" => time * 1000
+        "timestamp" => (time.to_f * 1000).floor()
       }
       event
     end
@@ -219,13 +234,23 @@ module Fluent::Plugin
       ret
     end
 
+    def get_body(req)
+      body = ""
+      if @http_compress
+        gzip_body = Zlib::GzipReader.new(StringIO.new(req.body.to_s))
+        body = gzip_body.read
+      else
+        body = req.body
+      end
+      return body[1..1024]
+    end
+
     def send_request(req, uri)
       is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
       if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
         $log.info('Dropped request due to rate limiting')
         return
       end
-
       if @auth and @auth.to_s.eql? "basic"
         req.basic_auth(@username, @password)
       end
@@ -253,7 +278,7 @@ module Fluent::Plugin
         # log-container logs to LI as well, you may end up in a cycle.
         # TODO handle the cyclic case at plugin level if possible.
         # $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: " \
-        #   "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
+        #   "#{e.class}, '#{e.message}', \n Request: #{get_body(req)}"
         retry unless (retries -= 1).zero?
         raise e if @raise_on_error
       else
@@ -267,17 +292,27 @@ module Fluent::Plugin
                           end
             # ditto cyclic warning
             # $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
-            #   "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
+            #   "Request Size: #{req.body.size} Request Body: #{get_body(req)}"
          end #end unless
       end # end begin
     end # end send_request
 
+    def set_body(req, event_req)
+        if @http_compress
+            gzip_body = Zlib::GzipWriter.new(StringIO.new)
+            gzip_body << event_req.to_json
+            req.body = gzip_body.close.string
+        else
+            req.body = event_req.to_json
+        end
+    end
+
     def send_events(uri, events)
       req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
       event_req = {
         "events" => events
       }
-      req.body = event_req.to_json
+      set_body(req, event_req)
       set_header(req)
       send_request(req, uri)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-vmware-loginsight
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.2
 platform: ruby
 authors:
 - Vishal Mohite
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-16 00:00:00.000000000 Z
+date: 2023-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -121,8 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: Fluend output plugin to forward logs to VMware Aria Operations For Logs