fluent-plugin-vmware-loginsight 1.3.1 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3d297a36377a2b0ccc771968c32a452144c902c768e4e37005b78346726b884f
-  data.tar.gz: 87f18756a8f44bfab712f683039243847120305ddb0dfbf6c09b8d3ee5159c43
+  metadata.gz: b1f7c396ae7d9694669619e530a495192cb3ba56d6eb62838d27c5333d6a8f2a
+  data.tar.gz: ea053415b736253ecae294ca26930dd3408755498ec38e2cbdea68d3c31b698b
 SHA512:
-  metadata.gz: 9262a17913a26fccd037c59a48656c815a3cffb4e49a531394514e7b7559d2bf510b393e36ce56402e6a53e0088ecb07c1aa700603e1d595c924d54250e11415
-  data.tar.gz: ad6d6702e6676afab117122ba4fdbede96ac568c08f69bbd97e294c6af6053e79cf7d328dfae0ca9e7fe89fd23879400e78a243be1bbef02b7f66b500f5daa8d
+  metadata.gz: fd8dda762a1fd5d72caa1bebbbe7ecea69e79685df2bb3a22bb79730a61197f82f7dcced544a4f01075cfaf197e3eae63a0d739aff5834878285b42612444f39
+  data.tar.gz: 3b5395db3fa3b7125ac6144d23ad789e349de49c20577cd40e87a7501c5e39c057285a22165d115fdf34adc988ab754532c299b7d8fffb0aa908883119f0060b

data/.github/workflows/gem-push.yml

@@ -9,10 +9,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby 2.6
+    - name: Set up Ruby 3.0
       uses: actions/setup-ruby@v1
       with:
-        ruby-version: 2.6.x
+        ruby-version: 3.0
 
     - name: Publish to GPR
       run: |

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v1.4.1 - Apr 25, 2023
+
+* Add support for gzip compression ([#17](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/17),[#18](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/18), [@toddc-vmware](https://github.com/toddc-vmware))
+* Add support for multi-worker ([#27](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/27),[@yajith](https://github.com/yajith))
+* Update default batch size to 4MB from 512KB ([#28](https://github.com/vmware/fluent-plugin-vmware-loginsight/pull/28), [@mohitevishal](https://github.com/mohitevishal))
+
+## v1.4.0 - Mar 07, 2023
+
+* Base Photon image has been updated to photon:4.0-20230227
+* Log Insight has been changed to VMware Aria Operations For Logs
+
 ## v1.3.1 - Nov 01, 2022
 
 * Based Photon image has been updated to photon:4.0-20221029

data/Dockerfile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2019 VMware, Inc. All Rights Reserved. 
 # 
@@ -14,7 +14,7 @@
 # Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
 # a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
 # https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
-FROM photon:4.0-20221029
+FROM photon:4.0-20230227
 
 USER root
 
@@ -45,7 +45,7 @@ RUN buildDeps="\
     && gem install --norc --no-document fluent-plugin-kubernetes_metadata_filter \
     #
     # Install Log Insight plugin
-    && gem install --norc --no-document -v 1.3.0 fluent-plugin-vmware-loginsight \
+    && gem install --norc --no-document -v 1.4.0 fluent-plugin-vmware-loginsight \
     #
     # Install jemalloc 5.3.0
     && curl -L --output /tmp/jemalloc-5.3.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \

data/Gemfile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018 VMware, Inc. All Rights Reserved. 
 # 

data/LICENSE

@@ -1,8 +1,8 @@
-Fluentd plugin for VMware Log Insight
+Fluentd plugin for VMware Aria Operations For Logs
 
 Copyright 2018 VMware, Inc.  All rights reserved				
 
-The MIT license (the ìLicenseî) set forth below applies to all parts of the Fluentd plugin for VMware Log Insight project.  You may not use this file except in compliance with the License.†
+The MIT license (the ìLicenseî) set forth below applies to all parts of the Fluentd plugin for VMware Aria Operations For Logs.  You may not use this file except in compliance with the License.†
 
 MIT License
 

data/README.md

@@ -3,7 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight.svg)](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
 
 ## Overview
-output plugin to do forward logs to VMware Log Insight
+output plugin to do forward logs to VMware Aria Operations for Logs
 
 ## Installation
 
@@ -72,13 +72,13 @@ $ bundle
   @id out_vmw_li_all_container_logs
   scheme https
   ssl_verify true
-  # Loginsight host: One may use IP address or cname
+  # VMware Aria Operations For Logs host: One may use IP address or cname
   #host X.X.X.X
   host MY_LOGINSIGHT_HOST
   port 9543
   agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
   # Keys from log event whose values should be added as log message/text to
-  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  # VMware Aria Operations For Logs. Note these key/value pairs  won't be added as metadata/fields
   log_text_keys ["log","msg","message"]
   # Use this flag if you want to enable http debug logs
   http_conn_debug false
@@ -92,13 +92,13 @@ For more examples look at [examples](./examples/)
 ```
 scheme, :string, :default => 'http' :: Valid Values: http/https 
 
-# Loginsight Host ex. localhost
+# VMware Aria Operations For Logs Host ex. localhost
 host, :string,  :default => 'localhost' :: Valid Values: loginsight_url | loginsight_ip
 
-# Loginsight port ex. 9000
+# VMware Aria Operations For Logs port ex. 9000
 port, :integer, :default => 80
 
-# Loginsight ingestion api path ex. 'api/v1/events/ingest'
+# VMware Aria Operations For Logs ingestion api path ex. 'api/v1/events/ingest'
 path, :string, :default => 'api/v1/events/ingest'
 
 # agent_id generated by your LI
@@ -131,7 +131,7 @@ request_timeout, :time, :default => 5
 http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
 # Number of bytes per post request
-max_batch_size, :integer, :default => 512000
+max_batch_size, :integer, :default => 4000000
 
 # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
 rate_limit_msec, :integer, :default => 0
@@ -139,7 +139,7 @@ rate_limit_msec, :integer, :default => 0
 # Raise errors that were rescued during HTTP requests?
 raise_on_error, :bool, :default => false :: Valid Value: true | false 
 
-# Keys from log event whose values should be added as log message/text to loginsight.
+# Keys from log event whose values should be added as log message/text to VMware Aria Operations For Logs.
 # These key/value pairs won't expanded/flattened and won't be added as metadata/fields.
 log_text_keys, :array, :default => ["log", "message", "msg"] :: Valid Value: Array of strings
 
@@ -183,7 +183,7 @@ shorten_keys, :hash, value_type: :string, default:
 The fluent-plugin-vmware-loginsight project team welcomes contributions from the community. Before you start working with fluent-plugin-vmware-loginsight, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch. For more detailed information, refer to [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## License
-Fluentd plugin for VMware Log Insight
+Fluentd plugin for VMware Aria Operations For Logs
 
 Copyright 2018 VMware, Inc. All Rights Reserved. 
 

data/Rakefile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018 VMware, Inc. All Rights Reserved. 
 # 

data/VERSION

@@ -1 +1 @@
-1.3.1
+1.4.1

data/examples/fluent.conf

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018 VMware, Inc. All Rights Reserved. 
 # 
@@ -100,7 +100,7 @@
   </parse>
 </source>
 
-# Loginsight doesn't support ingesting `source` as a field name, get rid of it
+# VMware Aria Operations For Logs doesn't support ingesting `source` as a field name, get rid of it
 <filter kube-audit>
   @type record_transformer
   @id filter_kube_audit_logs
@@ -117,13 +117,13 @@
   @id out_vmw_li_my_namespace_logs
   scheme http
   ssl_verify false
-  # Loginsight host: One may use IP address or cname
+  # VMware Aria Operations For Logs host: One may use IP address or cname
   #host X.X.X.X
   host MY_LOGINSIGHT_HOST
   port 9000
   agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
   # Keys from log event whose values should be added as log message/text to
-  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  # VMware Aria Operations For Logs. Note these key/value pairs  won't be added as metadata/fields
   log_text_keys ["log","msg","message"]
   # Use this flag if you want to enable http debug logs
   http_conn_debug false
@@ -137,13 +137,13 @@
     @id out_vmw_li_all_container_logs
     scheme https
     ssl_verify true
-    # Loginsight host: One may use IP address or cname
+    # VMware Aria Operations For Logs host: One may use IP address or cname
     #host X.X.X.X
     host MY_LOGINSIGHT_HOST
     port 9543
     agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
     # Keys from log event whose values should be added as log message/text to
-    # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+    # VMware Aria Operations For Logs. Note these key/value pairs  won't be added as metadata/fields
     log_text_keys ["log","msg","message"]
     # Use this flag if you want to enable http debug logs
     http_conn_debug false

data/examples/fluentd-vrli-plugin-debian.dockerfile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018-2019 VMware, Inc. All Rights Reserved. 
 # 

data/examples/fluentd-vrli-plugin-photon-tdnf.dockerfile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2019 VMware, Inc. All Rights Reserved. 
 # 
@@ -30,7 +30,7 @@ RUN tdnf distro-sync --refresh -y \
     rubygem-async-http-0.48.2 \
     jemalloc-4.5.0 \
     #
-    # Install Log Insight plugin
+    # Install VMware Aria Operations For Logs plugin
     rubygem-fluent-plugin-vmware-loginsight-0.1.5
 
 RUN ln -s /usr/lib/ruby/gems/2.5.0/bin/fluentd /usr/bin/fluentd \

data/examples/fluentd-vrli-plugin-photon.dockerfile

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2019 VMware, Inc. All Rights Reserved. 
 # 
@@ -38,7 +38,7 @@ RUN buildDeps="\
     && gem install --norc --no-document fluentd -v 1.6.3 \
     && mkdir -p /fluentd/etc /fluentd/plugins \
     #
-    # Install Log Insight plugin
+    # Install VMware Aria Operations For Logs plugin
     && gem install --norc --no-document -v 0.1.5 fluent-plugin-vmware-loginsight \
     #
     # Install jemalloc 4.5.0

data/examples/k8s-log-collector-ds.yaml

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018 VMware, Inc. All Rights Reserved. 
 # 
@@ -121,7 +121,7 @@ data:
         time_format %Y-%m-%dT%H:%M:%SZ
       </parse>
     </source>
-    # Loginsight doesn't support ingesting `source` as a field name, get rid of it
+    # VMware Aria Operations For Logs doesn't support ingesting `source` as a field name, get rid of it
     <filter kube-audit>
       @type record_transformer
       @id filter_kube_audit_logs
@@ -145,13 +145,13 @@ data:
         @id out_vmw_li_all_container_logs
         scheme https
         ssl_verify true
-        # Loginsight host: One may use IP address or cname
+        # VMware Aria Operations For Logs host: One may use IP address or cname
         #host X.X.X.X
         host MY_LOGINSIGHT_HOST
         port 9543
         agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
         # Keys from log event whose values should be added as log message/text to
-        # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+        # VMware Aria Operations For Logs. Note these key/value pairs  won't be added as metadata/fields
         log_text_keys ["log","msg","message"]
         # Use this flag if you want to enable http debug logs
         http_conn_debug false

data/fluent-plugin-vmware-loginsight.gemspec

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 # 
 # Copyright 2018 VMware, Inc. All Rights Reserved. 
 # 
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.authors = ["Vishal Mohite", "Chris Todd", "Samvel Israelyan"]
   spec.email   = ["vmohite@vmware.com", "toddc@vmware.com", "sisraelyan@vmware.com"]
 
-  spec.summary       = %q{Fluend output plugin to forward logs to VMware Log Insight}
+  spec.summary       = %q{Fluend output plugin to forward logs to VMware Aria Operations For Logs}
   spec.description   = spec.summary
   spec.homepage      = "https://github.com/vmware/fluent-plugin-vmware-loginsight"
   spec.license       = "MIT"

data/lib/fluent/plugin/out_vmware_loginsight.rb

@@ -1,4 +1,4 @@
-# Fluentd plugin for VMware Log Insight
+# Fluentd plugin for VMware Aria Operations For Logs
 #
 # Copyright 2018 VMware, Inc. All Rights Reserved.
 #
@@ -12,6 +12,7 @@
 require 'fluent/plugin/output'
 require 'json'
 require 'net/http'
+require 'zlib'
 require 'uri'
 
 module Fluent::Plugin
@@ -20,13 +21,13 @@ module Fluent::Plugin
 
     ### Connection Params ###
     config_param :scheme, :string, :default => 'http'
-    # Loginsight Host ex. localhost
+    # VMware Aria Operations For Logs Host ex. localhost
     config_param :host, :string,  :default => 'localhost'
     # In case we want to post to  multiple hosts. This is futuristic, Fluentd copy plugin can support this as is
     #config_param :hosts, :string, :default => nil
-    # Loginsight port ex. 9000. Default 80
+    # VMware Aria Operations For Logs port ex. 9000. Default 80
     config_param :port, :integer, :default => 80
-    # Loginsight ingestion api path ex. 'api/v1/events/ingest'
+    # VMware Aria Operations For Logs ingestion api path ex. 'api/v1/events/ingest'
     config_param :path, :string, :default => 'api/v1/events/ingest'
     # agent_id generated by your LI
     config_param :agent_id, :string, :default => '0'
@@ -44,13 +45,15 @@ module Fluent::Plugin
     # HTTP method
     # post | put
     config_param :http_method, :string, :default => :post
+    config_param :http_compress, :bool, :default => true
     # form | json
     config_param :serializer, :string, :default => :json
     config_param :request_retries, :integer, :default => 3
     config_param :request_timeout, :time, :default => 5
     config_param :http_conn_debug, :bool, :default => false
-    # in bytes
-    config_param :max_batch_size, :integer, :default => 512000
+    # Log events batched per request.
+    # in bytes | default ~4MB
+    config_param :max_batch_size, :integer, :default => 4000000
 
     # Simple rate limiting: ignore any records within `rate_limit_msec`
     # since the last one.
@@ -58,7 +61,7 @@ module Fluent::Plugin
     # Raise errors that were rescued during HTTP requests?
     config_param :raise_on_error, :bool, :default => false
     # Keys from log event whose values should be added as log message/text
-    # to loginsight. Note these key/value pairs  won't be added as metadata/fields
+    # to VMware Aria Operations For Logs. Note these key/value pairs  won't be added as metadata/fields
     config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
     # Flatten hashes to create one key/val pair w/o losing log data
     config_param :flatten_hashes, :bool, :default => true
@@ -114,10 +117,17 @@ module Fluent::Plugin
       url
     end
 
+    def multi_workers_ready?
+      true
+    end
+
     def set_header(req)
       if @serializer == 'json'
         set_json_header(req)
       end
+      if @http_compress
+        set_gzip_header(req)
+      end
       req
     end
 
@@ -126,6 +136,11 @@ module Fluent::Plugin
       req
     end
 
+    def set_gzip_header(req)
+      req['Content-Encoding'] = 'gzip'
+      req
+    end
+
     def shorten_key(key)
       # LI doesn't allow some characters in field 'name'
       # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
@@ -219,13 +234,23 @@ module Fluent::Plugin
       ret
     end
 
+    def get_body(req)
+      body = ""
+      if @http_compress
+        gzip_body = Zlib::GzipReader.new(StringIO.new(req.body.to_s))
+        body = gzip_body.read
+      else
+        body = req.body
+      end
+      return body[1..1024]
+    end
+
     def send_request(req, uri)
       is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
       if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
         $log.info('Dropped request due to rate limiting')
         return
       end
-
       if @auth and @auth.to_s.eql? "basic"
         req.basic_auth(@username, @password)
       end
@@ -253,7 +278,7 @@ module Fluent::Plugin
         # log-container logs to LI as well, you may end up in a cycle.
         # TODO handle the cyclic case at plugin level if possible.
         # $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: " \
-        #   "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
+        #   "#{e.class}, '#{e.message}', \n Request: #{get_body(req)}"
         retry unless (retries -= 1).zero?
         raise e if @raise_on_error
       else
@@ -267,17 +292,27 @@ module Fluent::Plugin
                           end
             # ditto cyclic warning
             # $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
-            #   "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
+            #   "Request Size: #{req.body.size} Request Body: #{get_body(req)}"
          end #end unless
       end # end begin
     end # end send_request
 
+    def set_body(req, event_req)
+        if @http_compress
+            gzip_body = Zlib::GzipWriter.new(StringIO.new)
+            gzip_body << event_req.to_json
+            req.body = gzip_body.close.string
+        else
+            req.body = event_req.to_json
+        end
+    end
+
     def send_events(uri, events)
       req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
       event_req = {
         "events" => events
       }
-      req.body = event_req.to_json
+      set_body(req, event_req)
       set_header(req)
       send_request(req, uri)
     end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-vmware-loginsight
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.1
 platform: ruby
 authors:
 - Vishal Mohite
 - Chris Todd
 - Samvel Israelyan
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-05 00:00:00.000000000 Z
+date: 2023-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -74,7 +74,7 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-description: Fluend output plugin to forward logs to VMware Log Insight
+description: Fluend output plugin to forward logs to VMware Aria Operations For Logs
 email:
 - vmohite@vmware.com
 - toddc@vmware.com
@@ -106,7 +106,7 @@ homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -121,11 +121,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
-summary: Fluend output plugin to forward logs to VMware Log Insight
+summary: Fluend output plugin to forward logs to VMware Aria Operations For Logs
 test_files:
 - test/helper.rb
 - test/plugin/test_out_vmware_loginsight.rb