fluent-plugin-vmware-loginsight 1.0.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7e7d17226c807f4fffdd8723c9f260730af5de29334a1b10271993b21709185
-  data.tar.gz: 768c2724b2b84ac8ae8c1e4473dbe850c37a87102aaed95d6c52fc4f06f9a0df
+  metadata.gz: 3d297a36377a2b0ccc771968c32a452144c902c768e4e37005b78346726b884f
+  data.tar.gz: 87f18756a8f44bfab712f683039243847120305ddb0dfbf6c09b8d3ee5159c43
 SHA512:
-  metadata.gz: 912e7dfa34b63dc9da92635603e88fea94d340ce8fb9aef99827bfd6896690a89533fe523416c2eafd34da2470016b13d23c290c74d93dc05fdb320beec78b55
-  data.tar.gz: 5998ef3e2869d13cd30dc934347b3dad6ebe2337de5678a7fb12f641b15a5b0a5a60daaa8bd9b5490ecdf58e5fb5f3529dd313ea2606126aaae8456dad023ffb
+  metadata.gz: 9262a17913a26fccd037c59a48656c815a3cffb4e49a531394514e7b7559d2bf510b393e36ce56402e6a53e0088ecb07c1aa700603e1d595c924d54250e11415
+  data.tar.gz: ad6d6702e6676afab117122ba4fdbede96ac568c08f69bbd97e294c6af6053e79cf7d328dfae0ca9e7fe89fd23879400e78a243be1bbef02b7f66b500f5daa8d

data/.github/workflows/gem-push.yml

@@ -26,13 +26,13 @@ jobs:
         GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
         OWNER: ${{ github.repository_owner }}
 
-    - name: Publish to RubyGems
-      run: |
-        mkdir -p $HOME/.gem
-        touch $HOME/.gem/credentials
-        chmod 0600 $HOME/.gem/credentials
-        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
-        gem build *.gemspec
-        gem push *.gem
-      env:
-        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"
+#     - name: Publish to RubyGems
+#       run: |
+#         mkdir -p $HOME/.gem
+#         touch $HOME/.gem/credentials
+#         chmod 0600 $HOME/.gem/credentials
+#         printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+#         gem build *.gemspec
+#         gem push *.gem
+#       env:
+#         GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
-## v1.0.0 - April 19, 2021
+## v1.3.1 - Nov 01, 2022
+
+* Based Photon image has been updated to photon:4.0-20221029
+
+## v1.3.0 - July 29, 2022
+
+* Added buffering support which allows sending out logs in chunks rather than one by one 
+
+## v1.0.0 - April 20, 2021
 
 * Update plugin structure to use Fluentd 1.x syntax
 

data/Dockerfile

@@ -0,0 +1,72 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2019 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
+# (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
+#
+# Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
+# a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
+# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
+FROM photon:4.0-20221029
+
+USER root
+
+RUN buildDeps="\
+    binutils linux-api-headers glibc-devel \
+    make gcc gmp-devel libffi-devel \
+    tar bzip2 sed gawk build-essential" \
+    #
+    # Distro sync and install build dependencies
+    && tdnf distro-sync --refresh -y \
+    && tdnf remove -y toybox \
+    && tdnf install -y $buildDeps ruby \
+    #
+    # These are not required but are used if available
+    && gem install oj -v 3.3.10 \
+    && gem install json -v 2.2.0 \
+    && gem install async-http -v 0.46.3 \
+    #
+    # Install fluentd
+    && gem install --norc --no-document fluentd \
+    && gem install --norc --no-document fluent-plugin-systemd \
+    && gem install --norc --no-document i18n \
+    && gem install --norc --no-document fluent-plugin-concat \
+    && gem install --norc --no-document fluent-plugin-remote_syslog \
+    && gem install --norc --no-document fluent-plugin-docker_metadata_filter \
+    && gem install --norc --no-document fluent-plugin-detect-exceptions \
+    && gem install --norc --no-document fluent-plugin-multi-format-parser \
+    && gem install --norc --no-document fluent-plugin-kubernetes_metadata_filter \
+    #
+    # Install Log Insight plugin
+    && gem install --norc --no-document -v 1.3.0 fluent-plugin-vmware-loginsight \
+    #
+    # Install jemalloc 5.3.0
+    && curl -L --output /tmp/jemalloc-5.3.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2 \
+    && tar -C /tmp/ -xjvf /tmp/jemalloc-5.3.0.tar.bz2 \
+    && cd /tmp/jemalloc-5.3.0 \
+    && ./configure && make \
+    && mv lib/libjemalloc.so.2 /usr/lib \
+    && cd / \
+    && rm -rf /tmp/jemalloc-5.3.0* \
+    && tdnf remove -y $buildDeps \
+    && tdnf clean all \
+    && gem sources --clear-all \
+    && gem cleanup
+
+# Create default fluent.conf
+RUN fluentd --setup /etc/fluentd
+
+# Make sure fluentd picks jemalloc
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+# Standard fluentd ports
+EXPOSE 24224 5140
+
+ENTRYPOINT ["fluentd", "-c", "/etc/fluentd/fluent.conf"]

data/README.md

@@ -43,6 +43,8 @@ $ bundle
   enable_stat_watcher false
   refresh_interval 5
   tag kubernetes.*
+  # Open below line if you need have filename as tag field (now without prefix kubernetes.)
+  # path_key tag
   <parse>
     @type json
     time_key time
@@ -137,12 +139,6 @@ rate_limit_msec, :integer, :default => 0
 # Raise errors that were rescued during HTTP requests?
 raise_on_error, :bool, :default => false :: Valid Value: true | false 
 
-# Include tag key as log event?
-include_tag_key, :bool, :default => true :: Valid Value: true | false 
-
-# Metadata key that identifies Fluentd tags
-tag_key, :string, :default => 'tag'
-
 # Keys from log event whose values should be added as log message/text to loginsight.
 # These key/value pairs won't expanded/flattened and won't be added as metadata/fields.
 log_text_keys, :array, :default => ["log", "message", "msg"] :: Valid Value: Array of strings

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.3.1

data/fluent-plugin-vmware-loginsight.gemspec

@@ -15,8 +15,8 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-vmware-loginsight"
   spec.version = File.read("VERSION").strip
-  spec.authors = ["Vishal Mohite", "Chris Todd"]
-  spec.email   = ["vmohite@vmware.com", "toddc@vmware.com"]
+  spec.authors = ["Vishal Mohite", "Chris Todd", "Samvel Israelyan"]
+  spec.email   = ["vmohite@vmware.com", "toddc@vmware.com", "sisraelyan@vmware.com"]
 
   spec.summary       = %q{Fluend output plugin to forward logs to VMware Log Insight}
   spec.description   = spec.summary
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "bundler", ">= 2.1.0"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]

data/lib/fluent/plugin/out_vmware_loginsight.rb

@@ -49,6 +49,7 @@ module Fluent::Plugin
     config_param :request_retries, :integer, :default => 3
     config_param :request_timeout, :time, :default => 5
     config_param :http_conn_debug, :bool, :default => false
+    # in bytes
     config_param :max_batch_size, :integer, :default => 512000
 
     # Simple rate limiting: ignore any records within `rate_limit_msec`
@@ -56,10 +57,6 @@ module Fluent::Plugin
     config_param :rate_limit_msec, :integer, :default => 0
     # Raise errors that were rescued during HTTP requests?
     config_param :raise_on_error, :bool, :default => false
-    ### Additional Params
-    config_param :include_tag_key, :bool, :default => true
-    # Metadata key that identifies Fluentd tags
-    config_param :tag_key, :string, :default => 'tag'
     # Keys from log event whose values should be added as log message/text
     # to loginsight. Note these key/value pairs  won't be added as metadata/fields
     config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
@@ -92,6 +89,12 @@ module Fluent::Plugin
           'container_':''
       }
 
+    config_section :buffer do
+      config_set_default :@type, "memory"
+      config_set_default :chunk_keys, []
+      config_set_default :timekey_use_utc, true
+    end
+
     def configure(conf)
       super
 
@@ -134,15 +137,13 @@ module Fluent::Plugin
       key
     end
 
-    def create_loginsight_event(tag, time, record)
+    def create_loginsight_event(time, record)
       flattened_records = {}
       if @flatten_hashes
         flattened_records = flatten_record(record, [])
       else
         flattened_records = record
       end
-      # tag can be immutable in some cases, use a copy.
-      flattened_records[@tag_key] = tag.dup if @include_tag_key
       fields = []
       keys = []
       log = ''
@@ -218,15 +219,6 @@ module Fluent::Plugin
       ret
     end
 
-    def create_request(tag, time, record)
-      url = format_url()
-      uri = URI.parse(url)
-      req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
-      set_body(req, tag, time, record)
-      set_header(req)
-      return req, uri
-    end
-
     def send_request(req, uri)
       is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
       if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
@@ -290,13 +282,13 @@ module Fluent::Plugin
       send_request(req, uri)
     end
 
-    def handle_records(tag, es)
+    def handle_records(chunk)
       url = format_url()
       uri = URI.parse(url)
       events = []
       count = 0
-      es.each do |time, record|
-        new_event = create_loginsight_event(tag, time, record)
+      chunk.each do |time, record|
+        new_event = create_loginsight_event(time, record)
         new_event_size = new_event.to_json.size
         if new_event_size > @max_batch_size
             $log.warn "dropping event larger than max_batch_size: #{new_event.to_json[1..1024]}"
@@ -315,8 +307,9 @@ module Fluent::Plugin
       end
     end
 
-    def process(tag, es)
-      handle_records(tag, es)
+    # Sync Buffered Output
+    def write(chunk)
+      handle_records(chunk)
     end
   end
 end

metadata

@@ -1,30 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-vmware-loginsight
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Vishal Mohite
 - Chris Todd
-autorequire:
+- Samvel Israelyan
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2022-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 2.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -77,6 +78,7 @@ description: Fluend output plugin to forward logs to VMware Log Insight
 email:
 - vmohite@vmware.com
 - toddc@vmware.com
+- sisraelyan@vmware.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -85,6 +87,7 @@ files:
 - ".gitignore"
 - CHANGELOG.md
 - CONTRIBUTING.md
+- Dockerfile
 - Gemfile
 - LICENSE
 - README.md
@@ -103,7 +106,7 @@ homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -118,8 +121,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
 specification_version: 4
 summary: Fluend output plugin to forward logs to VMware Log Insight
 test_files: