fluent-plugin-vmware-loginsight 0.1.10 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18aba45331aaa3aa482ac466fa77747daec97df8e9d31fa231493df9a46c83de
-  data.tar.gz: 987665db9f586d6a4bde5921a68a9c18555b70000f16e900b29b0480e381593b
+  metadata.gz: d8c6cb1363b6364811d6c9d9c1c80067c46ed222eaa2a70034a7f2cd69b2b043
+  data.tar.gz: cad2d46faa95261265dc7c8aa2119ce8a389ca641c8ca789079a34a90d32fb19
 SHA512:
-  metadata.gz: 68448d50ae0dc8a84bf4f5c17d8a91053b22d240df04c73f2199043a994a95ac7c89ba34281ee5ff623d241d457ad786f8009261c3843ff12a53cdd818b9788e
-  data.tar.gz: a37769610e800178e2e0c9ea19dd3f58b91ff7b4ff38f9b896c8273d651b949dd52625e3bc310a11dc38e732762053199cccc13f2d8e94beb4b2f298f2c22ed0
+  metadata.gz: 320f33af67ad7102ea3212f217b1bcfa23c0a27f37f0b4f4873a66fd716ec4556431b0e14005a732541d985b980de1cb44b7fbde5a7d3dac980944593156ea1e
+  data.tar.gz: 1c172ba0c3ea4af9ac2a3e0c28037ff589163dc4b5ff82fc3c70efb7829deca8f0c2c5d4bd9a580bafc104e670e44ad7c322625e60a937ca4b2cc54ebeabd6b7

data/.github/workflows/gem-push.yml

@@ -0,0 +1,38 @@
+name: Ruby Gem
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Publish to GPR
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+      env:
+        GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+        OWNER: ${{ github.repository_owner }}
+
+#     - name: Publish to RubyGems
+#       run: |
+#         mkdir -p $HOME/.gem
+#         touch $HOME/.gem/credentials
+#         chmod 0600 $HOME/.gem/credentials
+#         printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+#         gem build *.gemspec
+#         gem push *.gem
+#       env:
+#         GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/CHANGELOG.md

@@ -0,0 +1,63 @@
+# Changelog
+
+## v1.0.0 - April 20, 2021
+
+* Update plugin structure to use Fluentd 1.x syntax
+
+## v0.1.11 - March 31, 2021
+
+* Add an option to rename Loginsight fields. This option could be used to rename certain fields that are reserved by Loginsight
+
+## v0.1.10 - May 13, 2020
+
+* Escape `@` char from Loginsight field
+
+## v0.1.9 - May 07, 2020
+
+* No change
+
+## v0.1.8 - May 06, 2020 yanked, Not available
+
+* Parameterize and add an option to shorten Loginsight field names
+
+## v0.1.7 - December 10, 2019
+
+* Fix basic authentication #8
+
+## v0.1.6 - September 13, 2019
+
+* For immutable log fields, use a copy to utf encode. This should fix 'can't modify frozen String' error in #5
+
+## v0.1.5 - October 22, 2018
+
+* Add option to display debug logs for http connection, default false
+* Flatten Lists/Arrays for LI fields
+* Convert LI field value to String to ensure no utf encoding errors
+* Update help doc/examples with sample use of @log_text_keys and @http_conn_debug options
+
+## v0.1.4 - October 17, 2018
+
+* Add option to specify a list of keys that plugin should treat as log messages and forward them as text to Loginsight. Plugin should not flatten these fields
+* If user specifies flatten_hashes option as false, plugin should try to add record key/values as is
+
+## v0.1.3 - September 13, 2018
+
+* Reorder namespace and name fields to be shorten
+
+## v0.1.2 - September 10, 2018
+
+* Republished yanked gem
+
+## v0.1.1 - August 30, 2018 yanked, Not available
+
+* Send log messages in batches, add max_batch_size parameter
+* Shorten common kubernetes Loginsight field names
+* Convert time to milliseconds
+
+
+## 0.1.0 - August 30, 2018
+
+### Initial release
+
+* Fluentd output plugin to push logs to VMware Log Insight
+

data/Dockerfile

@@ -0,0 +1,68 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2019 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
+# (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
+#
+# Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
+# a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
+# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
+FROM photon:4.0-20220722
+
+USER root
+
+RUN buildDeps="\
+    binutils linux-api-headers glibc-devel \
+    make gcc gmp-devel libffi-devel \
+    tar bzip2 sed gawk" \
+    #
+    # Distro sync and install build dependencies
+    && tdnf distro-sync --refresh -y \
+    # Toybox conflicts with bzip2. The latter is needed to unpack libjemalloc
+    && tdnf remove -y toybox \
+    && tdnf install -y $buildDeps ruby \
+    #
+    # These are not required but are used if available
+    && gem install oj -v 3.3.10 \
+    && gem install json -v 2.2.0 \
+    && gem install async-http -v 0.46.3 \
+    #
+    # Install fluentd
+    && gem install --norc --no-document fluentd -v 1.14.2 \
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    #
+    # Install Log Insight plugin
+    && gem install --norc --no-document -v 0.1.5 fluent-plugin-vmware-loginsight \
+    #
+    # Install jemalloc 4.5.0
+    && curl -L --output /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
+    && tar -C /tmp/ -xjvf /tmp/jemalloc-4.5.0.tar.bz2 \
+    && cd /tmp/jemalloc-4.5.0 \
+    && ./configure && make \
+    && mv lib/libjemalloc.so.2 /usr/lib \
+    && cd / \
+    #
+    # Cleanup to reduce image size
+    && rm -rf /tmp/jemalloc-4.5.0* \
+    && tdnf remove -y $buildDeps \
+    && tdnf clean all \
+    && gem sources --clear-all \
+    && gem cleanup
+
+# Create default fluent.conf
+RUN fluentd --setup
+
+# Make sure fluentd picks jemalloc
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+# Standard fluentd ports
+EXPOSE 24224 5140
+
+ENTRYPOINT ["/usr/bin/fluentd"]

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-vmware-loginsight
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight.svg)](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
+
 ## Overview
 output plugin to do forward logs to VMware Log Insight
 
@@ -28,39 +30,63 @@ $ bundle
 ## Usage
 
 ```
+# Collect all container logs
 <source>
   @type tail
+  @id in_tail_container_logs
   path /var/log/containers/*.log
+  # One could exclude certain logs like:
+  #exclude_path ["/var/log/containers/log-collector*.log"]
   pos_file /var/log/fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  # Set this watcher to false if you have many files to tail
+  enable_stat_watcher false
+  refresh_interval 5
+  tag kubernetes.*
+  # Open below line if you need have filename as tag field (now without prefix kubernetes.)
+  # path_key tag
+  <parse>
+    @type json
+    time_key time
+    keep_time_key true
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+  </parse>
 </source>
 
-# Kubernetes metadata filter that tags additional meta data for each event
-<filter kubernetes.var.log.containers.**.log>
+# Kubernetes metadata filter that tags additional meta data for each container event
+<filter kubernetes.**>
   @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
 </filter>
 
+# Match everything
 <match **>
   @type vmware_loginsight
+  @id out_vmw_li_all_container_logs
   scheme https
   ssl_verify true
-# Loginsight host: One may use IP address or cname
-# host X.X.X.X
-  host my-loginsight.mycompany.com
-  port 9000
-  path api/v1/events/ingest
+  # Loginsight host: One may use IP address or cname
+  #host X.X.X.X
+  host MY_LOGINSIGHT_HOST
+  port 9543
   agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-  http_method post
-  serializer json
-  rate_limit_msec 0
-  raise_on_error false
-  include_tag_key true
-  tag_key tag
+  # Keys from log event whose values should be added as log message/text to
+  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  log_text_keys ["log","msg","message"]
+  # Use this flag if you want to enable http debug logs
+  http_conn_debug false
 </match>
 ```
+
+For more examples look at [examples](./examples/)
+
 ### Configuration options
 
 ```
@@ -104,18 +130,15 @@ request_timeout, :time, :default => 5
 # If set, enables debug logs for http connection
 http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
+# Number of bytes per post request
+max_batch_size, :integer, :default => 512000
+
 # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
 rate_limit_msec, :integer, :default => 0
 
 # Raise errors that were rescued during HTTP requests?
 raise_on_error, :bool, :default => false :: Valid Value: true | false 
 
-# Include tag key as log event?
-include_tag_key, :bool, :default => true :: Valid Value: true | false 
-
-# Metadata key that identifies Fluentd tags
-tag_key, :string, :default => 'tag'
-
 # Keys from log event whose values should be added as log message/text to loginsight.
 # These key/value pairs won't expanded/flattened and won't be added as metadata/fields.
 log_text_keys, :array, :default => ["log", "message", "msg"] :: Valid Value: Array of strings
@@ -125,9 +148,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
 
 # Seperator to use for joining flattened keys
 flatten_hashes_separator, :string, :default => "_"
-```
 
-For more examples look at [examples](./examples/)
+# Rename fields names
+config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
+
+# Keys from log event to rewrite
+# for instance from 'kubernetes_namespace' to 'k8s_namespace'
+# tags will be rewritten with substring substitution
+# and applied in the order present in the hash
+# (Hashes enumerate their values in the order that the
+# corresponding keys were inserted
+# see https://ruby-doc.org/core-2.2.2/Hash.html)
+# example config:
+# shorten_keys {
+#    "__":"_",
+#    "container_":"",
+#    "kubernetes_":"k8s_",
+#    "labels_":"",
+# }
+shorten_keys, :hash, value_type: :string, default:
+        {
+            'kubernetes_':'k8s_',
+            'namespace':'ns',
+            'labels_':'',
+            '_name':'',
+            '_hash':'',
+            'container_':''
+        }
+
+```
 
 ## Contributing
 

data/VERSION

@@ -0,0 +1 @@
+1.3.0

data/examples/fluent.conf

@@ -8,18 +8,25 @@
 # 
 # SPDX-License-Identifier: MIT
 
+# Sample Fluentd config, edit as per your needs.
+# https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
 
+# System level configs
 <system>
   log_level info
 </system>
 
 # Prevent fluentd from handling records containing its own logs to handle cycles.
-<match fluent.**>
-  @type null
-</match>
+<label @FLUENT_LOG>
+  <match fluent.**>
+    @type null
+  </match>
+</label>
 
+# Collect all journal logs
 <source>
   @type systemd
+  @id in_systemd_logs
   path /run/log/journal
   # Can filter logs if we want, e.g.
   #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
@@ -33,76 +40,121 @@
   strip_underscores true
 </source>
 
+# Collect all container logs
 <source>
   @type tail
+  @id in_tail_container_logs
   path /var/log/containers/*.log
   # One could exclude certain logs like:
-  #  exclude_path ["/var/log/containers/log-collector*.log"]
+  #exclude_path ["/var/log/containers/log-collector*.log"]
   pos_file /var/log/fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  # Set this watcher to false if you have many files to tail
+  enable_stat_watcher false
+  refresh_interval 5
+  tag kubernetes.*
+  <parse>
+    @type json
+    time_key time
+    keep_time_key true
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+  </parse>
 </source>
 
-
-# Sample rule for services that generate java like stack trace
-#<source>
-#  @type tail
-#  path /var/log/containers/javaapp**.log
-#  pos_file /var/log/fluentd-dockerlog.pos
-#  time_format %b %d %H:%M:%S
-#  tag kubernetes.*
-#  format multiline
-#  format_firstline /\d{4}-\d{1,2}-\d{1,2}/
-#  format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
-#  read_from_head true
-#</source>
-
-# Kubernetes metadata filter that tags additional meta data for each event
-<filter kubernetes.var.log.containers.**.log>
+# Kubernetes metadata filter that tags additional meta data for each container event
+<filter kubernetes.**>
   @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
 </filter>
 
-# If we want to transform events we could use:
-#<filter **>
-#  @type record_transformer
-#  enable_ruby
-#  auto_typecast
-#  <record>
-#    hostname "#{Socket.gethostname}"
-#    mykey ${["message"=>record.to_json]}
-#  </record>
-#</filter>
+# Prefix the tag by namespace. This would make it easy to match logs by namespaces
+<match kubernetes.**>
+  @type rewrite_tag_filter
+  <rule>
+    key $.kubernetes.namespace_name
+    pattern ^(.+)$
+    tag $1.${tag}
+  </rule>
+</match>
 
-<match fluent.**>
-  @type null
+# Collect all kube apiserver audit logs
+<source>
+  @type tail
+  @id in_tail_kube_audit_logs
+  # audit log path of kube-apiserver
+  path "/var/log/kube-audit/audit.log"
+  pos_file /var/log/kube-audit.pos
+  tag kube-audit
+  <parse>
+    @type json
+    time_key timestamp
+    keep_time_key false
+    time_format %Y-%m-%dT%H:%M:%SZ
+  </parse>
+</source>
+
+# Loginsight doesn't support ingesting `source` as a field name, get rid of it
+<filter kube-audit>
+  @type record_transformer
+  @id filter_kube_audit_logs
+  enable_ruby
+  remove_keys source
+  <record>
+    log ${record}
+  </record>
+</filter>
+
+# You can catch and match logs by namespace
+<match my-namespace-one.** my-namespace-two.**>
+  @type vmware_loginsight
+  @id out_vmw_li_my_namespace_logs
+  scheme http
+  ssl_verify false
+  # Loginsight host: One may use IP address or cname
+  #host X.X.X.X
+  host MY_LOGINSIGHT_HOST
+  port 9000
+  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+  # Keys from log event whose values should be added as log message/text to
+  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  log_text_keys ["log","msg","message"]
+  # Use this flag if you want to enable http debug logs
+  http_conn_debug false
 </match>
 
+# Match everything else
 <match **>
   @type copy
   <store>
     @type vmware_loginsight
+    @id out_vmw_li_all_container_logs
     scheme https
     ssl_verify true
-#   Loginsight host: One may use IP address or cname
-#   host X.X.X.X
-    host my-loginsight.mycompany.com
-    port 9000
-    path api/v1/events/ingest
+    # Loginsight host: One may use IP address or cname
+    #host X.X.X.X
+    host MY_LOGINSIGHT_HOST
+    port 9543
     agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-    http_method post
-    serializer json
-    rate_limit_msec 0
-    raise_on_error false
+    # Keys from log event whose values should be added as log message/text to
+    # Loginsight. Note these key/value pairs  won't be added as metadata/fields
     log_text_keys ["log","msg","message"]
-    include_tag_key true
-    tag_key tag
+    # Use this flag if you want to enable http debug logs
+    http_conn_debug false
   </store>
-# copy plugin supports sending/copying logs to multiple plugins
-# One may choose to send them to multiple LIs
-# Or one may want send a copy to stdout for debugging
-#  <store>
-#    @type stdout
-#  </store>
+  # copy plugin supports sending/copying logs to multiple plugins
+  # One may choose to send them to multiple LIs
+  # Or one may want send a copy to stdout for debugging
+  # Please note, if you use stdout along with LI, catch the logger's log to make
+  # sure they're not cyclic
+  #<store>
+  #  @type stdout
+  #</store>
 </match>
+

data/examples/fluentd-vrli-plugin-debian.dockerfile

@@ -8,29 +8,41 @@
 # 
 # SPDX-License-Identifier: MIT
 
-# Builds a debian-based image that contains fluentd, fluent-plugin-vmware-loginsight, fluent-plugin-kubernetes_metadata_filter
-# and fluent-plugin-systemd.
+
+# Sample Dockerfile to use as log collector
+# Builds a debian-based fluentd image that has fluent-plugin-kubernetes_metadata_filter,
+# fluent-plugin-rewrite-tag-filter, fluent-plugin-systemd and
+# fluent-plugin-vmware-loginsight gem installed.
 #
-# The image is preconfigured with the fluent.conf from the examples dir. For more details see
+# This image will get preconfigured with the fluent.conf if avaialble at the
+# same dir level. For fluentd config example, see
 # https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
-FROM fluent/fluentd:v0.14.15-debian-onbuild
-# Above image expects the loginsight plugin vmware_loginsight to be available under ./plugins/vmware_loginsight.rb
-# and fluentd config under ./fluent.conf by default
 
+# This base image is built from https://github.com/fluent/fluentd-kubernetes-daemonset
+FROM fluent/fluentd:v1.11-debian-1
+
+# Use root account to use apt
 USER root
 
-RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
+# You can install your plugins here
+RUN buildDeps="sudo make gcc g++ libc-dev" \
  && apt-get update \
  && apt-get install -y --no-install-recommends $buildDeps \
  && sudo gem install \
-        fluent-plugin-systemd \
-        fluent-plugin-kubernetes_metadata_filter \
-        fluent-plugin-vmware-loginsight \
+        fluent-plugin-kubernetes_metadata_filter:2.4.6 \
+        fluent-plugin-rewrite-tag-filter:2.3.0 \
+        fluent-plugin-systemd:1.0.2 \
+        fluent-plugin-vmware-loginsight:0.1.10 \
  && sudo gem sources --clear-all \
  && SUDO_FORCE_REMOVE=yes \
     apt-get purge -y --auto-remove \
                   -o APT::AutoRemove::RecommendsImportant=false \
                   $buildDeps \
  && rm -rf /var/lib/apt/lists/* \
-           /home/fluent/.gem/ruby/2.3.0/cache/*.gem \
-           /home/root/.gem/ruby/2.3.0/cache/*.gem
+ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
+
+#  You can install the LI plugin using a gem or if you want to test your
+#  changes to plugin, you may add the .rb directly under `plugins` dir, then
+#  you don't need to install the gem
+COPY plugins /fluentd/plugins/
+