fluent-plugin-vmware-loginsight 0.1.6 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c51be49fa1014357b97a164dd1d7f560a3370abdee80aa461323864a08799c00
-  data.tar.gz: c6de42ca3daa2d432a95ead1a9534ce61a09379ecd24bca06a87dbaaa56b98cd
+  metadata.gz: f7e7d17226c807f4fffdd8723c9f260730af5de29334a1b10271993b21709185
+  data.tar.gz: 768c2724b2b84ac8ae8c1e4473dbe850c37a87102aaed95d6c52fc4f06f9a0df
 SHA512:
-  metadata.gz: eaa154fcc5d89035c02f6544b5badb1258cb3af4bef6642aa54cba61791af656b98ea53bdf84ec299fd8b198e6a6ef479fbfb7ea737e0a37a477a71d818fc795
-  data.tar.gz: dc3c4176ffb0dc4765bbaa3bba6dc1cf94528daac797d8c8b6dae878585434faf96074bcd24390ad249424867fc499b3fc83360e71124f1607aa1f94c11e60b7
+  metadata.gz: 912e7dfa34b63dc9da92635603e88fea94d340ce8fb9aef99827bfd6896690a89533fe523416c2eafd34da2470016b13d23c290c74d93dc05fdb320beec78b55
+  data.tar.gz: 5998ef3e2869d13cd30dc934347b3dad6ebe2337de5678a7fb12f641b15a5b0a5a60daaa8bd9b5490ecdf58e5fb5f3529dd313ea2606126aaae8456dad023ffb

data/.github/workflows/gem-push.yml

@@ -0,0 +1,38 @@
+name: Ruby Gem
+
+on: workflow_dispatch
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+
+    - name: Publish to GPR
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:github: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+      env:
+        GEM_HOST_API_KEY: "Bearer ${{secrets.GITHUB_TOKEN}}"
+        OWNER: ${{ github.repository_owner }}
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: "${{secrets.RUBYGEMS_AUTH_TOKEN}}"

data/CHANGELOG.md

@@ -0,0 +1,63 @@
+# Changelog
+
+## v1.0.0 - April 19, 2021
+
+* Update plugin structure to use Fluentd 1.x syntax
+
+## v0.1.11 - March 31, 2021
+
+* Add an option to rename Loginsight fields. This option could be used to rename certain fields that are reserved by Loginsight
+
+## v0.1.10 - May 13, 2020
+
+* Escape `@` char from Loginsight field
+
+## v0.1.9 - May 07, 2020
+
+* No change
+
+## v0.1.8 - May 06, 2020 yanked, Not available
+
+* Parameterize and add an option to shorten Loginsight field names
+
+## v0.1.7 - December 10, 2019
+
+* Fix basic authentication #8
+
+## v0.1.6 - September 13, 2019
+
+* For immutable log fields, use a copy to utf encode. This should fix 'can't modify frozen String' error in #5
+
+## v0.1.5 - October 22, 2018
+
+* Add option to display debug logs for http connection, default false
+* Flatten Lists/Arrays for LI fields
+* Convert LI field value to String to ensure no utf encoding errors
+* Update help doc/examples with sample use of @log_text_keys and @http_conn_debug options
+
+## v0.1.4 - October 17, 2018
+
+* Add option to specify a list of keys that plugin should treat as log messages and forward them as text to Loginsight. Plugin should not flatten these fields
+* If user specifies flatten_hashes option as false, plugin should try to add record key/values as is
+
+## v0.1.3 - September 13, 2018
+
+* Reorder namespace and name fields to be shorten
+
+## v0.1.2 - September 10, 2018
+
+* Republished yanked gem
+
+## v0.1.1 - August 30, 2018 yanked, Not available
+
+* Send log messages in batches, add max_batch_size parameter
+* Shorten common kubernetes Loginsight field names
+* Convert time to milliseconds
+
+
+## 0.1.0 - August 30, 2018
+
+### Initial release
+
+* Fluentd output plugin to push logs to VMware Log Insight
+

data/README.md

@@ -1,5 +1,7 @@
 # fluent-plugin-vmware-loginsight
 
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight.svg)](https://badge.fury.io/rb/fluent-plugin-vmware-loginsight)
+
 ## Overview
 output plugin to do forward logs to VMware Log Insight
 
@@ -28,39 +30,61 @@ $ bundle
 ## Usage
 
 ```
+# Collect all container logs
 <source>
   @type tail
+  @id in_tail_container_logs
   path /var/log/containers/*.log
+  # One could exclude certain logs like:
+  #exclude_path ["/var/log/containers/log-collector*.log"]
   pos_file /var/log/fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  # Set this watcher to false if you have many files to tail
+  enable_stat_watcher false
+  refresh_interval 5
+  tag kubernetes.*
+  <parse>
+    @type json
+    time_key time
+    keep_time_key true
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+  </parse>
 </source>
 
-# Kubernetes metadata filter that tags additional meta data for each event
-<filter kubernetes.var.log.containers.**.log>
+# Kubernetes metadata filter that tags additional meta data for each container event
+<filter kubernetes.**>
   @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
 </filter>
 
+# Match everything
 <match **>
   @type vmware_loginsight
+  @id out_vmw_li_all_container_logs
   scheme https
   ssl_verify true
-# Loginsight host: One may use IP address or cname
-# host X.X.X.X
-  host my-loginsight.mycompany.com
-  port 9000
-  path api/v1/events/ingest
+  # Loginsight host: One may use IP address or cname
+  #host X.X.X.X
+  host MY_LOGINSIGHT_HOST
+  port 9543
   agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-  http_method post
-  serializer json
-  rate_limit_msec 0
-  raise_on_error false
-  include_tag_key true
-  tag_key tag
+  # Keys from log event whose values should be added as log message/text to
+  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  log_text_keys ["log","msg","message"]
+  # Use this flag if you want to enable http debug logs
+  http_conn_debug false
 </match>
 ```
+
+For more examples look at [examples](./examples/)
+
 ### Configuration options
 
 ```
@@ -104,6 +128,9 @@ request_timeout, :time, :default => 5
 # If set, enables debug logs for http connection
 http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
+# Number of bytes per post request
+max_batch_size, :integer, :default => 512000
+
 # Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
 rate_limit_msec, :integer, :default => 0
 
@@ -125,9 +152,35 @@ flatten_hashes, :bool, :default => true :: Valid Value: true | false
 
 # Seperator to use for joining flattened keys
 flatten_hashes_separator, :string, :default => "_"
-```
 
-For more examples look at [examples](./examples/)
+# Rename fields names
+config_param :rename_fields, :hash, default: {"source" => "log_source"}, value_type: :string
+
+# Keys from log event to rewrite
+# for instance from 'kubernetes_namespace' to 'k8s_namespace'
+# tags will be rewritten with substring substitution
+# and applied in the order present in the hash
+# (Hashes enumerate their values in the order that the
+# corresponding keys were inserted
+# see https://ruby-doc.org/core-2.2.2/Hash.html)
+# example config:
+# shorten_keys {
+#    "__":"_",
+#    "container_":"",
+#    "kubernetes_":"k8s_",
+#    "labels_":"",
+# }
+shorten_keys, :hash, value_type: :string, default:
+        {
+            'kubernetes_':'k8s_',
+            'namespace':'ns',
+            'labels_':'',
+            '_name':'',
+            '_hash':'',
+            'container_':''
+        }
+
+```
 
 ## Contributing
 

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/examples/fluent.conf

@@ -8,18 +8,25 @@
 # 
 # SPDX-License-Identifier: MIT
 
+# Sample Fluentd config, edit as per your needs.
+# https://github.com/fluent/fluentd-kubernetes-daemonset/tree/master/templates/conf has some good fluentd config examples
 
+# System level configs
 <system>
   log_level info
 </system>
 
 # Prevent fluentd from handling records containing its own logs to handle cycles.
-<match fluent.**>
-  @type null
-</match>
+<label @FLUENT_LOG>
+  <match fluent.**>
+    @type null
+  </match>
+</label>
 
+# Collect all journal logs
 <source>
   @type systemd
+  @id in_systemd_logs
   path /run/log/journal
   # Can filter logs if we want, e.g.
   #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
@@ -33,76 +40,121 @@
   strip_underscores true
 </source>
 
+# Collect all container logs
 <source>
   @type tail
+  @id in_tail_container_logs
   path /var/log/containers/*.log
   # One could exclude certain logs like:
-  #  exclude_path ["/var/log/containers/log-collector*.log"]
+  #exclude_path ["/var/log/containers/log-collector*.log"]
   pos_file /var/log/fluentd-docker.pos
-  time_format %Y-%m-%dT%H:%M:%S
-  tag kubernetes.*
-  format json
   read_from_head true
+  # Set this watcher to false if you have many files to tail
+  enable_stat_watcher false
+  refresh_interval 5
+  tag kubernetes.*
+  <parse>
+    @type json
+    time_key time
+    keep_time_key true
+    time_format %Y-%m-%dT%H:%M:%S.%NZ
+  </parse>
 </source>
 
-
-# Sample rule for services that generate java like stack trace
-#<source>
-#  @type tail
-#  path /var/log/containers/javaapp**.log
-#  pos_file /var/log/fluentd-dockerlog.pos
-#  time_format %b %d %H:%M:%S
-#  tag kubernetes.*
-#  format multiline
-#  format_firstline /\d{4}-\d{1,2}-\d{1,2}/
-#  format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
-#  read_from_head true
-#</source>
-
-# Kubernetes metadata filter that tags additional meta data for each event
-<filter kubernetes.var.log.containers.**.log>
+# Kubernetes metadata filter that tags additional meta data for each container event
+<filter kubernetes.**>
   @type kubernetes_metadata
+  @id filter_kube_metadata
+  kubernetes_url "#{ENV['FLUENT_FILTER_KUBERNETES_URL'] || 'https://' + ENV.fetch('KUBERNETES_SERVICE_HOST') + ':' + ENV.fetch('KUBERNETES_SERVICE_PORT') + '/api'}"
+  verify_ssl "#{ENV['KUBERNETES_VERIFY_SSL'] || true}"
+  ca_file "#{ENV['KUBERNETES_CA_FILE']}"
+  skip_labels "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_LABELS'] || 'false'}"
+  skip_container_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_CONTAINER_METADATA'] || 'false'}"
+  skip_master_url "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_MASTER_URL'] || 'false'}"
+  skip_namespace_metadata "#{ENV['FLUENT_KUBERNETES_METADATA_SKIP_NAMESPACE_METADATA'] || 'false'}"
 </filter>
 
-# If we want to transform events we could use:
-#<filter **>
-#  @type record_transformer
-#  enable_ruby
-#  auto_typecast
-#  <record>
-#    hostname "#{Socket.gethostname}"
-#    mykey ${["message"=>record.to_json]}
-#  </record>
-#</filter>
+# Prefix the tag by namespace. This would make it easy to match logs by namespaces
+<match kubernetes.**>
+  @type rewrite_tag_filter
+  <rule>
+    key $.kubernetes.namespace_name
+    pattern ^(.+)$
+    tag $1.${tag}
+  </rule>
+</match>
 
-<match fluent.**>
-  @type null
+# Collect all kube apiserver audit logs
+<source>
+  @type tail
+  @id in_tail_kube_audit_logs
+  # audit log path of kube-apiserver
+  path "/var/log/kube-audit/audit.log"
+  pos_file /var/log/kube-audit.pos
+  tag kube-audit
+  <parse>
+    @type json
+    time_key timestamp
+    keep_time_key false
+    time_format %Y-%m-%dT%H:%M:%SZ
+  </parse>
+</source>
+
+# Loginsight doesn't support ingesting `source` as a field name, get rid of it
+<filter kube-audit>
+  @type record_transformer
+  @id filter_kube_audit_logs
+  enable_ruby
+  remove_keys source
+  <record>
+    log ${record}
+  </record>
+</filter>
+
+# You can catch and match logs by namespace
+<match my-namespace-one.** my-namespace-two.**>
+  @type vmware_loginsight
+  @id out_vmw_li_my_namespace_logs
+  scheme http
+  ssl_verify false
+  # Loginsight host: One may use IP address or cname
+  #host X.X.X.X
+  host MY_LOGINSIGHT_HOST
+  port 9000
+  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+  # Keys from log event whose values should be added as log message/text to
+  # Loginsight. Note these key/value pairs  won't be added as metadata/fields
+  log_text_keys ["log","msg","message"]
+  # Use this flag if you want to enable http debug logs
+  http_conn_debug false
 </match>
 
+# Match everything else
 <match **>
   @type copy
   <store>
     @type vmware_loginsight
+    @id out_vmw_li_all_container_logs
     scheme https
     ssl_verify true
-#   Loginsight host: One may use IP address or cname
-#   host X.X.X.X
-    host my-loginsight.mycompany.com
-    port 9000
-    path api/v1/events/ingest
+    # Loginsight host: One may use IP address or cname
+    #host X.X.X.X
+    host MY_LOGINSIGHT_HOST
+    port 9543
     agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
-    http_method post
-    serializer json
-    rate_limit_msec 0
-    raise_on_error false
+    # Keys from log event whose values should be added as log message/text to
+    # Loginsight. Note these key/value pairs  won't be added as metadata/fields
     log_text_keys ["log","msg","message"]
-    include_tag_key true
-    tag_key tag
+    # Use this flag if you want to enable http debug logs
+    http_conn_debug false
   </store>
-# copy plugin supports sending/copying logs to multiple plugins
-# One may choose to send them to multiple LIs
-# Or one may want send a copy to stdout for debugging
-#  <store>
-#    @type stdout
-#  </store>
+  # copy plugin supports sending/copying logs to multiple plugins
+  # One may choose to send them to multiple LIs
+  # Or one may want send a copy to stdout for debugging
+  # Please note, if you use stdout along with LI, catch the logger's log to make
+  # sure they're not cyclic
+  #<store>
+  #  @type stdout
+  #</store>
 </match>
+