fluent-plugin-vmware-loginsight 0.1.0 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7677f5c4289ba3ecb32f9aa6ddc35c6b8f70f4b6
-  data.tar.gz: 73023509bb66d43595176f3625256009e4187cac
+SHA256:
+  metadata.gz: c51be49fa1014357b97a164dd1d7f560a3370abdee80aa461323864a08799c00
+  data.tar.gz: c6de42ca3daa2d432a95ead1a9534ce61a09379ecd24bca06a87dbaaa56b98cd
 SHA512:
-  metadata.gz: d3662938668a3307722b0ac604186cfc7a1b397c18d16968cd669360fd2a5c148c4d7904547d9a7258abda2818cbce2a8ee1505206259120e9efb0955f109bcb
-  data.tar.gz: 12c29d1640c720f3b380475d3fd2f9e368817d3b5e794ad14960120c0ba6d70ab789764b64737eb28dcbdafed382af331a700499b0f18400b3d93f1f50ae4ec3
+  metadata.gz: eaa154fcc5d89035c02f6544b5badb1258cb3af4bef6642aa54cba61791af656b98ea53bdf84ec299fd8b198e6a6ef479fbfb7ea737e0a37a477a71d818fc795
+  data.tar.gz: dc3c4176ffb0dc4765bbaa3bba6dc1cf94528daac797d8c8b6dae878585434faf96074bcd24390ad249424867fc499b3fc83360e71124f1607aa1f94c11e60b7

data/CONTRIBUTING.md

@@ -2,9 +2,7 @@
 
 # Contributing to fluent-plugin-vmware-loginsight
 
-The fluent-plugin-vmware-loginsight project team welcomes contributions from the community. If you wish to contribute code and you have not
-signed our contributor license agreement (CLA), our bot will update the issue when you open a Pull Request. For any
-questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq).
+The fluent-plugin-vmware-loginsight project team welcomes contributions from the community. Before you start working with fluent-plugin-vmware-loginsight, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch.
 
 ## Community
 

data/Gemfile

@@ -0,0 +1,13 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,15 @@
+Fluentd plugin for VMware Log Insight
+
+Copyright 2018 VMware, Inc.  All rights reserved				
+
+The MIT license (the ìLicenseî) set forth below applies to all parts of the Fluentd plugin for VMware Log Insight project.  You may not use this file except in compliance with the License.†
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -25,26 +25,121 @@ And then execute:
 $ bundle
 ```
 
-## Configuration
-
-You can generate configuration template:
+## Usage
 
 ```
-$ fluent-plugin-config-format output vmware-loginsight
+<source>
+  @type tail
+  path /var/log/containers/*.log
+  pos_file /var/log/fluentd-docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag kubernetes.*
+  format json
+  read_from_head true
+</source>
+
+# Kubernetes metadata filter that tags additional meta data for each event
+<filter kubernetes.var.log.containers.**.log>
+  @type kubernetes_metadata
+</filter>
+
+<match **>
+  @type vmware_loginsight
+  scheme https
+  ssl_verify true
+# Loginsight host: One may use IP address or cname
+# host X.X.X.X
+  host my-loginsight.mycompany.com
+  port 9000
+  path api/v1/events/ingest
+  agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+  http_method post
+  serializer json
+  rate_limit_msec 0
+  raise_on_error false
+  include_tag_key true
+  tag_key tag
+</match>
+```
+### Configuration options
+
 ```
+scheme, :string, :default => 'http' :: Valid Values: http/https 
+
+# Loginsight Host ex. localhost
+host, :string,  :default => 'localhost' :: Valid Values: loginsight_url | loginsight_ip
+
+# Loginsight port ex. 9000
+port, :integer, :default => 80
+
+# Loginsight ingestion api path ex. 'api/v1/events/ingest'
+path, :string, :default => 'api/v1/events/ingest'
+
+# agent_id generated by your LI
+agent_id, :string, :default => '0'
+
+# Credentials if used
+username, :string, :default => nil
+password, :string, :default => nil,
+
+# Authentication
+authentication, :string, :default => nil :: Valid Value: nil | basic
+
+# SSL verification flag
+ssl_verify, :bool, :default => true :: Valid Value: true | false
+# CA Cert filep
+ca_file, :string, :default => nil
+
+# HTTP method
+http_method, :string, :default => :post :: Valid Value: post
+
+# Serialization
+serializer, :string, :default => :json :: Valid Value: json
 
-You can copy and paste generated documents here.
+# Number of retries
+request_retries, :integer, :default => 3
+# http connection ttl for each request
+request_timeout, :time, :default => 5
 
-## Documentation
+# If set, enables debug logs for http connection
+http_conn_debug, :bool, :default => false :: Valid Value: true | false
 
-## Releases & Major Branches
+# Simple rate limiting: ignore any records within `rate_limit_msec` since the last one
+rate_limit_msec, :integer, :default => 0
+
+# Raise errors that were rescued during HTTP requests?
+raise_on_error, :bool, :default => false :: Valid Value: true | false 
+
+# Include tag key as log event?
+include_tag_key, :bool, :default => true :: Valid Value: true | false 
+
+# Metadata key that identifies Fluentd tags
+tag_key, :string, :default => 'tag'
+
+# Keys from log event whose values should be added as log message/text to loginsight.
+# These key/value pairs won't expanded/flattened and won't be added as metadata/fields.
+log_text_keys, :array, :default => ["log", "message", "msg"] :: Valid Value: Array of strings
+
+# Flatten hashes to create one key/val pair w/o losing log data
+flatten_hashes, :bool, :default => true :: Valid Value: true | false
+
+# Seperator to use for joining flattened keys
+flatten_hashes_separator, :string, :default => "_"
+```
+
+For more examples look at [examples](./examples/)
 
 ## Contributing
 
-The fluent-plugin-vmware-loginsight project team welcomes contributions from the community. If you wish to contribute code and you have not
-signed our contributor license agreement (CLA), our bot will update the issue when you open a Pull Request. For any
-questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). For more detailed information,
-refer to [CONTRIBUTING.md](CONTRIBUTING.md).
+The fluent-plugin-vmware-loginsight project team welcomes contributions from the community. Before you start working with fluent-plugin-vmware-loginsight, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch. For more detailed information, refer to [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## License
+Fluentd plugin for VMware Log Insight
+
+Copyright 2018 VMware, Inc. All Rights Reserved. 
+
+This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+
+This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
 
+SPDX-License-Identifier: MIT

data/Rakefile

@@ -0,0 +1,24 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+
+require "bundler"
+Bundler::GemHelper.install_tasks
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs.push("lib", "test")
+  t.test_files = FileList["test/**/test_*.rb"]
+  t.verbose = true
+  t.warning = true
+end
+
+task default: [:test]

data/examples/fluent.conf

@@ -0,0 +1,108 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+
+<system>
+  log_level info
+</system>
+
+# Prevent fluentd from handling records containing its own logs to handle cycles.
+<match fluent.**>
+  @type null
+</match>
+
+<source>
+  @type systemd
+  path /run/log/journal
+  # Can filter logs if we want, e.g.
+  #filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+  <storage>
+    @type local
+    persistent true
+    path /var/log/fluentd-systemdlog.pos
+  </storage>
+  tag systemdlog
+  read_from_head true
+  strip_underscores true
+</source>
+
+<source>
+  @type tail
+  path /var/log/containers/*.log
+  # One could exclude certain logs like:
+  #  exclude_path ["/var/log/containers/log-collector*.log"]
+  pos_file /var/log/fluentd-docker.pos
+  time_format %Y-%m-%dT%H:%M:%S
+  tag kubernetes.*
+  format json
+  read_from_head true
+</source>
+
+
+# Sample rule for services that generate java like stack trace
+#<source>
+#  @type tail
+#  path /var/log/containers/javaapp**.log
+#  pos_file /var/log/fluentd-dockerlog.pos
+#  time_format %b %d %H:%M:%S
+#  tag kubernetes.*
+#  format multiline
+#  format_firstline /\d{4}-\d{1,2}-\d{1,2}/
+#  format1 /^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2}) \[(?<thread>.*)\] (?<level>[^\s]+)(?<message>.*)/
+#  read_from_head true
+#</source>
+
+# Kubernetes metadata filter that tags additional meta data for each event
+<filter kubernetes.var.log.containers.**.log>
+  @type kubernetes_metadata
+</filter>
+
+# If we want to transform events we could use:
+#<filter **>
+#  @type record_transformer
+#  enable_ruby
+#  auto_typecast
+#  <record>
+#    hostname "#{Socket.gethostname}"
+#    mykey ${["message"=>record.to_json]}
+#  </record>
+#</filter>
+
+<match fluent.**>
+  @type null
+</match>
+
+<match **>
+  @type copy
+  <store>
+    @type vmware_loginsight
+    scheme https
+    ssl_verify true
+#   Loginsight host: One may use IP address or cname
+#   host X.X.X.X
+    host my-loginsight.mycompany.com
+    port 9000
+    path api/v1/events/ingest
+    agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+    http_method post
+    serializer json
+    rate_limit_msec 0
+    raise_on_error false
+    log_text_keys ["log","msg","message"]
+    include_tag_key true
+    tag_key tag
+  </store>
+# copy plugin supports sending/copying logs to multiple plugins
+# One may choose to send them to multiple LIs
+# Or one may want send a copy to stdout for debugging
+#  <store>
+#    @type stdout
+#  </store>
+</match>

data/examples/fluentd-vrli-plugin-debian.dockerfile

@@ -0,0 +1,36 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018-2019 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# Builds a debian-based image that contains fluentd, fluent-plugin-vmware-loginsight, fluent-plugin-kubernetes_metadata_filter
+# and fluent-plugin-systemd.
+#
+# The image is preconfigured with the fluent.conf from the examples dir. For more details see
+# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
+FROM fluent/fluentd:v0.14.15-debian-onbuild
+# Above image expects the loginsight plugin vmware_loginsight to be available under ./plugins/vmware_loginsight.rb
+# and fluentd config under ./fluent.conf by default
+
+USER root
+
+RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
+ && apt-get update \
+ && apt-get install -y --no-install-recommends $buildDeps \
+ && sudo gem install \
+        fluent-plugin-systemd \
+        fluent-plugin-kubernetes_metadata_filter \
+        fluent-plugin-vmware-loginsight \
+ && sudo gem sources --clear-all \
+ && SUDO_FORCE_REMOVE=yes \
+    apt-get purge -y --auto-remove \
+                  -o APT::AutoRemove::RecommendsImportant=false \
+                  $buildDeps \
+ && rm -rf /var/lib/apt/lists/* \
+           /home/fluent/.gem/ruby/2.3.0/cache/*.gem \
+           /home/root/.gem/ruby/2.3.0/cache/*.gem

data/examples/fluentd-vrli-plugin-photon-tdnf.dockerfile

@@ -0,0 +1,45 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2019 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
+# (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
+# Furthermore, all of the needed components are installed from the trusted Photon repository by using the tdnf package manager.
+#
+# Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
+# a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
+# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
+
+FROM photon:3.0-20190705
+
+USER root
+
+# Distro sync and install components
+RUN tdnf distro-sync --refresh -y \
+    && tdnf install -y \
+    rubygem-fluentd-1.6.3 \
+    #
+    # optional but used by fluentd
+    rubygem-oj-3.3.10 \
+    rubygem-async-http-0.48.2 \
+    jemalloc-4.5.0 \
+    #
+    # Install Log Insight plugin
+    rubygem-fluent-plugin-vmware-loginsight-0.1.5
+
+RUN ln -s /usr/lib/ruby/gems/2.5.0/bin/fluentd /usr/bin/fluentd \
+    && fluentd --setup
+
+# Make sure fluentd picks jemalloc
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+# Standard fluentd ports
+EXPOSE 24224 5140
+
+ENTRYPOINT ["/usr/bin/fluentd"]

data/examples/fluentd-vrli-plugin-photon.dockerfile

@@ -0,0 +1,68 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2019 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# Builds a photon-based image that contains fluentd, fluent-plugin-vmware-loginsight some of the tools recommended by fluent
+# (libjemalloc, oj, assync-http). This image is based on the minimalistic VMware Photon OS so the result is smaller in size.
+#
+# Fluentd is configured with the default configuration that gets produced by the `fluentd --setup` command. For an example of
+# a configuration that uses the fluent-plugin-vmware-loginsight plugin check fluent.conf under the examples dir:
+# https://github.com/vmware/fluent-plugin-vmware-loginsight/blob/master/examples/fluent.conf
+FROM photon:3.0-20190705
+
+USER root
+
+RUN buildDeps="\
+    binutils linux-api-headers glibc-devel \
+    make gcc gmp-devel libffi-devel \
+    tar bzip2 sed gawk" \
+    #
+    # Distro sync and install build dependencies
+    && tdnf distro-sync --refresh -y \
+    # Toybox conflicts with bzip2. The latter is needed to unpack libjemalloc
+    && tdnf remove -y toybox \
+    && tdnf install -y $buildDeps ruby \
+    #
+    # These are not required but are used if available
+    && gem install oj -v 3.3.10 \
+    && gem install json -v 2.2.0 \
+    && gem install async-http -v 0.46.3 \
+    #
+    # Install fluentd
+    && gem install --norc --no-document fluentd -v 1.6.3 \
+    && mkdir -p /fluentd/etc /fluentd/plugins \
+    #
+    # Install Log Insight plugin
+    && gem install --norc --no-document -v 0.1.5 fluent-plugin-vmware-loginsight \
+    #
+    # Install jemalloc 4.5.0
+    && curl -L --output /tmp/jemalloc-4.5.0.tar.bz2 https://github.com/jemalloc/jemalloc/releases/download/4.5.0/jemalloc-4.5.0.tar.bz2 \
+    && tar -C /tmp/ -xjvf /tmp/jemalloc-4.5.0.tar.bz2 \
+    && cd /tmp/jemalloc-4.5.0 \
+    && ./configure && make \
+    && mv lib/libjemalloc.so.2 /usr/lib \
+    && cd / \
+    #
+    # Cleanup to reduce image size
+    && rm -rf /tmp/jemalloc-4.5.0* \
+    && tdnf remove -y $buildDeps \
+    && tdnf clean all \
+    && gem sources --clear-all \
+    && gem cleanup
+
+# Create default fluent.conf
+RUN fluentd --setup
+
+# Make sure fluentd picks jemalloc
+ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
+
+# Standard fluentd ports
+EXPOSE 24224 5140
+
+ENTRYPOINT ["/usr/bin/fluentd"]

data/examples/k8s-log-collector-ds.yaml

@@ -0,0 +1,174 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fluentd-config
+  namespace: kube-system
+data:
+  fluent.conf: |
+    # We can use this config to load some default config or load user defined config
+    @include myapp-fluent.conf
+
+  myapp-fluent.conf: |
+    # Input sources
+    @include general.conf
+    @include systemd-input.conf
+    @include kubernetes-input.conf
+
+    # Parsing/Filtering
+    @include kubernetes-filter.conf
+
+    # Forwading - Be vigilant of the order in which these plugins are specified. Order matters!
+    @include myapp-loginsight-output.conf
+
+  general.conf: |
+    <system>
+      log_level info
+    </system>
+    # Prevent fluentd from handling records containing its own logs to handle cycles.
+    <match fluent.**>
+      @type null
+    </match>
+
+  systemd-input.conf: |
+    <source>
+      @type systemd
+      path /run/log/journal
+      # Can filter logs if we want, e.g.
+      # filters [{ "_SYSTEMD_UNIT": "kubelet.service" }]
+      <storage>
+        @type local
+        persistent true
+        path /var/log/fluentd-systemdlog.pos
+      </storage>
+      tag systemdlog
+      read_from_head true
+      strip_underscores true
+    </source>
+
+  kubernetes-input.conf: |
+    <source>
+      @type tail
+      path /var/log/containers/*.log
+      # One could exclude certain logs like:
+      # exclude_path ["/var/log/containers/log-collector*.log"]
+      pos_file /var/log/fluentd-docker.pos
+      time_format %Y-%m-%dT%H:%M:%S
+      tag kubernetes.*
+      format json
+      read_from_head true
+    </source>
+
+  kubernetes-filter.conf: |
+    <filter kubernetes.**>
+      @type kubernetes_metadata
+      merge_json_log true
+      preserve_json_log true
+    </filter>
+
+  myapp-loginsight-output.conf: |
+    # We are capturing all log messages and redirecting them to endpoints mentioned in each <store> tag.
+    # One may redirect these logs to muliple endpoints (including multiple LI instances).
+    # Or one may chose to tag their specific logs and add their own config to capture those tagged logs and redirect
+    # them to appropriate endpoint. This specific config needs to preceed this generic one.
+    <match **>
+      @type copy
+      <store>
+        @type vmware_loginsight
+        scheme https
+        ssl_verify true
+        # Loginsight host: One may use IP address or cname
+        # host X.X.X.X
+        host my-loginsight.mycompany.com
+        port 9000
+        path api/v1/events/ingest
+        agent_id XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
+        http_method post
+        serializer json
+        rate_limit_msec 0
+        raise_on_error false
+        include_tag_key true
+        tag_key tag
+      </store>
+      # If we want to debug and send logs to stdout as well
+      # <store>
+      #   @type stdout
+      # </store>
+    </match>
+
+
+  extra.conf: |
+    # If we want to transform events we could use:
+    #<filter **>
+    #  @type record_transformer
+    #  enable_ruby
+    #  auto_typecast
+    #  <record>
+    #    hostname "#{Socket.gethostname}"
+    #    mykey ${["message"=>record.to_json]}
+    #  </record>
+    #</filter>
+
+
+---
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: "log-collector"
+  namespace: "kube-system"
+  labels:
+    app: "log-collector"
+    version: v1
+spec:
+  template:
+    metadata:
+      labels:
+        app: "log-collector"
+        version: v1
+    spec:
+      containers:
+        - name: "log-collector"
+          image: "my-container-hub.com/log-collector:1.0.0"
+          command: ["fluentd", "-c", "/fluentd/etc/fluent.conf", "-p", "/fluentd/plugins"]
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "500m"
+            limits:
+              memory: "512Mi"
+              cpu: "500m"
+          volumeMounts:
+            - name: varlog
+              mountPath: /var/log
+            - name: varlibdockercontainers
+              mountPath: /var/lib/docker/containers
+              readOnly: true
+            - name: runlogjournal
+              mountPath: /run/log/journal
+            - name: fluentdconfig
+              mountPath: /fluentd/etc
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - name: varlog
+          hostPath:
+            path: /var/log
+        - name: varlibdockercontainers
+          hostPath:
+            path: /var/lib/docker/containers
+        - name: runlogjournal
+          hostPath:
+            path: /run/log/journal
+        - name: fluentdconfig
+          configMap:
+            name: "fluentd-config"

data/fluent-plugin-vmware-loginsight.gemspec

@@ -0,0 +1,38 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name    = "fluent-plugin-vmware-loginsight"
+  spec.version = "0.1.6"
+  spec.authors = ["Vishal Mohite", "Chris Todd"]
+  spec.email   = ["vmohite@vmware.com", "toddc@vmware.com"]
+
+  spec.summary       = %q{Fluend output plugin to forward logs to VMware Log Insight}
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/vmware/fluent-plugin-vmware-loginsight"
+  spec.license       = "MIT"
+
+  test_files, files  = `git ls-files -z`.split("\x0").partition do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.files         = files
+  spec.executables   = files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = test_files
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+end

data/lib/fluent/plugin/out_vmware_loginsight.rb

@@ -0,0 +1,321 @@
+# Fluentd plugin for VMware Log Insight
+#
+# Copyright 2018 VMware, Inc. All Rights Reserved.
+#
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.
+#
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file.
+#
+# SPDX-License-Identifier: MIT
+
+
+require "fluent/plugin/output"
+require 'json'
+require 'net/http'
+require 'uri'
+
+module Fluent
+  module Plugin
+    class Fluent::VmwareLoginsightOutput < Fluent::Output
+      class ConnectionFailure < StandardError; end
+
+      Fluent::Plugin.register_output('vmware_loginsight', self)
+
+      ### Connection Params ###
+      config_param :scheme, :string, :default => 'http'
+      # Loginsight Host ex. localhost
+      config_param :host, :string,  :default => 'localhost'
+      # In case we want to post to  multiple hosts. This is futuristic, Fluentd copy plugin can support this as is
+      #config_param :hosts, :string, :default => nil
+      # Loginsight port ex. 9000. Default 80
+      config_param :port, :integer, :default => 80
+      # Loginsight ingestion api path ex. 'api/v1/events/ingest'
+      config_param :path, :string, :default => 'api/v1/events/ingest'
+      # agent_id generated by your LI
+      config_param :agent_id, :string, :default => '0'
+      # Credentials if used
+      config_param :username, :string, :default => nil
+      config_param :password, :string, :default => nil, :secret => true
+      # Authentication nil | 'basic'
+      config_param :authentication, :string, :default => nil
+
+      # Set Net::HTTP.verify_mode to `OpenSSL::SSL::VERIFY_NONE`
+      config_param :ssl_verify, :bool, :default => true
+      config_param :ca_file, :string, :default => nil
+
+      ### API Params ###
+      # HTTP method
+      # post | put
+      config_param :http_method, :string, :default => :post
+      # form | json
+      config_param :serializer, :string, :default => :json
+      config_param :request_retries, :integer, :default => 3
+      config_param :request_timeout, :time, :default => 5
+      config_param :http_conn_debug, :bool, :default => false
+      config_param :max_batch_size, :integer, :default => 512000
+
+      # Simple rate limiting: ignore any records within `rate_limit_msec`
+      # since the last one.
+      config_param :rate_limit_msec, :integer, :default => 0
+      # Raise errors that were rescued during HTTP requests?
+      config_param :raise_on_error, :bool, :default => false
+      ### Additional Params
+      config_param :include_tag_key, :bool, :default => true
+      # Metadata key that identifies Fluentd tags
+      config_param :tag_key, :string, :default => 'tag'
+      # Keys from log event whose values should be added as log message/text
+      # to loginsight. Note these key/value pairs  won't be added as metadata/fields
+      config_param :log_text_keys, :array, default: ["log", "message", "msg"], value_type: :string
+      # Flatten hashes to create one key/val pair w/o losing log data
+      config_param :flatten_hashes, :bool, :default => true
+      # Seperator to use for joining flattened keys
+      config_param :flatten_hashes_separator, :string, :default => "_"
+
+
+      def initialize
+        super
+      end
+
+      def configure(conf)
+        super
+
+        @ssl_verify_mode = @ssl_verify ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        @auth = case @authentication
+                when 'basic'
+                  :basic
+                else
+                  :none
+                end
+
+        @last_request_time = nil
+      end
+
+      def start
+        super
+      end
+
+      def shutdown
+        super
+      end
+
+      def format_url()
+        url = "#{@scheme}://#{host}:#{port}/#{path}/#{agent_id}"
+        url
+      end
+
+      def set_header(req)
+        if @serializer == 'json'
+          set_json_header(req)
+        end
+        req
+      end
+
+      def set_json_header(req)
+        req['Content-Type'] = 'application/json'
+        req
+      end
+
+      def shorten_key(key)
+        # LI doesn't allow some characters in field 'name'
+        # like '/', '-', '\', '.', etc. so replace them with @flatten_hashes_separator
+        key = key.gsub(/[\/\.\-\\]/,@flatten_hashes_separator).downcase
+        # shorten field names
+        key = key.gsub(/kubernetes_/,'k8s_')
+        key = key.gsub(/namespace/,'ns')
+        key = key.gsub(/labels_/,'')
+        key = key.gsub(/_name/,'')
+        key = key.gsub(/_hash/,'')
+        key = key.gsub(/container_/,'')
+        key
+      end
+
+      def create_loginsight_event(tag, time, record)
+        flattened_records = {}
+        if @flatten_hashes
+          flattened_records = flatten_record(record, [])
+        else
+          flattened_records = record
+        end
+        # tag can be immutable in some cases, use a copy.
+        flattened_records[@tag_key] = tag.dup if @include_tag_key
+        fields = []
+        keys = []
+        log = ''
+        flattened_records.each do |key, value|
+          begin
+            next if value.nil?
+            # LI doesn't support duplicate fields, make unique names by appending underscore
+            key = shorten_key(key)
+            while keys.include?(key)
+              key = key + '_'
+            end
+            keys.push(key)
+            key.force_encoding("utf-8")
+            # convert value to json string if its a hash and to string if not already a string
+            begin
+              value = value.to_json if value.is_a?(Hash)
+              value = value.to_s
+              value = value.frozen? ? value.dup : value # if value is immutable, use a copy.
+              value.force_encoding("utf-8")
+            rescue Exception=>e
+              $log.warn "force_encoding exception: " "#{e.class}, '#{e.message}', " \
+                        "\n Request: #{key} #{record.to_json[1..1024]}"
+              value = "Exception during conversion: #{e.message}"
+            end
+          end
+          if @log_text_keys.include?(key)
+            if log != "#{value}"
+              if log.empty?
+                log = "#{value}"
+              else
+                log += " #{value}"
+              end
+            end
+          else
+            # If there is time information available, update time for LI. LI ignores
+            # time if it is out of the error/adjusment window of 10 mins. in such
+            # cases we would still like to preserve time info, so add it as event.
+            # TODO Ignore the below block for now. Handle the case for time being in 
+            #      different formats than milliseconds
+            #if ['time', '_source_realtime_timestamp'].include?(key)
+            #  time = value
+            #end
+            fields << {"name" => key, "content" => value}
+          end
+        end
+        event = {
+          "fields" => fields,
+          "text" => log.gsub(/^$\n/, ''),
+          "timestamp" => time * 1000
+        }
+        event
+      end
+
+      def flatten_record(record, prefix=[])
+        ret = {}
+
+        case record
+          when Hash
+            record.each do |key, value|
+              if @log_text_keys.include?(key)
+                ret.merge!({key.to_s => value})
+              else
+                ret.merge! flatten_record(value, prefix + [key.to_s])
+              end
+            end
+          when Array
+            record.each do |value|
+              ret.merge! flatten_record(value, prefix)
+            end
+          else
+            return {prefix.join(@flatten_hashes_separator) => record}
+        end
+        ret
+      end
+
+      def create_request(tag, time, record)
+        url = format_url()
+        uri = URI.parse(url)
+        req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+        set_body(req, tag, time, record)
+        set_header(req)
+        return req, uri
+      end
+
+
+      def send_request(req, uri)
+        is_rate_limited = (@rate_limit_msec != 0 and not @last_request_time.nil?)
+        if is_rate_limited and ((Time.now.to_f - @last_request_time) * 1000.0 < @rate_limit_msec)
+          $log.info('Dropped request due to rate limiting')
+          return
+        end
+
+        if @auth and @auth == 'basic'
+          req.basic_auth(@username, @password)
+        end
+        begin
+          retries ||= 2
+          response = nil
+          @last_request_time = Time.now.to_f
+
+          http_conn = Net::HTTP.new(uri.host, uri.port)
+          # For debugging, set this
+          http_conn.set_debug_output($stdout) if @http_conn_debug
+          http_conn.use_ssl = (uri.scheme == 'https')
+          if http_conn.use_ssl?
+            http_conn.ca_file = @ca_file
+          end
+          http_conn.verify_mode = @ssl_verify_mode
+
+          response = http_conn.start do |http|
+            http.read_timeout = @request_timeout
+            http.request(req)
+          end
+        rescue => e # rescue all StandardErrors
+          # server didn't respond
+          # Be careful while turning on below log, if LI instance can't be reached and you're sending
+          # log-container logs to LI as well, you may end up in a cycle.
+          # TODO handle the cyclic case at plugin level if possible.
+          # $log.warn "Net::HTTP.#{req.method.capitalize} raises exception: " \
+          #   "#{e.class}, '#{e.message}', \n Request: #{req.body[1..1024]}"
+          retry unless (retries -= 1).zero?
+          raise e if @raise_on_error
+        else
+           unless response and response.is_a?(Net::HTTPSuccess)
+              res_summary = if response
+                               "Response Code: #{response.code}\n"\
+                               "Response Message: #{response.message}\n" \
+                               "Response Body: #{response.body}"
+                            else
+                               "Response = nil"
+                            end
+              # ditto cyclic warning
+              # $log.warn "Failed to #{req.method} #{uri}\n(#{res_summary})\n" \
+              #   "Request Size: #{req.body.size} Request Body: #{req.body[1..1024]}"
+           end #end unless
+        end # end begin
+      end # end send_request
+
+      def send_events(uri, events)
+        req = Net::HTTP.const_get(@http_method.to_s.capitalize).new(uri.path)
+        event_req = {
+          "events" => events
+        }
+        req.body = event_req.to_json
+        set_header(req)
+        send_request(req, uri)
+      end
+
+      def handle_records(tag, es)
+        url = format_url()
+        uri = URI.parse(url)
+        events = []
+        count = 0
+        es.each do |time, record|
+          new_event = create_loginsight_event(tag, time, record)
+          new_event_size = new_event.to_json.size
+          if new_event_size > @max_batch_size
+              $log.warn "dropping event larger than max_batch_size: #{new_event.to_json[1..1024]}"
+          else
+            if (count + new_event_size) > @max_batch_size
+              send_events(uri, events)
+              events = []
+              count = 0
+            end
+            count += new_event_size
+            events << new_event
+          end
+        end
+        if count > 0
+          send_events(uri, events)
+        end
+      end
+
+      def emit(tag, es, chain)
+        handle_records(tag, es)
+        chain.next
+      end
+    end
+  end
+end
+

data/test/helper.rb

@@ -0,0 +1,19 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/output"
+require "fluent/test/helpers"
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
+Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_out_vmware_loginsight.rb

@@ -0,0 +1,30 @@
+# Fluentd plugin for VMware Log Insight
+# 
+# Copyright 2018 VMware, Inc. All Rights Reserved. 
+# 
+# This product is licensed to you under the MIT license (the "License").  You may not use this product except in compliance with the MIT License.  
+# 
+# This product may include a number of subcomponents with separate copyright notices and license terms. Your use of these subcomponents is subject to the terms and conditions of the subcomponent's license, as noted in the LICENSE file. 
+# 
+# SPDX-License-Identifier: MIT
+
+# TODO Need to add tests.
+
+require "helper"
+require "fluent/plugin/out_vmware_loginsight.rb"
+
+class VmwareLoginsightOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  test "failure" do
+    flunk
+  end
+
+  private
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::VmwareLoginsightOutput).configure(conf)
+  end
+end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-vmware-loginsight
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.6
 platform: ruby
 authors:
 - Vishal Mohite
+- Chris Todd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-13 00:00:00.000000000 Z
+date: 2019-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -75,13 +76,26 @@ dependencies:
 description: Fluend output plugin to forward logs to VMware Log Insight
 email:
 - vmohite@vmware.com
+- toddc@vmware.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - CONTRIBUTING.md
+- Gemfile
+- LICENSE
 - README.md
+- Rakefile
+- examples/fluent.conf
+- examples/fluentd-vrli-plugin-debian.dockerfile
+- examples/fluentd-vrli-plugin-photon-tdnf.dockerfile
+- examples/fluentd-vrli-plugin-photon.dockerfile
+- examples/k8s-log-collector-ds.yaml
+- fluent-plugin-vmware-loginsight.gemspec
+- lib/fluent/plugin/out_vmware_loginsight.rb
+- test/helper.rb
+- test/plugin/test_out_vmware_loginsight.rb
 homepage: https://github.com/vmware/fluent-plugin-vmware-loginsight
 licenses:
 - MIT
@@ -102,8 +116,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Fluend output plugin to forward logs to VMware Log Insight
-test_files: []
+test_files:
+- test/helper.rb
+- test/plugin/test_out_vmware_loginsight.rb