fluent-plugin-papertrail 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13f071d87f88a75ef2453e21eafd29feebff7b91
-  data.tar.gz: 4abf3a957a68b902098a944577a1d2c3c5ec75ca
+  metadata.gz: 0c64afb4983305a88b12faeb066bedb5dd09c077
+  data.tar.gz: dd5f6d86951f19f121609a5fab36aac1b6a30658
 SHA512:
-  metadata.gz: 9fcdbd9fd6d970117b86d27c6455019ce459da0a5315521e37c0f0ddf21d5d182f2f5cbbd8de785cefb44ac744a70d667432cc887cf54ad9859a0c647a401378
-  data.tar.gz: 18dfa2b58cae05762579db20ddff31dbee6efc3b40f42797cdff7d5e9e41b42e4a98dca5a07c1b635f11b3b7d84ddbba929e10dae075c53dc16da595d7fd261f
+  metadata.gz: 991a840b6840f26e3812b95908905ce16e9151b3d1c1c15d0ca81a10079f2967324ee78614ca830db6d14211d0b48bd16cb54e365cb2ad809bf0df352506c679
+  data.tar.gz: '056480b9fb04bc1789745c5cc0481e21761e7e8651aad1b01946598dd6daa5194f73b3227349a5f70e98b061c145da14d2674f8d53abfde9965a9a060571a089'

data/README.md

@@ -31,7 +31,9 @@ To configure this in fluentd:
 </match>
 ```
 
-Use a record transform plugin to populate within the record the following fields:
+### Configuring a record_transformer
+
+This plugin expects the following fields to be set for each Fluent record:
 ```
     message   The log
     program   The program/tag
@@ -40,8 +42,7 @@ Use a record transform plugin to populate within the record the following fields
     hostname  The source hostname for papertrail logging
 ```
 
-The following snippet sets up the records for Kubernetes and assumes you are using
-the [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) plugin which populates the record with useful metadata:
+The following example is a `record_transformer` filter, from the [Kubernetes assets](docker/conf/kubernetes.conf) in this repo, that is used along with the [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) to populate the required fields for our plugin:
 ```yaml
 <filter kubernetes.**>
   type kubernetes_metadata
@@ -60,6 +61,8 @@ the [fluent-plugin-kubernetes_metadata_filter](https://github.com/fabric8io/flue
 </filter>
 ```
 
+If you don't set `hostname` and `program` values in your record, they will default to the environment variable `FLUENT_HOSTNAME` or `'unidentified'` and the fluent tag, respectively.
+
 ### Advanced Configuration
 This plugin inherits a few useful config parameters from Fluent's `BufferedOutput` class.
 
@@ -90,6 +93,17 @@ kubectl apply -f kubernetes/fluentd-daemonset-papertrail.yaml
 
 The Dockerfile that generates [the image used in this DaemonSet](https://quay.io/repository/solarwinds/fluentd-kubernetes), can be found at `docker/Dockerfile`.
 
+If you'd like to redirect Kubernetes API Server Audit logs to a seperate Papertrail destination, add the following to your `fluent.conf`:
+```
+<match kube-apiserver-audit>
+    type papertrail
+    num_threads 4
+
+    papertrail_host "#{ENV['FLUENT_PAPERTRAIL_AUDIT_HOST']}"
+    papertrail_port "#{ENV['FLUENT_PAPERTRAIL_AUDIT_PORT']}"
+</match>
+```
+
 ## Development
 
 This plugin is targeting Ruby 2.4 and Fluentd v1.0, although it should work with older versions of both.
@@ -113,7 +127,7 @@ To release a new version, update the version number in the [GemSpec](fluent-plug
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/solarwinds/fluent-plugin-papertrail.
+Bug reports and pull requests are welcome on GitHub at: https://github.com/solarwinds/fluent-plugin-papertrail
 
 ## License
 

data/docker/Dockerfile

@@ -3,25 +3,23 @@ FROM fluent/fluentd:v1.1.1-debian
 USER root
 WORKDIR /home/fluent
 ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
+ENV BUILDDEPS="sudo make gcc g++ libc-dev ruby-dev libffi-dev"
 
-RUN buildDeps="sudo make gcc g++ libc-dev ruby-dev libffi-dev" \
-     && apt-get update \
-     && apt-get upgrade -y \
-     && apt-get install \
-     -y --no-install-recommends \
-     $buildDeps \
-    && echo 'gem: --no-document' >> /etc/gemrc \
-    && gem install fluent-plugin-secure-forward \
-    && gem install fluent-plugin-record-reformer \
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get install -y --no-install-recommends \
+    $BUILDDEPS
+
+RUN echo 'gem: --no-document' >> /etc/gemrc \
     && gem install fluent-plugin-systemd -v 0.3.1 \
     && gem install fluent-plugin-rewrite-tag-filter -v 1.6.0 \
-    && gem install fluent-plugin-papertrail -v 0.1.2.pre.dev \
-    && gem install fluent-plugin-kubernetes_metadata_filter \
-    && gem install ffi \
-    && SUDO_FORCE_REMOVE=yes \
+    && gem install fluent-plugin-papertrail -v 0.1.3 \
+    && gem install fluent-plugin-kubernetes_metadata_filter
+
+RUN SUDO_FORCE_REMOVE=yes \
     apt-get purge -y --auto-remove \
                   -o APT::AutoRemove::RecommendsImportant=false \
-                  $buildDeps \
+                  $BUILDDEPS \
     && rm -rf /var/lib/apt/lists/* \
     && gem sources --clear-all \
     && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem
@@ -38,13 +36,8 @@ COPY plugins /fluentd/plugins/
 ENV FLUENTD_OPT=""
 ENV FLUENTD_CONF="fluent.conf"
 
-# jemalloc is memory optimization only available for td-agent
-# td-agent is provided and QA'ed by treasuredata as rpm/deb/.. package
-# -> td-agent (stable) vs fluentd (edge)
-#ENV LD_PRELOAD="/usr/lib/libjemalloc.so.2"
-
 # Overriding entrypoint, otherwise parent image causes to run as user fluent
 ENTRYPOINT []
 
 # Run Fluentd
-CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
+CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT

data/fluent-plugin-papertrail.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-papertrail"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["Jonathan Lozinski", "Alex Ouzounis", "Chris Rust"]
   spec.email         = ["jonathan.lozinski@solarwinds.com", "alex.ouzounis@solarwinds.com", "chris.rust@solarwinds.com"]
 

data/kubernetes/fluentd-daemonset-papertrail.yaml

@@ -1,4 +1,3 @@
-# Uncomment sections below to allow audit logs to go to a separate host
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
@@ -23,17 +22,13 @@ spec:
         effect: NoSchedule
       containers:
       - name: fluentd
-        image: quay.io/solarwinds/fluentd-kubernetes:v0.12.33-debian-papertrail
+        image: quay.io/solarwinds/fluentd-kubernetes:v1.1.1-debian-papertrail
         imagePullPolicy: Always
         env:
           - name: FLUENT_PAPERTRAIL_HOST
             value: 'logsN.papertrailapp.com'
           - name: FLUENT_PAPERTRAIL_PORT
             value: 'NNNNN'
-#          - name: FLUENT_PAPERTRAIL_AUDIT_HOST
-#            value: 'logsN.papertrailapp.com'
-#          - name: FLUENT_PAPERTRAIL_AUDIT_PORT
-#            value: 'NNNNN'
           - name: FLUENT_HOSTNAME
             value: 'my-cluster-name'
         resources:
@@ -77,16 +72,6 @@ data:
     @include kubernetes.conf
     @include systemd.conf
 
-#    # Capture audit logs
-#    <match kube-apiserver-audit>
-#      type papertrail
-#      num_threads 4
-#
-#      papertrail_host "#{ENV['FLUENT_PAPERTRAIL_AUDIT_HOST']}"
-#      papertrail_port "#{ENV['FLUENT_PAPERTRAIL_AUDIT_PORT']}"
-#    </match>
-#
-    # Capture all unmatched tags
     <match **>
       type papertrail
       num_threads 4

data/lib/fluent/plugin/out_papertrail.rb

@@ -8,15 +8,19 @@ module Fluent
     # if left empty in fluent config these config_param's will error
     config_param :papertrail_host, :string
     config_param :papertrail_port, :integer
+    # default to ENV['FLUENT_HOSTNAME'] or :default_hostname if no hostname in record
+    config_param :default_hostname, :string, default: 'unidentified'
     # overriding default flush_interval (60 sec) from Fluent::BufferedOutput
     config_param :flush_interval, :time, default: 1
 
+
     # register as 'papertrail' fluent plugin
     Fluent::Plugin.register_output('papertrail', self)
 
     def configure(conf)
       super
       @socket = create_socket(@papertrail_host, @papertrail_port)
+      @default_hostname = ENV['FLUENT_HOSTNAME'] || @default_hostname
     end
 
     def format(tag, time, record)
@@ -50,10 +54,10 @@ module Fluent
     def create_packet(tag,time,record)
       # construct syslog packet from fluent record
       packet = SyslogProtocol::Packet.new
-      packet.hostname = record['hostname']
-      packet.facility = record['facility']
-      packet.severity = record['severity']
-      packet.tag      = record['program']
+      packet.hostname = record['hostname'] || @default_hostname
+      packet.facility = record['facility'] || 'local0'
+      packet.severity = record['severity'] || 'info'
+      packet.tag      = record['program'] || tag
       packet.content  = record['message']
       packet.time     = time ? Time.at(time) : Time.now
       packet

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-papertrail
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Jonathan Lozinski
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-10 00:00:00.000000000 Z
+date: 2018-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -132,8 +132,6 @@ files:
 - Makefile
 - README.md
 - Rakefile
-- bin/console
-- bin/setup
 - docker/Dockerfile
 - docker/conf/fluent.conf
 - docker/conf/kubernetes.conf

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "fluent/plugin/papertrail"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here