fluent-plugin-papertrail 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0c64afb4983305a88b12faeb066bedb5dd09c077
-  data.tar.gz: dd5f6d86951f19f121609a5fab36aac1b6a30658
+  metadata.gz: c23981bf5baa13f2c2d8a8ec0dd586a37b5db034
+  data.tar.gz: 29a543b565b6a14d407f6daacced1b8a7ea2eb35
 SHA512:
-  metadata.gz: 991a840b6840f26e3812b95908905ce16e9151b3d1c1c15d0ca81a10079f2967324ee78614ca830db6d14211d0b48bd16cb54e365cb2ad809bf0df352506c679
-  data.tar.gz: '056480b9fb04bc1789745c5cc0481e21761e7e8651aad1b01946598dd6daa5194f73b3227349a5f70e98b061c145da14d2674f8d53abfde9965a9a060571a089'
+  metadata.gz: a91ac215ba2fcc071522845b42d4e5ec78566747a176d155605f8689195b5e2bdba809808337d7ba77b02a55000791333fba1ba4230ccd55885c6009c4e2df48
+  data.tar.gz: 1e421735baa691d46ba660717dc64d46f472983e0ca069203f29bfd18e6c3d6a7599885259308e1dfc516d68ad73c34c97d1ca26275ee51f0bd5007ca1941673

data/README.md

@@ -83,9 +83,9 @@ If you want to change any of these parameters simply add them to a match stanza.
 
 ## Kubernetes
 
-This repo also includes Kubernetes and Docker assets which do all of the heavy lifting for you.
+This repo includes a Kubernetes DaemonSet and accompanying Docker container which will stream all of your Kubernetes logs, containers and services, to Papertrail.
 
-If you'd like to deploy this plugin as a DaemonSet to your Kubernetes cluster, just adjust the `FLUENT_*` environment variables in `kubernetes/fluentd-daemonset-papertrail.yaml` and push it to your cluster with:
+To deploy this plugin as a DaemonSet to your Kubernetes cluster, just adjust the `FLUENT_*` environment variables in `kubernetes/fluentd-daemonset-papertrail.yaml` and push it to your cluster with:
 
 ```
 kubectl apply -f kubernetes/fluentd-daemonset-papertrail.yaml
@@ -93,6 +93,19 @@ kubectl apply -f kubernetes/fluentd-daemonset-papertrail.yaml
 
 The Dockerfile that generates [the image used in this DaemonSet](https://quay.io/repository/solarwinds/fluentd-kubernetes), can be found at `docker/Dockerfile`.
 
+### Annotations
+
+You can redirect logs to alternate Papertrail destinations by adding annotations to your Pods or Namespaces:
+
+```
+solarwinds.io/papertrail_host: 'logs0.papertrailapp.com'
+solarwinds.io/papertrail_port: '12345'
+```
+
+If both the Pod and Namespace have annotations for any running Pod, the Pod's annotation is used.
+
+### Audit Logs
+
 If you'd like to redirect Kubernetes API Server Audit logs to a seperate Papertrail destination, add the following to your `fluent.conf`:
 ```
 <match kube-apiserver-audit>
@@ -104,6 +117,8 @@ If you'd like to redirect Kubernetes API Server Audit logs to a seperate Papertr
 </match>
 ```
 
+This requires you to configure an [audit policy file](https://kubernetes.io/docs/tasks/debug-application-cluster/audit/) on your cluster.
+
 ## Development
 
 This plugin is targeting Ruby 2.4 and Fluentd v1.0, although it should work with older versions of both.

data/docker/Dockerfile

@@ -13,7 +13,7 @@ RUN apt-get update \
 RUN echo 'gem: --no-document' >> /etc/gemrc \
     && gem install fluent-plugin-systemd -v 0.3.1 \
     && gem install fluent-plugin-rewrite-tag-filter -v 1.6.0 \
-    && gem install fluent-plugin-papertrail -v 0.1.3 \
+    && gem install fluent-plugin-papertrail -v 0.1.4 \
     && gem install fluent-plugin-kubernetes_metadata_filter
 
 RUN SUDO_FORCE_REMOVE=yes \

data/docker/conf/kubernetes.conf

@@ -140,6 +140,8 @@
 
 <filter kubernetes.**>
   type kubernetes_metadata
+  include_namespace_metadata true
+  annotation_match ["solarwinds.io/papertrail_*"]
 </filter>
 
 <filter kube-apiserver-audit>

data/fluent-plugin-papertrail.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-papertrail"
-  spec.version       = "0.1.3"
+  spec.version       = "0.1.4"
   spec.authors       = ["Jonathan Lozinski", "Alex Ouzounis", "Chris Rust"]
   spec.email         = ["jonathan.lozinski@solarwinds.com", "alex.ouzounis@solarwinds.com", "chris.rust@solarwinds.com"]
 

data/kubernetes/fluentd-daemonset-papertrail.yaml

@@ -22,7 +22,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: fluentd
-        image: quay.io/solarwinds/fluentd-kubernetes:v1.1.1-debian-papertrail
+        image: quay.io/solarwinds/fluentd-kubernetes:v1.1.1-debian-papertrail-0.1.4
         imagePullPolicy: Always
         env:
           - name: FLUENT_PAPERTRAIL_HOST

data/lib/fluent/plugin/out_papertrail.rb

@@ -3,7 +3,7 @@ require 'syslog_protocol'
 module Fluent
   class Papertrail < Fluent::BufferedOutput
     class SocketFailureError < StandardError; end
-    attr_accessor :socket
+    attr_accessor :sockets
 
     # if left empty in fluent config these config_param's will error
     config_param :papertrail_host, :string
@@ -13,13 +13,16 @@ module Fluent
     # overriding default flush_interval (60 sec) from Fluent::BufferedOutput
     config_param :flush_interval, :time, default: 1
 
-
     # register as 'papertrail' fluent plugin
     Fluent::Plugin.register_output('papertrail', self)
 
     def configure(conf)
       super
-      @socket = create_socket(@papertrail_host, @papertrail_port)
+      # create initial sockets hash and socket based on config param
+      @sockets = {}
+      socket_key = "#{@papertrail_host}:#{@papertrail_port}"
+      @sockets[socket_key] = create_socket(@papertrail_host, @papertrail_port)
+      # redefine default hostname if it's been passed in through ENV
       @default_hostname = ENV['FLUENT_HOSTNAME'] || @default_hostname
     end
 
@@ -29,8 +32,9 @@ module Fluent
 
     def write(chunk)
       chunk.msgpack_each {|(tag, time, record)|
+        socket_key = pick_socket(record)
         packet = create_packet(tag, time, record)
-        send_to_papertrail(packet)
+        send_to_papertrail(packet, socket_key)
       }
     end
 
@@ -51,7 +55,7 @@ module Fluent
       ssl
     end
 
-    def create_packet(tag,time,record)
+    def create_packet(tag, time, record)
       # construct syslog packet from fluent record
       packet = SyslogProtocol::Packet.new
       packet.hostname = record['hostname'] || @default_hostname
@@ -63,25 +67,42 @@ module Fluent
       packet
     end
 
-    def send_to_papertrail(packet)
-      # recreate the socket if it's nil -- see below
-      @socket ||= create_socket(@papertrail_host, @papertrail_port)
-
-      papertrail_addr = "#{@papertrail_host}:#{@papertrail_port}"
+    def pick_socket(record)
+      # if kubernetes pod has papertrail destination as annotation, use it
+      if record.dig('kubernetes', 'annotations', 'solarwinds_io/papertrail_host') && \
+         record.dig('kubernetes', 'annotations', 'solarwinds_io/papertrail_port')
+        host = record['kubernetes']['annotations']['solarwinds_io/papertrail_host']
+        port = record['kubernetes']['annotations']['solarwinds_io/papertrail_port']
+      # else if kubernetes namespace has papertrail destination as annotation, use it
+      elsif record.dig('kubernetes', 'namespace_annotations', 'solarwinds_io/papertrail_host') && \
+            record.dig('kubernetes', 'namespace_annotations', 'solarwinds_io/papertrail_port')
+        host = record['kubernetes']['namespace_annotations']['solarwinds_io/papertrail_host']
+        port = record['kubernetes']['namespace_annotations']['solarwinds_io/papertrail_port']
+      # else use pre-configured destination
+      else
+        host = @papertrail_host
+        port = @papertrail_port
+      end
+      socket_key = "#{host}:#{port}"
+      # recreate the socket if it's nil
+      @sockets[socket_key] ||= create_socket(host, port)
+      socket_key
+    end
 
-      if @socket.nil?
-        err_msg = "Unable to create socket with #{papertrail_addr}"
+    def send_to_papertrail(packet, socket_key)
+      if @sockets[socket_key].nil?
+        err_msg = "Unable to create socket with #{socket_key}"
         log.error err_msg
         raise SocketFailureError, err_msg
       else
         begin
           # send it
-          @socket.puts packet.assemble
+          @sockets[socket_key].puts packet.assemble
         rescue => e
-          err_msg = "Error writing to #{papertrail_addr}: #{e}"
+          err_msg = "Error writing to #{socket_key}: #{e}"
           log.error err_msg
           # socket failed, reset to nil to recreate for the next write
-          @socket = nil
+          @sockets[socket_key] = nil
           raise SocketFailureError, err_msg, e.backtrace
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-papertrail
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Jonathan Lozinski
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-03-13 00:00:00.000000000 Z
+date: 2018-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -160,7 +160,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.5.2.2
 signing_key: 
 specification_version: 4
 summary: Remote Syslog Output Fluentd plugin for papertrail