fluent-plugin-kubernetes_tagged_remote_syslog 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 09b28d6629f6b4a6a9b4755f910df77b292a3670
-  data.tar.gz: e81e939e8f11546f1e25d2fe1309078a6b99b4bb
+  metadata.gz: 601c5afbbc0db25d4e4f755e3271f474e15705e6
+  data.tar.gz: 02bf9e9589975c2755d5d2fe6aa74c2cc7af2b6d
 SHA512:
-  metadata.gz: 7639cbf9f4cd64e53da20ce83a1fa589ea9bfb2dcbbac803aa283ac119d0a7934ab0bb19bf4bd377705bfe41b1810cdf46cfcc75703da74a4c33c3b34464f3e7
-  data.tar.gz: aea0fbe2b6b33f475d73976502dfac9d8abaf6f587716f7c5a5ef98398d95f172f02fc9e613d0fb0f4e4faef87cb7ad4d7637cc09dd48d8da03597b03f0edc8c
+  metadata.gz: 0f4d666b250c8286321f4685d18176f2ad959adebad8bd371940f2279086a7031142c69c10262b66185860093fec711d60cd1591324d2d8ce5f84c4f870abca1
+  data.tar.gz: cda027c1edfbcdd0bfeb5740cd79c24f5bfe691f97f42dc30c305ca9fa4284f13afa1954f3f756869f46d6a5f4737f23c317e22d8f1ff3909734f53b727dd489

data/.travis.yml

@@ -2,8 +2,6 @@ language: ruby
 sudo: false
 cache: bundler
 rvm:
-  - 2.1.10
-  - 2.2.5
   - 2.3.1
   - 2.4.1
 before_install:

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.3.6
+
+* Rewritten, failure preventions.
+
+## 0.3.4
+
+* Forking and creating new plugin.
+
 ## 0.3.3
 
 * Allow override "severity" or "facility" from records [#19](https://github.com/dlackty/fluent-plugin-remote_syslog/pull/19)

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Richard Lee
+Copyright (c) 2017 Jakub Kvita
 
 MIT License
 

data/README.md

@@ -1,17 +1,50 @@
 # fluent-plugin-kubernetes_tagged_remote_syslog
 
-<!-- [![Build Status](https://travis-ci.org/dlackty/fluent-plugin-remote_syslog.svg?branch=master)](https://travis-ci.org/dlackty/fluent-plugin-remote_syslog) -->
+<!-- [![Build Status](https://travis-ci.org/kvitajakub/fluent-plugin-kubernetes_tagged_remote_syslog.svg?branch=master)](https://travis-ci.org/kvitajakub/fluent-plugin-kubernetes_tagged_remote_syslog) -->
 
 [Fluentd](http://fluentd.org) plugin for output to remote syslog service (e.g. [Papertrail](http://papertrailapp.com/)). It is meant to use in Kubernetes environment with [kubernetes_metadata_filter](https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter) plugin annotating the messages.
 
+This plugin was created because regular remote syslog Fluentd plugins doesn't work well with kubernetes annotated messages. `hostname`/`program` fields have not helpful values `localhost`/`fluentd` and actual message is unreadable compact JSON.
+
 ## Installation
 
 ```bash
  fluent-gem install fluent-plugin-kubernetes_tagged_remote_syslog
 ```
 
-## Usage
+## Example input/output
+Default input should be JSON message annotated with kubernetes info:
+```
+{
+  "log": "2015/05/05 19:54:41 log me\n",
+  "stream": "stderr",
+  "docker": {
+    "id": "df14e0d5ae4c07284fa636d739c8fc2e6b52bc344658de7d3f08c36a2e804115",
+  }
+  "kubernetes": {
+    "host": "jimmi-redhat.localnet",
+    "pod_name":"fabric8-console-controller-98rqc",
+    "pod_id": "c76927af-f563-11e4-b32d-54ee7527188d",
+    "container_name": "fabric8-console-container",
+    "namespace_name": "default",
+    "namespace_id": "23437884-8e08-4d95-850b-e94378c9b2fd",
+    "labels": {
+      "component": "fabric8Console"
+    }
+  }
+}
+```
+This will become:
+```
+default fabric8-console-controller-98rqc: 2015/05/05 19:54:41 \n
+```
+
+And at the message receive Papertrail will add a timestamp in front of the message:
+```
+May 05 19:54:45 default fabric8-console-controller-98rqc: 2015/05/05 19:54:41 log me\n
+```
 
+## Usage
 ```
 <match foo>
   type kubernetes_tagged_remote_syslog
@@ -21,6 +54,37 @@
   hostname 'namespace_name'
 </match>
 ```
+`hostname` value will look for the key with that name in message and adds it as hostname. If no key is found it will use the value of the key itself.
+`program` value works in the same way as `hostname`.
+
+## Dropping messages from namespaces you don't need
+Parameter `skip_namespaces` will drop messages from selected namespaces. This should be technically in its own filter plugin, but that would require installing and configuring two pluggins so I just dropped it here.
+
+Default configuration will drop messages from `kube-system` and `deis` namespaces.
+
+## Special Use Case - Deis Workflow
+
+I am running this plugin myself as part of [Deis Workflow Fluentd](https://github.com/deis/fluentd) DaemonSet. This requires editing `values.yaml`:
+```
+...
+fluentd:
+  syslog:
+    # Configure the following ONLY if using Fluentd to send log messages to both
+    # the Logger component and external syslog endpoint
+    # external syslog endpoint url
+    host: ""
+    # external syslog endpoint port
+    port: ""
+  daemon_environment:
+    FLUENTD_PLUGIN_1: "fluent-plugin-kubernetes_tagged_remote_syslog"
+    CUSTOM_STORE_1: "<store> \n
+                       @type kubernetes_tagged_remote_syslog \n
+                       host logs5.papertrailapp.com \n
+                       port REDACTED \n
+                     </store>"
+...
+```
+Plugin will be installed with `fluent-gem` from [rubygems.com](https://rubygems.org/).
 
 ## License
 

data/VERSION

@@ -1 +1 @@
-0.3.5
+0.3.6

data/fluent-plugin-kubernetes_tagged_remote_syslog.gemspec

@@ -7,11 +7,13 @@ Gem::Specification.new do |spec|
   spec.version       = File.read("VERSION").strip
   spec.authors       = ["Richard Lee, Jakub Kvita"]
   spec.email         = ["kvitajakub@gmail.com"]
-  spec.summary       = %q{Fluentd output plugin for remote syslog from Kubernetes.}
+  spec.summary       = %q{Fluentd output plugin for remote syslog. This is meant for processing kubernetes annotated messages.}
   spec.description   = spec.description
   spec.homepage      = "https://github.com/kvitajakub/fluent-plugin-kubernetes_tagged_remote_syslog"
   spec.license       = "MIT"
 
+  spec.required_ruby_version = '>= 2.3.0'
+
   spec.files         = `git ls-files -z`.split("\x0")
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]

data/lib/fluent/plugin/out_kubernetes_tagged_remote_syslog.rb

@@ -23,25 +23,24 @@ module Fluent
     end
 
     def emit(tag, es, chain)
-      es.each do |time, record|
-        record.each_pair do |k, v|
+      es.each do |_time, record|
+        record.each_pair do |_k, v|
           if v.is_a?(String)
             v.force_encoding('utf-8')
           end
         end
 
-        next if !record.key?('kubernetes') ||
-                skip_namespaces.include?(record['kubernetes']['namespace_name'])
+        next if skip_namespaces.include?(record.dig('kubernetes', 'namespace_name'))
 
         @loggers[tag] ||= RemoteSyslogLogger::UdpSender.new(
           @host,
           @port,
           facility: record['facility'] || @facility,
           severity: record['severity'] || @severity,
-          program: (record['kubernetes'][@program] || @program)[0...31],
-          local_hostname: (record['kubernetes'][@hostname] || @hostname)[0...31])
+          program: (record.dig('kubernetes', @program) || @program)[0...31],
+          local_hostname: (record.dig('kubernetes', @hostname) || @hostname)[0...31])
 
-        @loggers[tag].transmit record['log'].to_s
+        @loggers[tag].transmit((if record.key?('log') then record['log'] else record end).to_s)
       end
       chain.next
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kubernetes_tagged_remote_syslog
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Richard Lee, Jakub Kvita
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-07 00:00:00.000000000 Z
+date: 2017-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -111,7 +111,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -122,7 +122,8 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Fluentd output plugin for remote syslog from Kubernetes.
+summary: Fluentd output plugin for remote syslog. This is meant for processing kubernetes
+  annotated messages.
 test_files:
 - test/plugin/out_kubernetes_tagged_remote_syslog.rb
 - test/test_helper.rb