fluent-plugin-systemd 1.0.0.rc3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d36c2726704e80195922ab5a2e7cb0f89825f27e
-  data.tar.gz: 93d387a176097b858761bbeb5a0f01682ed30b0b
+SHA256:
+  metadata.gz: 194ee210390aeeb11282e5d7691bb4694d1978fd760abc8eb1ef078864ecc362
+  data.tar.gz: 8ccbbcff3e50c691a36f6fdaf66122882af8daa0201ff137218aa28018e31f84
 SHA512:
-  metadata.gz: 495c1d1fe28618d1e131a1c540c0143bb4935e80f5f86393bd5504a38fbb04c353bb859e59f6c19773e69d54165094f4a8db2f9d0be20d49565fb83ee836d06c
-  data.tar.gz: de2ac2e4860d441bbd8bc9e3bdaae98cc97aa456b83fc051b7706c8885ef6e56c90b8f0b3905e46156592d346728d6fd89493a02577ec467f011495b082bcb1c
+  metadata.gz: 39823bce21570d0b8721e7427589a6b85f811f70d79e9a61b959e6408fff607d73786e1e2d431f912d79892fcf41c892cb4dd6d67ba2a26f9b923f747d2beb05
+  data.tar.gz: 12355bccdcec5a37420a50f81e295c3778b233ecba77748bf88fcbe4c9d042d519550a26f9143ad98401ff541c1810bceefe0b5ee41f9e196f2c42323e1662d6

data/README.md

@@ -1,19 +1,16 @@
 # systemd plugin for [Fluentd](http://github.com/fluent/fluentd)
 
-[![Build Status](https://travis-ci.org/reevoo/fluent-plugin-systemd.svg?branch=master)](https://travis-ci.org/reevoo/fluent-plugin-systemd) [![Code Climate GPA](https://codeclimate.com/github/reevoo/fluent-plugin-systemd/badges/gpa.svg)](https://codeclimate.com/github/reevoo/fluent-plugin-systemd) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-systemd.svg)](https://rubygems.org/gems/fluent-plugin-systemd)
+[![Build Status](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd/actions/workflows/ci.yml/badge.svg)](https://github.com/fluent-plugin-systemd/fluent-plugin-systemd/actions/workflows/ci.yml) [![Maintainability](https://api.codeclimate.com/v1/badges/5c773a4c1c6a6964fa4b/maintainability)](https://codeclimate.com/github/fluent-plugin-systemd/fluent-plugin-systemd/maintainability) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-systemd.svg)](https://rubygems.org/gems/fluent-plugin-systemd)
 
 ## Overview
 
-**systemd** input plugin reads logs from the systemd journal  
-**systemd** filter plugin allows for basic manipulation of systemd journal entries
+* **systemd** input plugin to read logs from the systemd journal
+* **systemd** filter plugin for basic manipulation of systemd journal entries
 
 ## Support
 
-[![Fluentd Slack](http://slack.fluentd.org/badge.svg)](http://slack.fluentd.org/)
-
 Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org/)
 
-
 ## Requirements
 
 |fluent-plugin-systemd|fluentd|td-agent|ruby|
@@ -21,51 +18,58 @@ Join the #plugin-systemd channel on the [Fluentd Slack](http://slack.fluentd.org
 | > 0.1.0 | >= 0.14.11, < 2 | 3 | >= 2.1 |
 | 0.0.x | ~> 0.12.0       | 2 | >= 1.9  |
 
-* The 0.x.x series is developed from this branch (master)
-* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x)
-* The next version is currently under development on the [1.0.0 branch](https://github.com/reevoo/fluent-plugin-systemd/tree/1.0.0) it's progress is tracked [here](https://github.com/reevoo/fluent-plugin-systemd/issues/53)
+* The 1.x.x series is developed from this branch (master)
+* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is maintained on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x)
 
 ## Installation
 
 Simply use RubyGems:
 
-    gem install fluent-plugin-systemd -v 0.3.1
+    gem install fluent-plugin-systemd -v 1.0.3
 
 or
 
-    td-agent-gem install fluent-plugin-systemd -v 0.3.1
+    td-agent-gem install fluent-plugin-systemd -v 1.0.3
+
+## Upgrading
+
+If you are upgrading to version 1.0 from a previous version of this plugin take a look at the [upgrade documentation](docs/upgrading.md). A number of deprecated config options were removed so you might need to update your configuration.
 
 ## Input Plugin Configuration
 
     <source>
       @type systemd
-      tag kube-proxy
+      tag kubelet
       path /var/log/journal
-      matches [{ "_SYSTEMD_UNIT": "kube-proxy.service" }]
+      matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
       read_from_head true
+
       <storage>
         @type local
-        persistent false
-        path kube-proxy.pos
+        path /var/log/fluentd-journald-kubelet-cursor.json
       </storage>
+
       <entry>
-        field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
         fields_strip_underscores true
         fields_lowercase true
       </entry>
     </source>
 
-    <match kube-proxy>
+    <match kubelet>
       @type stdout
     </match>
 
+    <system>
+      root_dir /var/log/fluentd
+    </system>
+
 **`path`**
 
 Path to the systemd journal, defaults to `/var/log/journal`
 
 **`filters`**
 
-_This parameter name is deprecated and should be renamed to `matches`_
+_This parameter name is depreciated and should be renamed to `matches`_
 
 **`matches`**
 
@@ -73,40 +77,21 @@ Expects an array of hashes defining desired matches to filter the log
 messages with. When this property is not specified, this plugin will default to
 reading all logs from the journal.
 
-See [matching details](docs/Matching-Details.md) for a more exhaustive
+See [matching details](docs/matching.md) for a more exhaustive
 description of this property and how to use it.
 
-**`pos_file`**
-
-_This parameter is deprecated and will be removed in favour of storage in v1.0._
-
-
-Path to pos file, stores the journald cursor. File is created if does not exist.
-
 **`storage`**
 
-Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/storage-plugin-overview) used to store the journald cursor.
-
-_Upgrading from `pos_file`_
-
-If `pos_file` is specified in addition to a storage plugin with persistent set to true, the cursor will be
-copied from the `pos_file` on startup, and the old `pos_file` removed.
+Configuration for a [storage plugin](https://docs.fluentd.org/storage) used to store the journald cursor.
 
 **`read_from_head`**
 
 If true reads all available journal from head, otherwise starts reading from tail,
- ignored if pos file exists (and is valid). Defaults to false.
-
-**`strip_underscores`**
-
-_This parameter is deprecated and will be removed in favour of entry in v1.0._
-
-If true strips underscores from the beginning of systemd field names.
-May be useful if outputting to kibana, as underscore prefixed fields are unindexed there.
+ ignored if cursor exists in storage (and is valid). Defaults to false.
 
 **`entry`**
 
-Optional configuration for an embeded systemd entry filter. See the  [Filter Plugin Configuration](#filter-plugin-configuration) for config reference.
+Optional configuration for an embedded systemd entry filter. See the  [Filter Plugin Configuration](#filter-plugin-configuration) for config reference.
 
 **`tag`**
 
@@ -114,19 +99,20 @@ _Required_
 
 A tag that will be added to events generated by this input.
 
-### Example
-
-For an example of a full working setup including the plugin, [take a look at](https://github.com/assemblyline/fluentd)
 
 ## Filter Plugin Configuration
 
-    <filter kube-proxy>
-      @type systemd_entry
-      field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
-      field_map_strict false
-      fields_lowercase true
-      fields_strip_underscores true
-    </filter>
+```
+<filter kube-proxy>
+  @type systemd_entry
+  field_map {"MESSAGE": "log", "_PID": ["process", "pid"], "_CMDLINE": "process", "_COMM": "cmd"}
+  field_map_strict false
+  fields_lowercase true
+  fields_strip_underscores true
+</filter>
+```
+
+_Note that the following configurations can be embedded in a systemd source block, within an entry block, you only need to use a filter directly for more complicated workflows._
 
 **`field_map`**
 
@@ -151,7 +137,7 @@ If true, lowercase all non-mapped fields. Defaults to false.
 
 If true, strip leading underscores from all non-mapped fields. Defaults to false.
 
-### Example
+### Filter Example
 
 Given a systemd journal source entry:
 ```
@@ -178,26 +164,73 @@ The resulting entry using the above sample configuration:
 
 ## Common Issues
 
-> ### When I look at fluentd logs, everything looks fine but no journal logs are read
+> ### When I look at fluentd logs, everything looks fine but no journal logs are read ?
 
-This is commonly caused when the user running fluentd does not have enough permisions
+This is commonly caused when the user running fluentd does not have the correct permissions
 to read the systemd journal.
 
-Acording to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html):
+According to the [systemd documentation](https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html):
 > Journal files are, by default, owned and readable by the "systemd-journal" system group but are not writable. Adding a user to this group thus enables her/him to read the journal files.
 
+> ### How can I deal with multi-line logs ?
+
+Ideally you want to ensure that your logs are saved to the systemd journal as a single entry regardless of how many lines they span.
+
+It is possible for applications to naively support this (but only if they have tight integration with systemd it seems) see: https://github.com/systemd/systemd/issues/5188.
+
+Typically you would not be able to this, so another way is to configure your logger to replace newline characters with something else. See this blog post for an example configuring a Java logging library to do this https://fabianlee.org/2018/03/09/java-collapsing-multiline-stack-traces-into-a-single-log-event-using-spring-backed-by-logback-or-log4j2/
+
+Another strategy would be to use a plugin like [fluent-plugin-concat](https://github.com/fluent-plugins-nursery/fluent-plugin-concat) to combine multi line logs into a single event, this is more tricky though because you need to be able to identify the first and last lines of a multi line message with a regex.
+
+> ### How can I use this plugin inside of a docker container ?
+
+* Install the [systemd dependencies](#dependencies) if required
+* You can use an [offical fluentd docker](https://github.com/fluent/fluentd-docker-image) image as a base, (choose the debian based version, as alpine linux doesn't support systemd).
+* Bind mount `/var/log/journal` into your container.
+
+> ### I am seeing lots of logs being generated very rapidly!
+
+This commonly occurs when a loop is created when fluentd is logging to STDOUT, and the collected logs are then written to the systemd journal. This could happen if you run fluentd as a systemd serivce, or as a docker container with the systemd log driver.
+
+Workarounds include:
+
+* Use another fluentd output
+* Don't read every message from the journal, set some `matches` so you only read the messages you are interested in.
+* Disable the systemd log driver when you launch your fluentd docker container, e.g. by passing `--log-driver json-file`
+
+### Example
+
+For an example of a full working setup including the plugin, take a look at [the fluentd kubernetes daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset)
 
 ## Dependencies
 
 This plugin depends on libsystemd
 
+On Debian or Ubuntu you might need to install the libsystemd0 package:
+
+```
+apt-get install libsystemd0
+```
+
+On CentOS or RHEL you might need to install the systemd package:
+
+```
+yum install -y systemd
+```
+
+If you want to do this in a CentOS docker image you might first need to remove the `fakesystemd` package.
+
+```
+yum remove -y fakesystemd
+```
+
 ## Running the tests
 
 To run the tests with docker on several distros simply run `rake`
 
 For systems with systemd installed you can run the tests against your installed libsystemd with `rake test`
 
-## Licence
+## License
 
 [Apache-2.0](LICENCE)
 
@@ -217,14 +250,16 @@ We have adopted the [Contributor Covenant](CODE_OF_CONDUCT.md) and thus expect a
 
 Many thanks to our fantastic contributors
 
-* [Hiroshi Hatake](https://github.com/cosmo0920)
 * [Erik Maciejewski](https://github.com/emacski)
-* [Masahiro Nakagawa](https://github.com/repeatedly)
-* [Richard Megginson](https://github.com/richm)
-* [Mike Kaplinskiy](https://github.com/mikekap)
-* [neko-neko](https://github.com/neko-neko)
-* [Sadayuki Furuhashi](https://github.com/frsyuki)
+* [Ernie Hershey](https://github.com/ehershey)
+* [Hiroshi Hatake](https://github.com/cosmo0920)
 * [Jesus Rafael Carrillo](https://github.com/jescarri)
+* [Joel Gerber](https://github.com/Jitsusama)
 * [John Thomas Wile II](https://github.com/jtwile2)
 * [Kazuhiro Suzuki](https://github.com/ksauzz)
-* [Joel Gerber](https://github.com/Jitsusama)
+* [Marius Grigoriu](https://github.com/mariusgrigoriu)
+* [Masahiro Nakagawa](https://github.com/repeatedly)
+* [Mike Kaplinskiy](https://github.com/mikekap)
+* [Richard Megginson](https://github.com/richm)
+* [Sadayuki Furuhashi](https://github.com/frsyuki)
+* [Seiichi Nishikata](https://github.com/neko-neko)

data/lib/fluent/plugin/in_systemd.rb

@@ -57,18 +57,21 @@ module Fluent
 
       def start
         super
+        @running = true
         timer_execute(:in_systemd_emit_worker, 1, &method(:run))
       end
 
+      def shutdown
+        @running = false
+        super
+      end
+
       private
 
       def init_journal
         # TODO: ruby 2.3
         @journal.close if @journal # rubocop:disable Style/SafeNavigation
         @journal = Systemd::Journal.new(path: @path)
-        # make sure initial call to wait doesn't return :invalidate
-        # see https://github.com/ledbettj/systemd-journal/issues/70
-        @journal.wait(0)
         @journal.filter(*(@matches || @filters))
         seek
         true
@@ -95,6 +98,7 @@ module Fluent
       def seek_to(pos)
         @journal.seek(pos)
         return if pos == :head
+
         if pos == :tail
           @journal.move(-2)
         else
@@ -108,6 +112,7 @@ module Fluent
 
       def run
         return unless @journal || init_journal
+
         init_journal if @journal.wait(0) == :invalidate
         watch do |entry|
           emit(entry)
@@ -119,6 +124,7 @@ module Fluent
       rescue Fluent::Plugin::Buffer::BufferOverflowError => e
         retries ||= 0
         raise e if retries > 10
+
         retries += 1
         sleep 1.5**retries + rand(0..3)
         retry
@@ -131,7 +137,9 @@ module Fluent
       end
 
       def watch(&block)
-        yield_current_entry(&block) while @journal.move_next
+        yield_current_entry(&block) while @running && @journal.move_next
+      rescue Systemd::JournalError => e
+        log.warn("Error moving to next Journal entry: #{e.class}: #{e.message}")
       end
 
       def yield_current_entry

data/lib/fluent/plugin/systemd/entry_mutator.rb

@@ -67,6 +67,7 @@ module Fluent
       # Expose config state as read-only instance properties of the mutator.
       def method_missing(sym, *args)
         return @opts[sym] if @opts.members.include?(sym)
+
         super
       end
 
@@ -80,6 +81,7 @@ module Fluent
       def run(entry)
         return entry.to_h if @no_transform
         return map_fields(entry) if @opts.field_map_strict
+
         format_fields(entry, map_fields(entry))
       end
 
@@ -90,6 +92,7 @@ module Fluent
         @map.each_with_object({}) do |(cstm, sysds), mapped|
           vals = sysds.collect { |fld| entry[fld] }.compact
           next if vals.empty? # systemd field does not exist in source entry
+
           mapped[cstm] = join_if_needed(vals)
         end
       end
@@ -103,6 +106,7 @@ module Fluent
         entry.each_with_object(mapped || {}) do |(fld, val), formatted_entry|
           # don't mess with explicitly mapped fields
           next if @map_src_fields.include?(fld)
+
           fld = format_field_name(fld)
           # account for mapping (appending) to an existing systemd field
           formatted_entry[fld] = join_if_needed([val, mapped[fld]])
@@ -111,6 +115,7 @@ module Fluent
 
       def warnings
         return [] unless field_map_strict && field_map.empty?
+
         '`field_map_strict` set to true with empty `field_map`, expect no fields'
       end
 
@@ -119,6 +124,7 @@ module Fluent
       def join_if_needed(values)
         values.compact!
         return values.first if values.length == 1
+
         values.join(' ')
       end
 
@@ -146,7 +152,7 @@ module Fluent
         end
       end
 
-      def validate_all_strings(arr, message, allow_nesting = false)
+      def validate_all_strings(arr, message, allow_nesting = false) # rubocop:disable Style/OptionalBooleanParameter
         valid = arr.all? do |value|
           value.is_a?(String) || allow_nesting && value.is_a?(Array) && value.all? { |key| key.is_a?(String) }
         end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-systemd
 version: !ruby/object:Gem::Version
-  version: 1.0.0.rc3
+  version: 1.0.4
 platform: ruby
 authors:
 - Ed Robinson
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-03 00:00:00.000000000 Z
+date: 2021-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
@@ -44,28 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.5'
+        version: '3.4'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.53.0
+        version: 1.13.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: 0.53.0
+        version: 1.13.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -92,14 +92,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 1.4.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: 1.4.2
 description: This is a fluentd input plugin. It reads logs from the systemd journal.
 email:
 - edward-robinson@cookpad.com
@@ -116,7 +116,7 @@ homepage: https://github.com/reevoo/fluent-plugin-systemd
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,13 +127,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.2.15
+signing_key:
 specification_version: 4
 summary: Input plugin to read from systemd journal.
 test_files: []