fluent-plugin-systemd 0.3.1 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c2cebb67ce9b89e20f0354bdbb45f7f3c8f1c817
-  data.tar.gz: b59f85ccd7e8550cdf7835c2fa3449406e9c0769
+  metadata.gz: 13f78fa020542fb3c022c6d0870ecf1ec2d307ae
+  data.tar.gz: 5cac751f2b33bc5783bf3a721999890e9f2c51ba
 SHA512:
-  metadata.gz: 8a115c1eb388b1263f933d1f3283c84b6dbbfc4093756842095879c7f07312c0562ccbf87093fb4defd643e17cc5219569c795a48e82f3e79200990d8b22360f
-  data.tar.gz: ba30dc865abe28b552379c7a680f6ffff3303990d98eea0747d4bda5810a585cbb866c9410a57302e6f4e9e5385d33894c0dacaca192fec4c207f5dc2e17917c
+  metadata.gz: 89677dd5dbf1c6648e37ec969590392cf54a929ae15d30a64824df1b3a93f1a7ea54f71c83839f484bcc914f5ffbaf8b2b32c05320e1dc45084ea30ec48afdae
+  data.tar.gz: dcd1c5f2eaac7c766344bbbae29d7fc8fb10cc951cc7dfbb07982ae5946dc8eef6ec26593b8ade6bac1c8fe981b78bb5f291982b9941a6ef02a85801919893c8

data/README.md

@@ -44,7 +44,7 @@ or
       read_from_head true
       <storage>
         @type local
-        persistent true
+        persistent false
         path kube-proxy.pos
       </storage>
       <entry>
@@ -196,7 +196,7 @@ For systems with systemd installed you can run the tests against your installed
 
 Issues and pull requests are very welcome.
 
-If you want to make a contribution but need some help or advice feel free to message me @errm on the [Fluentd Slack](http://slack.fluentd.org/), or send me an email ed@reevoo.com.
+If you want to make a contribution but need some help or advice feel free to message me @errm on the [Fluentd Slack](http://slack.fluentd.org/), or send me an email edward-robinson@cookpad.com
 
 We have adopted the [Contributor Covenant](CODE_OF_CONDUCT.md) and thus expect anyone interacting with contributors, maintainers and users of this project to abide by it.
 
@@ -216,3 +216,5 @@ Many thanks to our fantastic contributors
 * [neko-neko](https://github.com/neko-neko)
 * [Sadayuki Furuhashi](https://github.com/frsyuki)
 * [Jesus Rafael Carrillo](https://github.com/jescarri)
+* [John Thomas Wile II](https://github.com/jtwile2)
+* [Kazuhiro Suzuki](https://github.com/ksauzz)

data/lib/fluent/plugin/filter_systemd_entry.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
-require "fluent/plugin/filter"
-require "fluent/plugin/systemd/entry_mutator"
+
+require 'fluent/plugin/filter'
+require 'fluent/plugin/systemd/entry_mutator'
 
 module Fluent
   module Plugin
     # Fluentd systemd/journal filter plugin
     class SystemdEntryFilter < Filter
-      Fluent::Plugin.register_filter("systemd_entry", self)
+      Fluent::Plugin.register_filter('systemd_entry', self)
 
       config_param :field_map, :hash, default: {}
       config_param :field_map_strict, :bool, default: false
@@ -16,9 +17,7 @@ module Fluent
       def configure(conf)
         super
         @mutator = SystemdEntryMutator.new(**@config_root_section.to_h)
-        if @mutator.field_map_strict && @mutator.field_map.empty?
-          log.warn("`field_map_strict` set to true with empty `field_map`, expect no fields")
-        end
+        @mutator.warnings.each { |warning| log.warn(warning) }
       end
 
       def filter(_tag, _time, entry)

data/lib/fluent/plugin/in_systemd.rb

@@ -1,33 +1,31 @@
 # frozen_string_literal: true
-require "systemd/journal"
-require "fluent/plugin/input"
-require "fluent/plugin/systemd/pos_writer"
-require "fluent/plugin/systemd/entry_mutator"
+
+require 'systemd/journal'
+require 'fluent/plugin/input'
+require 'fluent/plugin/systemd/entry_mutator'
 
 module Fluent
   module Plugin
-    class SystemdInput < Input # rubocop:disable Metrics/ClassLength
-      Fluent::Plugin.register_input("systemd", self)
+    # Fluentd plugin for reading from the systemd journal
+    class SystemdInput < Input
+      Fluent::Plugin.register_input('systemd', self)
 
       helpers :timer, :storage
 
-      DEFAULT_STORAGE_TYPE = "local"
+      DEFAULT_STORAGE_TYPE = 'local'
 
-      config_param :path, :string, default: "/var/log/journal"
+      config_param :path, :string, default: '/var/log/journal'
       config_param :filters, :array, default: []
-      config_param :pos_file, :string, default: nil, deprecated: "Use <storage> section with `persistent: true' instead"
       config_param :read_from_head, :bool, default: false
-      config_param :strip_underscores, :bool, default: false, deprecated: "Use <entry> section or `systemd_entry` " \
-                                                                          "filter plugin instead"
       config_param :tag, :string
 
       config_section :storage do
-        config_set_default :usage, "positions"
+        config_set_default :usage, 'positions'
         config_set_default :@type, DEFAULT_STORAGE_TYPE
         config_set_default :persistent, false
       end
 
-      config_section :entry, param_name: "entry_opts", required: false, multi: false do
+      config_section :entry, param_name: 'entry_opts', required: false, multi: false do
         config_param :field_map, :hash, default: {}
         config_param :field_map_strict, :bool, default: false
         config_param :fields_strip_underscores, :bool, default: false
@@ -37,27 +35,16 @@ module Fluent
       def configure(conf)
         super
         @journal = nil
-        @pos_storage = PosWriter.new(@pos_file, storage_create(usage: "positions"))
-        # legacy strip_underscores backwards compatibility (legacy takes
-        # precedence and is mutually exclusive with the entry block)
-        mut_opts = @strip_underscores ? { fields_strip_underscores: true } : @entry_opts.to_h
-        @mutator = SystemdEntryMutator.new(**mut_opts)
-        if @mutator.field_map_strict && @mutator.field_map.empty?
-          log.warn("`field_map_strict` set to true with empty `field_map`, expect no fields")
-        end
+        @pos_storage = storage_create(usage: 'positions')
+        @mutator = SystemdEntryMutator.new(**@entry_opts.to_h)
+        @mutator.warnings.each { |warning| log.warn(warning) }
       end
 
       def start
         super
-        @pos_storage.start
         timer_execute(:in_systemd_emit_worker, 1, &method(:run))
       end
 
-      def shutdown
-        @pos_storage.shutdown
-        super
-      end
-
       private
 
       def init_journal
@@ -80,8 +67,8 @@ module Fluent
         seek_to(cursor || read_from)
       rescue Systemd::JournalError
         log.warn(
-          "Could not seek to cursor #{cursor} found in pos file: #{@pos_storage.path}, " \
-          "falling back to reading from #{read_from}",
+          "Could not seek to cursor #{cursor} found in position file: #{@pos_storage.path}, " \
+          "falling back to reading from #{read_from}"
         )
         seek_to(read_from)
       end
@@ -120,7 +107,7 @@ module Fluent
         retries += 1
         sleep 1.5**retries + rand(0..3)
         retry
-      rescue => e
+      rescue => e # rubocop:disable Style/RescueStandardError
         log.error("Exception emitting record: #{e}")
       end
 

data/lib/fluent/plugin/systemd/entry_mutator.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-require "fluent/config/error"
+
+require 'fluent/config/error'
 
 module Fluent
   module Plugin
@@ -19,12 +20,11 @@ module Fluent
     #   "<new_field2>" => ["<source_field2>"]
     # }
     class SystemdEntryMutator
-
       Options = Struct.new(
         :field_map,
         :field_map_strict,
         :fields_lowercase,
-        :fields_strip_underscores,
+        :fields_strip_underscores
       )
 
       def self.default_opts
@@ -56,6 +56,10 @@ module Fluent
         super
       end
 
+      def respond_to_missing?(sym, include_private = false)
+        @opts.members.include?(sym) || super
+      end
+
       # The main run method that performs all configured mutations, if any,
       # against a single journal entry. Returns the mutated entry hash.
       # entry - hash or `Systemd::Journal:Entry`
@@ -69,13 +73,11 @@ module Fluent
       # entry hash.
       # entry - hash or `Systemd::Journal:Entry`
       def map_fields(entry)
-        mapped = {}
-        @map.each do |cstm, sysds|
+        @map.each_with_object({}) do |(cstm, sysds), mapped|
           vals = sysds.collect { |fld| entry[fld] }.compact
           next if vals.empty? # systemd field does not exist in source entry
-          mapped[cstm] = vals.length == 1 ? vals[0] : vals.join(" ")
+          mapped[cstm] = join_if_needed(vals)
         end
-        mapped
       end
 
       # Run field formatting (mutations applied to all non-mapped fields)
@@ -84,20 +86,34 @@ module Fluent
       # mapped - Optional hash that represents a previously mapped entry to
       #          which the formatted fields will be added
       def format_fields(entry, mapped = nil)
-        mapped ||= {}
-        entry.each do |fld, val|
+        entry.each_with_object(mapped || {}) do |(fld, val), formatted_entry|
           # don't mess with explicitly mapped fields
           next if @map_src_fields.include?(fld)
-          fld = fld.gsub(/\A_+/, "") if @opts.fields_strip_underscores
-          fld = fld.downcase if @opts.fields_lowercase
+          fld = format_field_name(fld)
           # account for mapping (appending) to an existing systemd field
-          mapped[fld] = mapped.key?(fld) ? [val, mapped[fld]].join(" ") : val
+          formatted_entry[fld] = join_if_needed([val, mapped[fld]])
         end
-        mapped
+      end
+
+      def warnings
+        return [] unless field_map_strict && field_map.empty?
+        '`field_map_strict` set to true with empty `field_map`, expect no fields'
       end
 
       private
 
+      def join_if_needed(values)
+        values.compact!
+        return values.first if values.length == 1
+        values.join(' ')
+      end
+
+      def format_field_name(name)
+        name = name.gsub(/\A_+/, '') if @opts.fields_strip_underscores
+        name = name.downcase if @opts.fields_lowercase
+        name
+      end
+
       # Returns a `SystemdEntryMutator::Options` struct derived from the
       # elements in the supplied hash merged with the option defaults
       def options_from_hash(opts)
@@ -109,39 +125,32 @@ module Fluent
       end
 
       def validate_options(opts)
-        unless validate_strings_or_empty(opts[:field_map].keys)
-          err = "`field_map` keys must be strings"
-        end
-        unless validate_strings_or_empty(opts[:field_map].values, true)
-          err = "`field_map` values must be strings or array of strings"
-        end
+        validate_all_strings opts[:field_map].keys, '`field_map` keys must be strings'
+        validate_all_strings opts[:field_map].values, '`field_map` values must be strings or an array of strings', true
         %i[field_map_strict fields_strip_underscores fields_lowercase].each do |opt|
-          err = "`#{opt}` must be boolean" unless [true, false].include?(opts[opt])
+          validate_boolean opts[opt], opt
         end
-        fail Fluent::ConfigError, err unless err.nil?
       end
 
-      # Validates that values in array `arr` are strings. If `nested` is true
-      # also allow and validate that `arr` values can be an array of strings
-      def validate_strings_or_empty(arr, nested = false)
-        return true if arr.empty?
-        arr.each do |v|
-          return true if v.is_a?(String)
-          if v.is_a?(Array) && nested
-            v.each { |nstd| return false unless nstd.is_a?(String) }
-          end
+      def validate_all_strings(arr, message, allow_nesting = false)
+        valid = arr.all? do |value|
+          value.is_a?(String) || allow_nesting && value.is_a?(Array) && value.all? { |key| key.is_a?(String) }
         end
-        false
+        raise Fluent::ConfigError, message unless valid
+      end
+
+      def validate_boolean(value, name)
+        raise Fluent::ConfigError, "`#{name}` must be boolean" unless [true, false].include?(value)
       end
 
-      # Compute the inverse of a human friendly field map `fm` which is what
+      # Compute the inverse of a human friendly field map `field_map` which is what
       # the mutator uses for the actual mapping. The resulting structure for
       # the inverse field map hash is:
       # {"<new_field_name>" => ["<source_field_name>", ...], ...}
-      def invert_field_map(fm)
+      def invert_field_map(field_map)
         invs = {}
-        fm.values.flatten.uniq.each do |cstm|
-          sysds = fm.select { |_, v| (v == cstm || v.include?(cstm)) }
+        field_map.values.flatten.uniq.each do |cstm|
+          sysds = field_map.select { |_, v| (v == cstm || v.include?(cstm)) }
           invs[cstm] = sysds.keys
         end
         invs

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-systemd
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Ed Robinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-03 00:00:00.000000000 Z
+date: 2018-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,19 +53,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.5'
 - !ruby/object:Gem::Dependency
-  name: reevoocop
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.53.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.53.0
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -102,7 +102,7 @@ dependencies:
         version: '1.3'
 description: This is a fluentd input plugin. It reads logs from the systemd journal.
 email:
-- ed@reevoo.com
+- edward-robinson@cookpad.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -112,7 +112,6 @@ files:
 - lib/fluent/plugin/filter_systemd_entry.rb
 - lib/fluent/plugin/in_systemd.rb
 - lib/fluent/plugin/systemd/entry_mutator.rb
-- lib/fluent/plugin/systemd/pos_writer.rb
 homepage: https://github.com/reevoo/fluent-plugin-systemd
 licenses:
 - MIT
@@ -128,12 +127,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Input plugin to read from systemd journal.

data/lib/fluent/plugin/systemd/pos_writer.rb

@@ -1,80 +0,0 @@
-# frozen_string_literal: true
-require "fluent/plugin/input"
-
-module Fluent
-  module Plugin
-    class SystemdInput < Input
-      class PosWriter
-        def initialize(pos_file, storage)
-          @path = pos_file
-          @lock = Mutex.new
-          @storage = storage
-          @cursor = nil
-          @written_cursor = nil
-          setup
-        end
-
-        def get(key)
-          @storage ? @storage.get(key) : @cursor
-        end
-
-        def put(key, cursor)
-          return @storage.put(key, cursor) if @storage
-          @lock.synchronize { @cursor = cursor }
-        end
-
-        def path
-          @path || @storage.path
-        end
-
-        def start
-          return unless @path
-          @running = true
-          @thread = Thread.new(&method(:work))
-        end
-
-        def shutdown
-          return unless @path
-          @running = false
-          @thread.join
-          write_pos
-        end
-
-        private
-
-        def setup
-          if @storage.persistent
-            migrate_to_storage if @path && File.exist?(@path)
-          elsif @path
-            @cursor = IO.read(@path).chomp if File.exist?(@path)
-            @storage = nil
-          end
-        end
-
-        def migrate_to_storage
-          @storage.put(:journal, IO.read(@path).chomp)
-          File.delete(@path)
-          @path = nil
-        end
-
-        def work
-          while @running
-            write_pos
-            sleep 1
-          end
-        end
-
-        def write_pos
-          @lock.synchronize do
-            if @written_cursor != @cursor
-              file = File.open(@path, "w+", 0o644)
-              file.print @cursor
-              file.close
-              @written_cursor = @cursor
-            end
-          end
-        end
-      end
-    end
-  end
-end