fluent-plugin-systemd 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 027149ee076af73b625bdf8dcd22dd25a7f0e833
-  data.tar.gz: b729df398f867f111fd1a3f188eb7298437195c4
+  metadata.gz: 07f12e01a1d81c731f44477a051807042230adfa
+  data.tar.gz: 8717ea703cd39fb947af86b5773a68de5b55c040
 SHA512:
-  metadata.gz: 979f117f454ba7a4d72ce0c62fa82fa5cae5c18be170443a4d77160c2b6f58b2f2c7de8032d6abda96b4070ef83bdb6bf94bd6746277b56984550af1157821ff
-  data.tar.gz: 8cbcef9a6b30dee72ee515140ede68aa7317d3c9b8d338bdb76c84897d5eae0030ba810f4b840e8d902b8aab9573d58cb81314d2e5eb5b2d71eff9b4da894a60
+  metadata.gz: 92b3275665979af1a9d691f05acdc65a88fd88b062c6224673cde0c691c9c7a4376a1c97c96c7e1dc9cc54f224fe27e7c39600bc0b6307c88ad182e442971520
+  data.tar.gz: 110c2d6ff7ebeac0eb96ceaf0d470d5bdfdf5da2e4556f8b2fc98b213e08b87553d231073f2828a5050dc88437f307e2846cea0ebfd39c131465a34b83123e5e

data/README.md

@@ -2,6 +2,17 @@
 
 [![Build Status](https://travis-ci.org/reevoo/fluent-plugin-systemd.svg?branch=master)](https://travis-ci.org/reevoo/fluent-plugin-systemd) [![Code Climate GPA](https://codeclimate.com/github/reevoo/fluent-plugin-systemd/badges/gpa.svg)](https://codeclimate.com/github/reevoo/fluent-plugin-systemd) [![Gem Version](https://badge.fury.io/rb/fluent-plugin-systemd.svg)](https://rubygems.org/gems/fluent-plugin-systemd)
 
+# Requirements <a name="requirements"></a>
+
+
+|fluent-plugin-systemd|fluentd|td-agent|ruby|
+|----|----|----|----|
+| 0.2.x | >= 0.14.11, < 2 | 3 | >= 2.1 |
+| 0.0.x | ~> 0.12.0       | 2 | >= 1.9  |
+
+* The 0.1.x series is developed from this branch (master)
+* The 0.0.x series (compatible with fluentd v0.12, and td-agent 2) is developed on the [0.0.x branch](https://github.com/reevoo/fluent-plugin-systemd/tree/0.0.x)
+
 ## Overview
 
 **systemd** input plugin reads logs from the systemd journal
@@ -10,15 +21,11 @@
 
 Simply use RubyGems:
 
-    gem install fluent-plugin-systemd -v 0.1.0
+    gem install fluent-plugin-systemd -v 0.2.0
 
-    or
+or
 
-    fluent-gem install fluent-plugin-systemd -v 0.1.0
-
-    or
-
-    td-agent-gem install fluent-plugin-systemd -v 0.1.0
+    td-agent-gem install fluent-plugin-systemd -v 0.2.0
 
 ## Configuration
 
@@ -26,37 +33,59 @@ Simply use RubyGems:
       @type systemd
       path /var/log/journal
       filters [{ "_SYSTEMD_UNIT": "kube-proxy.service" }]
-      pos_file kube-proxy.pos
+      <storage>
+        @type local
+        persistent true
+        path kube-proxy.pos
+      </storage>
       tag kube-proxy
       read_from_head true
     </source>
 
-**path**
+    <match kube-proxy>
+      @type stdout
+    </match>
+
+**`path`**
 
 Path to the systemd journal, defaults to `/var/log/journal`
 
-**filters**
+**`filters`**
 
 Array of filters, see [here](http://www.rubydoc.info/gems/systemd-journal/Systemd%2FJournal%2FFilterable%3Afilter) for further
 documentation, defaults to no filtering.
 
-**pos file**
+**`pos_file`**
+
+_This parameter is deprecated and will be removed in favour of storage in v1.0._
+
 
 Path to pos file, stores the journald cursor. File is created if does not exist.
 
-**read_from_head**
+**`storage`**
+
+Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/storage-plugin-overview) used to store the journald cursor.
+
+_Upgrading from `pos_file`_
+
+If `pos_file` is specified in addition to a storage plugin with persistent set to true, the cursor will be
+copied from the `pos_file` on startup, and the old `pos_file` removed.
+
+**`read_from_head`**
 
 If true reads all available journal from head, otherwise starts reading from tail,
  ignored if pos file exists (and is valid). Defaults to false.
 
-**strip_underscores**
+**`strip_underscores`**
 
 If true strips underscores from the beginning of systemd field names.
 May be useful if outputting to kibana, as underscore prefixed fields are unindexed there.
 
-**tag**
+**`tag`**
+
+_Required_ 
 
-_Required_ A tag that will be added to events generated by this input.
+A tag that will be added to events generated by this input.
 
 ## Example
 
@@ -92,3 +121,4 @@ Many thanks to our fantastic contributors
 * [Richard Megginson](https://github.com/richm)
 * [Masahiro Nakagawa](https://github.com/repeatedly)
 * [Hiroshi Hatake](https://github.com/cosmo0920)
+* [neko-neko](https://github.com/neko-neko)

data/lib/fluent/plugin/in_systemd.rb

@@ -7,40 +7,63 @@ module Fluent
     class SystemdInput < Input
       Fluent::Plugin.register_input("systemd", self)
 
-      helpers :timer
+      helpers :timer, :storage
+
+      DEFAULT_STORAGE_TYPE = "local".freeze
 
       config_param :path, :string, default: "/var/log/journal"
       config_param :filters, :array, default: []
-      config_param :pos_file, :string, default: nil
+      config_param :pos_file, :string, default: nil, deprecated: "Use <storage> section with `persistent: true' instead"
       config_param :read_from_head, :bool, default: false
       config_param :strip_underscores, :bool, default: false
       config_param :tag, :string
 
+      config_section :storage do
+        config_set_default :usage, "positions"
+        config_set_default :@type, DEFAULT_STORAGE_TYPE
+        config_set_default :persistent, false
+      end
+
       def configure(conf)
         super
-        @pos_writer = PosWriter.new(@pos_file)
-        @journal = Systemd::Journal.new(path: @path)
-        @journal.filter(*@filters)
-        seek
+        @pos_storage = PosWriter.new(@pos_file, storage_create(usage: "positions"))
+        @journal    = nil
       end
 
       def start
         super
-        @pos_writer.start
+        @pos_storage.start
         timer_execute(:in_systemd_emit_worker, 1, &method(:run))
       end
 
       def shutdown
-        @pos_writer.shutdown
+        @pos_storage.shutdown
         super
       end
 
       private
 
+      def init_journal
+        @journal = Systemd::Journal.new(path: @path)
+        # make sure initial call to wait doesn't return :invalidate
+        # see https://github.com/ledbettj/systemd-journal/issues/70
+        @journal.wait(0)
+        @journal.filter(*@filters)
+        seek
+        true
+      rescue Systemd::JournalError => e
+        log.warn("#{e.class}: #{e.message} retrying in 1s")
+        false
+      end
+
       def seek
-        seek_to(@pos_writer.cursor || read_from)
+        cursor = @pos_storage.get(:journal)
+        seek_to(cursor || read_from)
       rescue Systemd::JournalError
-        log.warn("Could not seek to cursor #{@pos_writer.cursor} found in pos file: #{@pos_writer.path}")
+        log.warn(
+          "Could not seek to cursor #{cursor} found in pos file: #{@pos_storage.path}, " \
+          "falling back to reading from #{read_from}",
+        )
         seek_to(read_from)
       end
 
@@ -50,8 +73,12 @@ module Fluent
       # record
       def seek_to(pos)
         @journal.seek(pos)
-        return unless pos == :tail
-        @journal.move(-2)
+        return if pos == :head
+        if pos == :tail
+          @journal.move(-2)
+        else
+          @journal.move(1)
+        end
       end
 
       def read_from
@@ -59,7 +86,8 @@ module Fluent
       end
 
       def run
-        Thread.current.abort_on_exception = true
+        return unless @journal || init_journal
+        init_journal if @journal.wait(0) == :invalidate
         watch do |entry|
           begin
             router.emit(@tag, Fluent::EventTime.from_time(entry.realtime_timestamp), formatted(entry))
@@ -77,7 +105,7 @@ module Fluent
       def watch
         while @journal.move_next
           yield @journal.current_entry
-          @pos_writer.update(@journal.cursor)
+          @pos_storage.put(:journal, @journal.cursor)
         end
       end
     end

data/lib/fluent/plugin/systemd/pos_writer.rb

@@ -4,15 +4,27 @@ module Fluent
   module Plugin
     class SystemdInput < Input
       class PosWriter
-        def initialize(pos_file)
+        def initialize(pos_file, storage)
           @path = pos_file
           @lock = Mutex.new
+          @storage = storage
           @cursor = nil
           @written_cursor = nil
           setup
         end
 
-        attr_reader :cursor, :path
+        def get(key)
+          @storage ? @storage.get(key) : @cursor
+        end
+
+        def put(key, cursor)
+          return @storage.put(key, cursor) if @storage
+          @lock.synchronize { @cursor = cursor }
+        end
+
+        def path
+          @path || @storage.path
+        end
 
         def start
           return unless @path
@@ -27,16 +39,21 @@ module Fluent
           write_pos
         end
 
-        def update(c)
-          return unless @path
-          @lock.synchronize { @cursor = c }
-        end
-
         private
 
         def setup
-          return unless @path && File.exist?(@path)
-          @cursor = IO.read(@path).chomp
+          if @storage.persistent
+            migrate_to_storage if @path && File.exist?(@path)
+          elsif @path
+            @cursor = IO.read(@path).chomp if File.exist?(@path)
+            @storage = nil
+          end
+        end
+
+        def migrate_to_storage
+          @storage.put(:journal, IO.read(@path).chomp)
+          File.delete(@path)
+          @path = nil
         end
 
         def work
@@ -49,7 +66,7 @@ module Fluent
         def write_pos
           @lock.synchronize do
             if @written_cursor != @cursor
-              file = File.open(@path, "w+")
+              file = File.open(@path, "w+", 0644)
               file.print @cursor
               file.close
               @written_cursor = @cursor

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-systemd
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Ed Robinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-12 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -107,17 +107,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- Gemfile
-- Gemfile.lock
 - LICENCE
 - README.md
-- Rakefile
-- fluent-plugin-systemd.gemspec
 - lib/fluent/plugin/in_systemd.rb
 - lib/fluent/plugin/systemd/pos_writer.rb
-- pkg/fluent-plugin-systemd-0.0.3.gem
-- pkg/fluent-plugin-systemd-0.0.4.gem
-- pkg/fluent-plugin-systemd-0.0.5.gem
 homepage: https://github.com/reevoo/fluent-plugin-systemd
 licenses:
 - MIT
@@ -138,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.9
 signing_key: 
 specification_version: 4
 summary: Input plugin to read from systemd journal.

data/Gemfile

@@ -1,3 +0,0 @@
-source "https://rubygems.org"
-
-gemspec

data/Gemfile.lock

@@ -1,67 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-systemd (0.1.0)
-      fluentd (>= 0.14.11, < 2)
-      systemd-journal (~> 1.2)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.3.0)
-    astrolabe (1.3.1)
-      parser (~> 2.2)
-    cool.io (1.4.5)
-    ffi (1.9.14)
-    fluentd (0.14.11)
-      cool.io (~> 1.4.5)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (~> 0.1.7)
-      tzinfo (~> 1.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.0.2)
-    parser (2.3.3.1)
-      ast (~> 2.2)
-    powerpack (0.0.9)
-    rainbow (2.2.1)
-    rake (10.5.0)
-    reevoocop (0.0.9)
-      rubocop (= 0.28.0)
-    rubocop (0.28.0)
-      astrolabe (~> 1.3)
-      parser (>= 2.2.0.pre.7, < 3.0)
-      powerpack (~> 0.0.6)
-      rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.4)
-    ruby-progressbar (1.8.1)
-    serverengine (2.0.4)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.1.9)
-    systemd-journal (1.2.3)
-      ffi (~> 1.9)
-    test-unit (2.5.5)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2016.10)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.3.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.10)
-  fluent-plugin-systemd!
-  rake
-  reevoocop
-  test-unit (~> 2.5)
-
-BUNDLED WITH
-   1.13.2

data/Rakefile

@@ -1,31 +0,0 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-require "reevoocop/rake_task"
-require "fileutils"
-
-ReevooCop::RakeTask.new(:reevoocop)
-
-Rake::TestTask.new(:test) do |t|
-  t.test_files = Dir["test/**/test_*.rb"]
-end
-
-task default: "docker:test"
-task build: "docker:test"
-task default: :reevoocop
-
-namespace :docker do
-  distros = [:ubuntu, :"tdagent-ubuntu", :"tdagent-centos"]
-  task test: distros
-
-  distros.each do |distro|
-    task distro do
-      puts "testing on #{distro}"
-      begin
-        FileUtils.cp("test/docker/Dockerfile.#{distro}", "Dockerfile")
-        sh "docker build ."
-      ensure
-        FileUtils.rm("Dockerfile")
-      end
-    end
-  end
-end

data/fluent-plugin-systemd.gemspec

@@ -1,28 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-systemd"
-  spec.version       = "0.1.0"
-  spec.authors       = ["Ed Robinson"]
-  spec.email         = ["ed@reevoo.com"]
-
-  spec.summary       = "Input plugin to read from systemd journal."
-  spec.description   = "This is a fluentd input plugin. It reads logs from the systemd journal."
-  spec.homepage      = "https://github.com/reevoo/fluent-plugin-systemd"
-  spec.license       = "MIT"
-
-
-  spec.files         = Dir["**/**"].reject { |f| f.match(/^(test|spec|features)\//) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "test-unit", "~> 2.5"
-  spec.add_development_dependency "reevoocop"
-
-  spec.add_runtime_dependency "fluentd", [">= 0.14.11", "< 2"]
-  spec.add_runtime_dependency "systemd-journal", "~> 1.2"
-end