fluent-plugin-systemd 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d1e4a2ade25b42fdf5d04e16ce877c6c380953f
-  data.tar.gz: 1dacc441622a3dfae2985a6be2f41f26efcc7341
+  metadata.gz: 07f12e01a1d81c731f44477a051807042230adfa
+  data.tar.gz: 8717ea703cd39fb947af86b5773a68de5b55c040
 SHA512:
-  metadata.gz: 186f98b177cde9d3537bb4e0b06e337cc94c149e679aa044b173accb56ea76b0125fd6bbc154894ca01bfb75b7f34b579e98d7cb49a2f3fc7ea70a379e6503da
-  data.tar.gz: a7e29f074c8c01afd459cb33c4f048b63317fcfc2cba4d5146c684a4cd9b36363363ea7b3585d12654f6e85efab1a97f746ace81abf91baa8a2210ce52ca3a28
+  metadata.gz: 92b3275665979af1a9d691f05acdc65a88fd88b062c6224673cde0c691c9c7a4376a1c97c96c7e1dc9cc54f224fe27e7c39600bc0b6307c88ad182e442971520
+  data.tar.gz: 110c2d6ff7ebeac0eb96ceaf0d470d5bdfdf5da2e4556f8b2fc98b213e08b87553d231073f2828a5050dc88437f307e2846cea0ebfd39c131465a34b83123e5e

data/README.md

@@ -6,8 +6,8 @@
 
 
 |fluent-plugin-systemd|fluentd|td-agent|ruby|
-|----|----|----|
-| 0.1.x | >= 0.14.11, < 2 | 3 | >= 2.1 |
+|----|----|----|----|
+| 0.2.x | >= 0.14.11, < 2 | 3 | >= 2.1 |
 | 0.0.x | ~> 0.12.0       | 2 | >= 1.9  |
 
 * The 0.1.x series is developed from this branch (master)
@@ -21,11 +21,11 @@
 
 Simply use RubyGems:
 
-    gem install fluent-plugin-systemd -v 0.1.1
+    gem install fluent-plugin-systemd -v 0.2.0
 
-    or
+or
 
-    td-agent-gem install fluent-plugin-systemd -v 0.1.1
+    td-agent-gem install fluent-plugin-systemd -v 0.2.0
 
 ## Configuration
 
@@ -33,37 +33,59 @@ Simply use RubyGems:
       @type systemd
       path /var/log/journal
       filters [{ "_SYSTEMD_UNIT": "kube-proxy.service" }]
-      pos_file kube-proxy.pos
+      <storage>
+        @type local
+        persistent true
+        path kube-proxy.pos
+      </storage>
       tag kube-proxy
       read_from_head true
     </source>
 
-**path**
+    <match kube-proxy>
+      @type stdout
+    </match>
+
+**`path`**
 
 Path to the systemd journal, defaults to `/var/log/journal`
 
-**filters**
+**`filters`**
 
 Array of filters, see [here](http://www.rubydoc.info/gems/systemd-journal/Systemd%2FJournal%2FFilterable%3Afilter) for further
 documentation, defaults to no filtering.
 
-**pos file**
+**`pos_file`**
+
+_This parameter is deprecated and will be removed in favour of storage in v1.0._
+
 
 Path to pos file, stores the journald cursor. File is created if does not exist.
 
-**read_from_head**
+**`storage`**
+
+Configuration for a [storage plugin](http://docs.fluentd.org/v0.14/articles/storage-plugin-overview) used to store the journald cursor.
+
+_Upgrading from `pos_file`_
+
+If `pos_file` is specified in addition to a storage plugin with persistent set to true, the cursor will be
+copied from the `pos_file` on startup, and the old `pos_file` removed.
+
+**`read_from_head`**
 
 If true reads all available journal from head, otherwise starts reading from tail,
  ignored if pos file exists (and is valid). Defaults to false.
 
-**strip_underscores**
+**`strip_underscores`**
 
 If true strips underscores from the beginning of systemd field names.
 May be useful if outputting to kibana, as underscore prefixed fields are unindexed there.
 
-**tag**
+**`tag`**
+
+_Required_ 
 
-_Required_ A tag that will be added to events generated by this input.
+A tag that will be added to events generated by this input.
 
 ## Example
 
@@ -99,3 +121,4 @@ Many thanks to our fantastic contributors
 * [Richard Megginson](https://github.com/richm)
 * [Masahiro Nakagawa](https://github.com/repeatedly)
 * [Hiroshi Hatake](https://github.com/cosmo0920)
+* [neko-neko](https://github.com/neko-neko)

data/lib/fluent/plugin/in_systemd.rb

@@ -7,29 +7,37 @@ module Fluent
     class SystemdInput < Input
       Fluent::Plugin.register_input("systemd", self)
 
-      helpers :timer
+      helpers :timer, :storage
+
+      DEFAULT_STORAGE_TYPE = "local".freeze
 
       config_param :path, :string, default: "/var/log/journal"
       config_param :filters, :array, default: []
-      config_param :pos_file, :string, default: nil
+      config_param :pos_file, :string, default: nil, deprecated: "Use <storage> section with `persistent: true' instead"
       config_param :read_from_head, :bool, default: false
       config_param :strip_underscores, :bool, default: false
       config_param :tag, :string
 
+      config_section :storage do
+        config_set_default :usage, "positions"
+        config_set_default :@type, DEFAULT_STORAGE_TYPE
+        config_set_default :persistent, false
+      end
+
       def configure(conf)
         super
-        @pos_writer = PosWriter.new(@pos_file)
+        @pos_storage = PosWriter.new(@pos_file, storage_create(usage: "positions"))
         @journal    = nil
       end
 
       def start
         super
-        @pos_writer.start
+        @pos_storage.start
         timer_execute(:in_systemd_emit_worker, 1, &method(:run))
       end
 
       def shutdown
-        @pos_writer.shutdown
+        @pos_storage.shutdown
         super
       end
 
@@ -49,10 +57,11 @@ module Fluent
       end
 
       def seek
-        seek_to(@pos_writer.cursor || read_from)
+        cursor = @pos_storage.get(:journal)
+        seek_to(cursor || read_from)
       rescue Systemd::JournalError
         log.warn(
-          "Could not seek to cursor #{@pos_writer.cursor} found in pos file: #{@pos_writer.path}, " \
+          "Could not seek to cursor #{cursor} found in pos file: #{@pos_storage.path}, " \
           "falling back to reading from #{read_from}",
         )
         seek_to(read_from)
@@ -96,7 +105,7 @@ module Fluent
       def watch
         while @journal.move_next
           yield @journal.current_entry
-          @pos_writer.update(@journal.cursor)
+          @pos_storage.put(:journal, @journal.cursor)
         end
       end
     end

data/lib/fluent/plugin/systemd/pos_writer.rb

@@ -4,15 +4,27 @@ module Fluent
   module Plugin
     class SystemdInput < Input
       class PosWriter
-        def initialize(pos_file)
+        def initialize(pos_file, storage)
           @path = pos_file
           @lock = Mutex.new
+          @storage = storage
           @cursor = nil
           @written_cursor = nil
           setup
         end
 
-        attr_reader :cursor, :path
+        def get(key)
+          @storage ? @storage.get(key) : @cursor
+        end
+
+        def put(key, cursor)
+          return @storage.put(key, cursor) if @storage
+          @lock.synchronize { @cursor = cursor }
+        end
+
+        def path
+          @path || @storage.path
+        end
 
         def start
           return unless @path
@@ -27,15 +39,21 @@ module Fluent
           write_pos
         end
 
-        def update(c)
-          @lock.synchronize { @cursor = c }
-        end
-
         private
 
         def setup
-          return unless @path && File.exist?(@path)
-          @cursor = IO.read(@path).chomp
+          if @storage.persistent
+            migrate_to_storage if @path && File.exist?(@path)
+          elsif @path
+            @cursor = IO.read(@path).chomp if File.exist?(@path)
+            @storage = nil
+          end
+        end
+
+        def migrate_to_storage
+          @storage.put(:journal, IO.read(@path).chomp)
+          File.delete(@path)
+          @path = nil
         end
 
         def work

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-systemd
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Ed Robinson
@@ -107,23 +107,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- Gemfile
-- Gemfile.lock
 - LICENCE
 - README.md
-- Rakefile
-- Vagrantfile
-- fluent-plugin-systemd.gemspec
 - lib/fluent/plugin/in_systemd.rb
 - lib/fluent/plugin/systemd/pos_writer.rb
-- pkg/fluent-plugin-systemd-0.0.3.gem
-- pkg/fluent-plugin-systemd-0.0.4.gem
-- pkg/fluent-plugin-systemd-0.0.5.gem
-- pkg/fluent-plugin-systemd-0.0.6.gem
-- pkg/fluent-plugin-systemd-0.1.0.gem
-- pkg/fluent-plugin-systemd-0.1.1.pre.gem
-- pkg/fluent-plugin-systemd-0.1.1.pre2.gem
-- pkg/fluent-plugin-systemd-0.1.1.pre3.gem
 homepage: https://github.com/reevoo/fluent-plugin-systemd
 licenses:
 - MIT

data/Gemfile

@@ -1,3 +0,0 @@
-source "https://rubygems.org"
-
-gemspec

data/Gemfile.lock

@@ -1,67 +0,0 @@
-PATH
-  remote: .
-  specs:
-    fluent-plugin-systemd (0.1.1)
-      fluentd (>= 0.14.11, < 2)
-      systemd-journal (~> 1.2)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    ast (2.3.0)
-    astrolabe (1.3.1)
-      parser (~> 2.2)
-    cool.io (1.4.5)
-    ffi (1.9.17)
-    fluentd (0.14.11)
-      cool.io (~> 1.4.5)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
-      msgpack (>= 0.7.0, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
-      sigdump (~> 0.2.2)
-      strptime (~> 0.1.7)
-      tzinfo (~> 1.0)
-      tzinfo-data (~> 1.0)
-      yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.0.2)
-    parser (2.3.3.1)
-      ast (~> 2.2)
-    powerpack (0.0.9)
-    rainbow (2.2.1)
-    rake (12.0.0)
-    reevoocop (0.0.9)
-      rubocop (= 0.28.0)
-    rubocop (0.28.0)
-      astrolabe (~> 1.3)
-      parser (>= 2.2.0.pre.7, < 3.0)
-      powerpack (~> 0.0.6)
-      rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.4)
-    ruby-progressbar (1.8.1)
-    serverengine (2.0.4)
-      sigdump (~> 0.2.2)
-    sigdump (0.2.4)
-    strptime (0.1.9)
-    systemd-journal (1.2.3)
-      ffi (~> 1.9)
-    test-unit (2.5.5)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2016.10)
-      tzinfo (>= 1.0.0)
-    yajl-ruby (1.3.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.10)
-  fluent-plugin-systemd!
-  rake
-  reevoocop
-  test-unit (~> 2.5)
-
-BUNDLED WITH
-   1.13.2

data/Rakefile

@@ -1,31 +0,0 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
-require "reevoocop/rake_task"
-require "fileutils"
-
-ReevooCop::RakeTask.new(:reevoocop)
-
-Rake::TestTask.new(:test) do |t|
-  t.test_files = Dir["test/**/test_*.rb"]
-end
-
-task default: "docker:test"
-task build: "docker:test"
-task default: :reevoocop
-
-namespace :docker do
-  distros = [:ubuntu, :"tdagent-ubuntu", :"tdagent-centos"]
-  task test: distros
-
-  distros.each do |distro|
-    task distro do
-      puts "testing on #{distro}"
-      begin
-        FileUtils.cp("test/docker/Dockerfile.#{distro}", "Dockerfile")
-        sh "docker build ."
-      ensure
-        FileUtils.rm("Dockerfile")
-      end
-    end
-  end
-end

data/Vagrantfile

@@ -1,71 +0,0 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
-
-# All Vagrant configuration is done below. The "2" in Vagrant.configure
-# configures the configuration version (we support older styles for
-# backwards compatibility). Please don't change it unless you know what
-# you're doing.
-Vagrant.configure("2") do |config|
-  # The most common configuration options are documented and commented below.
-  # For a complete reference, please see the online documentation at
-  # https://docs.vagrantup.com.
-
-  # Every Vagrant development environment requires a box. You can search for
-  # boxes at https://atlas.hashicorp.com/search.
-  config.vm.box = "centos/7"
-
-  # Disable automatic box update checking. If you disable this, then
-  # boxes will only be checked for updates when the user runs
-  # `vagrant box outdated`. This is not recommended.
-  # config.vm.box_check_update = false
-
-  # Create a forwarded port mapping which allows access to a specific port
-  # within the machine from a port on the host machine. In the example below,
-  # accessing "localhost:8080" will access port 80 on the guest machine.
-  # config.vm.network "forwarded_port", guest: 80, host: 8080
-
-  # Create a private network, which allows host-only access to the machine
-  # using a specific IP.
-  # config.vm.network "private_network", ip: "192.168.33.10"
-
-  # Create a public network, which generally matched to bridged network.
-  # Bridged networks make the machine appear as another physical device on
-  # your network.
-  # config.vm.network "public_network"
-
-  # Share an additional folder to the guest VM. The first argument is
-  # the path on the host to the actual folder. The second argument is
-  # the path on the guest to mount the folder. And the optional third
-  # argument is a set of non-required options.
-  # config.vm.synced_folder "../data", "/vagrant_data"
-
-  # Provider-specific configuration so you can fine-tune various
-  # backing providers for Vagrant. These expose provider-specific options.
-  # Example for VirtualBox:
-  #
-  # config.vm.provider "virtualbox" do |vb|
-  #   # Display the VirtualBox GUI when booting the machine
-  #   vb.gui = true
-  #
-  #   # Customize the amount of memory on the VM:
-  #   vb.memory = "1024"
-  # end
-  #
-  # View the documentation for the provider you are using for more
-  # information on available options.
-
-  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
-  # such as FTP and Heroku are also available. See the documentation at
-  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
-  # config.push.define "atlas" do |push|
-  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
-  # end
-
-  # Enable provisioning with a shell script. Additional provisioners such as
-  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
-  # documentation for more information about their specific syntax and use.
-  # config.vm.provision "shell", inline: <<-SHELL
-  #   apt-get update
-  #   apt-get install -y apache2
-  # SHELL
-end

data/fluent-plugin-systemd.gemspec

@@ -1,28 +0,0 @@
-# -*- encoding: utf-8 -*-
-
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-Gem::Specification.new do |spec|
-  spec.name          = "fluent-plugin-systemd"
-  spec.version       = "0.1.1"
-  spec.authors       = ["Ed Robinson"]
-  spec.email         = ["ed@reevoo.com"]
-
-  spec.summary       = "Input plugin to read from systemd journal."
-  spec.description   = "This is a fluentd input plugin. It reads logs from the systemd journal."
-  spec.homepage      = "https://github.com/reevoo/fluent-plugin-systemd"
-  spec.license       = "MIT"
-
-
-  spec.files         = Dir["**/**"].reject { |f| f.match(/^(test|spec|features)\//) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "test-unit", "~> 2.5"
-  spec.add_development_dependency "reevoocop"
-
-  spec.add_runtime_dependency "fluentd", [">= 0.14.11", "< 2"]
-  spec.add_runtime_dependency "systemd-journal", "~> 1.2"
-end