logstash-input-journald 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df132fdfd9561d73df5f49ac4c7408a15df597d3
+  data.tar.gz: 3c34d90fecb45efee4fa6eadad7e37233539be51
+SHA512:
+  metadata.gz: 355034993d7381aa58acead667a6e5a3fa79cae74387c1acb118e9c02f3ec321409467a5b2655a9d7673c068b2b0280779a28ecb3e87d8924f85d6931355f413
+  data.tar.gz: 7350329c4f27f81c495fcd498f89fa5443110a10b50280f04c05463cd5ece4ebf85fbfb5fb6b5eae46991532489c45508ca83733aac24700d47d46405b5034fd

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+Gemfile.lock
+.bundle
+vendor

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,47 @@
+logstash-input-journald
+=======================
+
+POC systemd journal Logstash input
+
+Example config
+--------------
+```
+input {
+     journald {
+       lowercase => true
+       seekto => "head"
+       thisboot => true
+       type => "systemd"
+       tags => [ "coreos" ]
+     }
+}
+
+output {
+  stdout {codec => rubydebug}
+}
+```
+
+Install with
+------------
+```
+git clone https://github.com/stuart-warren/logstash-input-journald.git
+cd logstash-input-journald
+gem build logstash-input-journald.gemspec
+sudo /path/to/logstash/bin/plugin install /path/to/git/logstash-input-journald/logstash-input-journald-0.0.1.gem
+```
+Sincedb
+----
+
+This plugin creates a sincedb in your home, called .sincedb\_journal.
+It automatically stores the cursor to the journal there, so when you restart logstash, only new messages are read.
+When executing the plugin the second time (which means the sincedb exists), ``seekto``, and ``thisboot`` are ignored.
+If you don't want the sincedb, configure it's path to /dev/null.
+Tips
+----
+
+Ensure the user you are running logstash as has read access to the journal files ``/var/log/journal/*/*``
+
+Issues
+------
+
+Killing the logstash process takes a long time...

data/Rakefile

@@ -0,0 +1,7 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+
+require "logstash/devutils/rake"

data/lib/logstash/inputs/journald.rb

@@ -0,0 +1,163 @@
+# encoding: utf-8
+require "logstash/inputs/threadable"
+require "logstash/namespace"
+require "socket"
+require "systemd/journal"
+require "fileutils" # For touch
+
+# Pull events from a local systemd journal.
+#
+# See requirements https://github.com/ledbettj/systemd-journal
+class LogStash::Inputs::Journald < LogStash::Inputs::Threadable
+
+    config_name "journald"
+
+    # Where in the journal to start capturing logs
+    # Options: head, tail
+    config :seekto, :validate => [ "head", "tail" ], :default => "tail"
+
+    # System journal flags
+    # 0 = all avalable
+    # 1 = local only
+    # 2 = runtime only
+    # 4 = system only
+    #
+    config :flags, :validate => [0, 1, 2, 4], :default => 0
+
+    # Path to read journal files from
+    #
+    config :path, :validate => :string, :default => "/var/log/journal"
+
+    # Filter on events. Not heavily tested.
+    #
+    config :filter, :validate => :hash, :required => false, :default => {}
+
+    # Filter logs since the system booted (only relevant with seekto => "head")
+    #
+    config :thisboot, :validate => :boolean, :default => true
+
+    # Lowercase annoying UPPERCASE fieldnames. (May clobber existing fields)
+    #
+    config :lowercase, :validate => :boolean, :default => false
+
+    # Where to write the sincedb database (keeps track of the current
+    # position of the journal). The default will write
+    # the sincedb file to matching `$HOME/.sincedb_journal`
+    #
+    config :sincedb_path, :validate => :string
+
+    # How often (in seconds) to write a since database with the current position of
+    # the journal.
+    #
+    config :sincedb_write_interval, :validate => :number, :default => 15
+
+    public
+    def register
+        opts = {
+            flags: @flags,
+            path: @path,
+        }
+        @hostname = Socket.gethostname
+        @journal = Systemd::Journal.new(opts)
+        @cursor = ""
+        @written_cursor = ""
+        @cursor_lock = Mutex.new
+        if @thisboot
+            @filter[:_boot_id] = Systemd::Id128.boot_id
+        end
+        if @sincedb_path.nil?
+            if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil?
+                @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \
+                              "to keep track of the files I'm watching. Either set " \
+                              "HOME or SINCEDB_DIR in your environment, or set sincedb_path in " \
+                              "in your Logstash config for the file input with " \
+                              "path '#{@path.inspect}'")
+                raise(LogStash::ConfigurationError, "Sincedb can not be created.")
+            end
+            sincedb_dir = ENV["SINCEDB_DIR"] || ENV["HOME"]
+            @sincedb_path = File.join(sincedb_dir, ".sincedb_journal")
+            @logger.info("No sincedb_path set, generating one for the journal",
+                         :sincedb_path => @sincedb_path)
+        end
+        # (Create and) read sincedb
+        FileUtils.touch(@sincedb_path)
+        @cursor = IO.read(@sincedb_path)
+        # Write sincedb in thread
+        @sincedb_writer = Thread.new do
+            loop do
+                sleep @sincedb_write_interval
+                if @cursor != @written_cursor
+                    file = File.open(@sincedb_path, 'w+')
+                    file.puts @cursor
+                    file.close
+                    @cursor_lock.synchronize {
+                        @written_cursor = @cursor
+                    }
+                 end
+            end
+        end
+    end # def register
+
+    def run(queue)
+        if @cursor.strip.length == 0
+            @journal.seek(@seekto.to_sym)
+
+            # We must make one movement in order for the journal C api or else
+            # the @journal.watch call will start from the beginning of the
+            # journal. see:
+            # https://github.com/ledbettj/systemd-journal/issues/55
+            if @seekto == 'tail'
+              @journal.move_previous
+            end
+
+            @journal.filter(@filter)
+        else
+            @journal.seek(@cursor)
+            @journal.move_next # Without this, the last event will be read again
+        end
+        @journal.watch do |entry|
+            timestamp = entry.realtime_timestamp
+            event = LogStash::Event.new(
+                entry.to_h_lower(@lowercase).merge(
+                    "@timestamp" => timestamp,
+                    "host" => entry._hostname || @hostname,
+                    "cursor" => @journal.cursor
+                )
+            )
+            decorate(event)
+            queue << event
+            @cursor_lock.synchronize {
+                @cursor = @journal.cursor
+            }
+        end
+    end # def run
+
+    public
+    def teardown # FIXME: doesn't really seem to work...
+        return finished unless @journal # Ignore multiple calls
+
+        @logger.debug("journald shutting down.")
+        @journal = nil
+        Thread.kill(@sincedb_writer)
+        # Write current cursor
+        file = File.open(@sincedb_path, 'w+')
+        file.puts @cursor
+        file.close
+        @cursor = nil
+        finished
+    end # def teardown
+
+end # class LogStash::Inputs::Journald
+
+# Monkey patch Systemd::JournalEntry
+module Systemd
+    class JournalEntry
+        def to_h_lower(is_lowercase)
+            if is_lowercase
+                @entry.each_with_object({}) { |(k, v), h| h[k.downcase] = v.dup.force_encoding('iso-8859-1').encode('utf-8') }
+            else
+                @entry.each_with_object({}) { |(k, v), h| h[k] = v.dup.force_encoding('iso-8859-1').encode('utf-8') }
+            end
+        end
+    end
+end

data/logstash-input-journald.gemspec

@@ -0,0 +1,32 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-input-journald'
+  s.version         = '0.0.4'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "Read events from local systemd journal"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["stuart-warren"]
+  s.email           = 'stuart.warren@ocado.com'
+  s.homepage        = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*')
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-line'
+  s.add_runtime_dependency 'logstash-codec-json'
+  s.add_runtime_dependency 'logstash-codec-json_lines'
+  s.add_runtime_dependency 'ffi'
+  s.add_runtime_dependency 'systemd-journal', '~> 1.2.2'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/inputs/journald_spec.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "socket"
+require "logstash/inputs/journald"
+
+describe LogStash::Inputs::Journald do
+    context "Ummm" do
+        it "no tests here" do
+
+        end
+    end
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-journald
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- stuart-warren
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-10-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json_lines
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: ffi
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+  name: systemd-journal
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.2
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: stuart.warren@ocado.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/inputs/journald.rb
+- logstash-input-journald.gemspec
+- spec/inputs/journald_spec.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Read events from local systemd journal
+test_files:
+- spec/inputs/journald_spec.rb