logstash-input-syslog 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fbc628627bc59738a707a9de52d92689c1d535ca
-  data.tar.gz: 718a95e238d3c2bbb856cbce165230ac9e5a018c
+  metadata.gz: b78d69100942cdae0c081913bf27f54faf8e395a
+  data.tar.gz: d4a62595577b76589e0362e5b87fd9ca394cf0c0
 SHA512:
-  metadata.gz: 6574205308fd259f3923f15d51f98e828cf67d29a0ffe491a2c4bcc0b8cb0c91e7e24ed3c99f132e46bcc15cf7d4a3373cb33fa6cc12b30be4432e66d940d025
-  data.tar.gz: d4478eab6dbaff92abc8ada784285bdfc8cf6451a39507a72a78410d375c3753537c94b0b493025456503731e1e43568ba96fba4c166ef3d0676e0eae95cc5c7
+  metadata.gz: 3ee9f00aaba2b9c4a77a2c03184812df896d796822d8540e56d6d44c9ef1cc6c4c4875ff0a6ddc768fe71eb213aaccdb8d0117092540b99468488c59c4629f4d
+  data.tar.gz: 4523dd8f04baa8fbfaafc1221c0792e383865b38df4975832bd49d6bb939031f40e3f4be8510e61aec3cf876e925241a3fdaa5aca73268cc9af8de5693714986

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -0,0 +1,95 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+```sh
+bundle exec rspec
+```
+
+The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
+```
+To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :ref => "master"
+```
+```ruby
+gem "logstash", :path => "/your/local/logstash"
+```
+
+Then update your dependencies and run your tests:
+
+```sh
+bundle install
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Update Logstash dependencies
+```sh
+rake vendor:gems
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to me that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/syslog.rb

@@ -1,10 +1,11 @@
 # encoding: utf-8
 require "date"
+require "socket"
+require "concurrent_ruby"
 require "logstash/filters/grok"
 require "logstash/filters/date"
 require "logstash/inputs/base"
 require "logstash/namespace"
-require "socket"
 
 # Read syslog messages as events over the network.
 #
@@ -24,7 +25,6 @@ require "socket"
 #
 class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   config_name "syslog"
-  milestone 1
 
   default :codec, "plain"
 
@@ -44,10 +44,28 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   # Labels for severity levels. These are defined in RFC3164.
   config :severity_labels, :validate => :array, :default => [ "Emergency" , "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug" ]
 
+  # Specify a time zone canonical ID to be used for date parsing.
+  # The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html).
+  # This is useful in case the time zone cannot be extracted from the value,
+  # and is not the platform default.
+  # If this is not specified the platform default will be used.
+  # Canonical ID is good as it takes care of daylight saving time for you
+  # For example, `America/Los_Angeles` or `Europe/France` are valid IDs.
+  config :timezone, :validate => :string
+
+  # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag.
+  # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX.
+  # If not specified, the platform default will be used.
+  #
+  # The locale is mostly necessary to be set for parsing month names (pattern with MMM) and
+  # weekday names (pattern with EEE).
+  #
+  config :locale, :validate => :string
+
   public
   def initialize(params)
     super
-    @shutdown_requested = false
+    @shutdown_requested = Concurrent::AtomicBoolean.new(false)
     BasicSocket.do_not_reverse_lookup = true
   end # def initialize
 
@@ -61,44 +79,27 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
     )
 
     @date_filter = LogStash::Filters::Date.new(
-      "match" => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"]
+      "match" => [ "timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601"],
+      "locale" => @locale,
+      "timezone" => @timezone,
     )
 
     @grok_filter.register
     @date_filter.register
 
-    @tcp_clients = ThreadSafe::Array.new
+    @tcp_sockets = ThreadSafe::Array.new
+    @tcp = @udp = nil
   end # def register
 
   public
   def run(output_queue)
-    # udp server
-    udp_thr = Thread.new do
-      begin
-        udp_listener(output_queue)
-      rescue => e
-        break if @shutdown_requested
-        @logger.warn("syslog udp listener died",
-                     :address => "#{@host}:#{@port}", :exception => e,
-                     :backtrace => e.backtrace)
-        sleep(5)
-        retry
-      end # begin
-    end # Thread.new
-
-    # tcp server
-    tcp_thr = Thread.new do
-      begin
-        tcp_listener(output_queue)
-      rescue => e
-        break if @shutdown_requested
-        @logger.warn("syslog tcp listener died",
-                     :address => "#{@host}:#{@port}", :exception => e,
-                     :backtrace => e.backtrace)
-        sleep(5)
-        retry
-      end # begin
-    end # Thread.new
+    udp_thr = Thread.new(output_queue) do |output_queue|
+      server(:udp, output_queue)
+    end
+
+    tcp_thr = Thread.new(output_queue) do |output_queue|
+      server(:tcp, output_queue)
+    end
 
     # If we exit and we're the only input, the agent will think no inputs
     # are running and initiate a shutdown.
@@ -107,65 +108,94 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   end # def run
 
   private
+  # server call the specified protocol listener and basically restarts on
+  # any listener uncatched exception
+  #
+  # @param protocol [Symbol] either :udp or :tcp
+  # @param output_queue [Queue] the pipeline input to filters queue
+  def server(protocol, output_queue)
+    self.send("#{protocol}_listener", output_queue)
+  rescue => e
+    if @shutdown_requested.false?
+      @logger.warn("syslog listener died", :protocol => protocol, :address => "#{@host}:#{@port}", :exception => e, :backtrace => e.backtrace)
+      sleep(5)
+      retry
+    end
+  end
+
+  private
+  # udp_listener creates the udp socket and continously read from it.
+  # upon exception the socket will be closed and the exception bubbled
+  # in the server which will restart the listener
   def udp_listener(output_queue)
     @logger.info("Starting syslog udp listener", :address => "#{@host}:#{@port}")
 
-    if @udp
-      @udp.close
-    end
-
+    @udp.close if @udp
     @udp = UDPSocket.new(Socket::AF_INET)
     @udp.bind(@host, @port)
 
-    loop do
+    while true
       payload, client = @udp.recvfrom(9000)
-      # Ruby uri sucks, so don't use it.
-      @codec.decode(payload) do |event|
-        decorate(event)
-        event["host"] = client[3]
-        syslog_relay(event)
-        output_queue << event
-      end
+      decode(client[3], output_queue, payload)
     end
   ensure
     close_udp
   end # def udp_listener
 
   private
+  # tcp_listener accepts tcp connections and creates a new tcp_receiver thread
+  # for each accepted socket.
+  # upon exception all tcp sockets will be closed and the exception bubbled
+  # in the server which will restart the listener.
   def tcp_listener(output_queue)
     @logger.info("Starting syslog tcp listener", :address => "#{@host}:#{@port}")
     @tcp = TCPServer.new(@host, @port)
-    @tcp_clients = []
 
     loop do
-      client = @tcp.accept
-      @tcp_clients << client
-      Thread.new(client) do |client|
-        ip, port = client.peeraddr[3], client.peeraddr[1]
-        @logger.info("new connection", :client => "#{ip}:#{port}")
-        LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}")
-        begin
-          client.each do |line|
-            @codec.decode(line) do |event|
-              decorate(event)
-              event["host"] = ip
-              syslog_relay(event)
-              output_queue << event
-            end
-          end
-        rescue Errno::ECONNRESET
-        ensure
-          @tcp_clients.delete(client)
-        end
-      end # Thread.new
-    end # loop do
+      socket = @tcp.accept
+      @tcp_sockets << socket
+
+      break if @shutdown_requested.true?
+
+      Thread.new(output_queue, socket) do |output_queue, socket|
+        tcp_receiver(output_queue, socket)
+      end
+    end
   ensure
     close_tcp
   end # def tcp_listener
 
+  # tcp_receiver is executed in a thread, any uncatched exception will be bubbled up to the
+  # tcp server thread and all tcp connections will be closed and the listener restarted.
+  def tcp_receiver(output_queue, socket)
+    ip, port = socket.peeraddr[3], socket.peeraddr[1]
+    @logger.info("new connection", :client => "#{ip}:#{port}")
+    LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}")
+
+    socket.each { |line| decode(ip, output_queue, line) }
+  rescue Errno::ECONNRESET
+    # swallow connection reset exceptions to avoid bubling up the tcp_listener & server
+  ensure
+    @tcp_sockets.delete(socket)
+    socket.close rescue nil
+  end
+
+  private
+  def decode(host, output_queue, data)
+    @codec.decode(data) do |event|
+      decorate(event)
+      event["host"] = host
+      syslog_relay(event)
+      output_queue << event
+    end
+  rescue => e
+    # swallow and log all decoding exceptions, these will never be socket related
+    @logger.error("Error decoding data", :data => line.inspect, :exception => e, :backtrace => e.backtrace)
+  end
+
   public
   def teardown
-    @shutdown_requested = true
+    @shutdown_requested.make_true
     close_udp
     close_tcp
     finished
@@ -183,8 +213,8 @@ class LogStash::Inputs::Syslog < LogStash::Inputs::Base
   private
   def close_tcp
     # If we somehow have this left open, close it.
-    @tcp_clients.each do |client|
-      client.close rescue nil
+    @tcp_sockets.each do |socket|
+      socket.close rescue nil
     end
     @tcp.close if @tcp rescue nil
     @tcp = nil

data/logstash-input-syslog.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-syslog'
-  s.version         = '0.1.1'
+  s.version         = '0.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Read syslog messages as events over the network."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -22,6 +22,9 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
 
+  s.add_runtime_dependency 'concurrent-ruby'
+  s.add_runtime_dependency 'thread_safe'
+
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-filter-grok'
   s.add_runtime_dependency 'logstash-filter-date'

data/spec/inputs/syslog_spec.rb

@@ -1,13 +1,14 @@
-# coding: utf-8
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
-require "socket"
 require "logstash/inputs/syslog"
 require "logstash/event"
+require "stud/try"
+require "socket"
 
 describe "inputs/syslog" do
-  
+  SYSLOG_LINE = "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]"
 
-  it "should properly handle priority, severity and facilities", :socket => true do
+  describe "should properly handle priority, severity and facilities" do
     port = 5511
     event_count = 10
 
@@ -21,27 +22,33 @@ describe "inputs/syslog" do
     CONFIG
 
     input do |pipeline, queue|
-      Thread.new { pipeline.run }
+      t = Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
 
       socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
       event_count.times do |i|
-        socket.puts("<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]")
+        socket.puts(SYSLOG_LINE)
       end
       socket.close
 
       events = event_count.times.collect { queue.pop }
 
+      # important to shutdown here before any assertion so that the pipeline + socket
+      # cleanups are correctly done before any potential spec error that would result
+      # in aborting execution and not doing the cleanup.
+      pipeline.shutdown
+      t.join
+
       insist { events.length } == event_count
-      event_count.times do |i|
-        insist { events[i]["priority"] } == 164
-        insist { events[i]["severity"] } == 4
-        insist { events[i]["facility"] } == 20
+      events.each do |event|
+        insist { event["priority"] } == 164
+        insist { event["severity"] } == 4
+        insist { event["facility"] } == 20
       end
     end
   end
 
-  it "should add unique tag when grok parsing fails with live syslog input", :socket => true do
+  describe "should add unique tag when grok parsing fails with live syslog input" do
     port = 5511
     event_count = 10
 
@@ -55,7 +62,7 @@ describe "inputs/syslog" do
     CONFIG
 
     input do |pipeline, queue|
-      Thread.new { pipeline.run }
+      t = Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
 
       socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
@@ -66,13 +73,103 @@ describe "inputs/syslog" do
 
       events = event_count.times.collect { queue.pop }
 
+      # important to shutdown here before any assertion so that the pipeline + socket
+      # cleanups are correctly done before any potential spec error that would result
+      # in aborting execution and not doing the cleanup.
+      pipeline.shutdown
+      t.join
+
       insist { events.length } == event_count
       event_count.times do |i|
-        insist { events[i]["tags"] } == ["_grokparsefailure_sysloginputplugin"]
+        insist { events[i]["tags"] } == ["_grokparsefailure_sysloginput"]
       end
     end
   end
 
+  describe "should properly handle locale and timezone" do
+    port = 5511
+    event_count = 10
+
+    config <<-CONFIG
+      input {
+        syslog {
+          type => "blah"
+          port => #{port}
+          locale => "en"
+          timezone => "UTC"
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      t = Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      event_count.times do |i|
+        socket.puts(SYSLOG_LINE)
+      end
+      socket.close
+
+      events = event_count.times.collect { queue.pop }
+
+      # important to shutdown here before any assertion so that the pipeline + socket
+      # cleanups are correctly done before any potential spec error that would result
+      # in aborting execution and not doing the cleanup.
+      pipeline.shutdown
+      t.join
+
+      insist { events.length } == event_count
+      events.each do |event|
+        insist { event["@timestamp"].to_iso8601 } == "#{Time.now.year}-10-26T15:19:25.000Z"
+      end
+    end
+  end
+
+  describe "should properly handle no locale and no timezone" do
+    port = 5511
+
+    config <<-CONFIG
+      input {
+        syslog {
+          type => "blah"
+          port => #{port}
+        }
+      }
+    CONFIG
+
+    input do |pipeline, queue|
+      t = Thread.new { pipeline.run }
+      sleep 0.1 while !pipeline.ready?
+
+      socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) }
+      socket.puts(SYSLOG_LINE)
+      socket.close
+
+      event = queue.pop
+
+      # important to shutdown here before any assertion so that the pipeline + socket
+      # cleanups are correctly done before any potential spec error that would result
+      # in aborting execution and not doing the cleanup.
+      pipeline.shutdown
+      t.join
+
+      # chances platform timezone is not UTC so ignore the hours
+      insist { event["@timestamp"].to_iso8601 } =~ /#{Time.now.year}-10-26T\d\d:19:25.000Z/
+    end
+  end
+
+  it "should support non UTC timezone" do
+    input = LogStash::Inputs::Syslog.new({"timezone" => "-05:00"})
+    input.register
+
+    # event which is not syslog should have a new tag
+
+    syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" })
+    input.syslog_relay(syslog_event)
+    insist { syslog_event["@timestamp"].to_iso8601 } == "#{Time.now.year}-10-26T20:19:25.000Z"
+  end
+
   it "should add unique tag when grok parsing fails" do
     input = LogStash::Inputs::Syslog.new({})
     input.register

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-syslog
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2015-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash
@@ -30,6 +30,34 @@ dependencies:
         version: 2.0.0
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: thread_safe
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
@@ -95,6 +123,7 @@ files:
 - .gitignore
 - Gemfile
 - LICENSE
+- README.md
 - Rakefile
 - lib/logstash/inputs/syslog.rb
 - logstash-input-syslog.gemspec
@@ -121,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Read syslog messages as events over the network.