syslogstash 0.4.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84f7fa134c15c370b86eb8bc685b4fb900c7c68d
-  data.tar.gz: 31fbe3ad2d1e28f43a9ab3631b634c3c9fb78ee7
+  metadata.gz: 88eb57340c1514fab16aca8395e8dc0821309e95
+  data.tar.gz: bfe7b8d15263768f868d42e8749261388b24496a
 SHA512:
-  metadata.gz: 9b8d20f5cc43ab7746247866358b6599a1bd92dcefee0b5681db88b820481159d6d15bd0d3167eec071824866f72d9383b1477d1cc420548cdc1e65440f48b81
-  data.tar.gz: be68092ce035ebec4da5e5a643bbc07b8312cf085a5083f94ab8b6605a903d67b4ba4e06f60ee10fabdd4f71b700427fb021233b1771d95e479ca6628362e23d
+  metadata.gz: cee42ae44c931c73e868e6254987fc1900f5e541bbeb4b44dd5c07b8c6feecf2f572223b543648a7a30ba33c6c8e08b54e3e7c162ada82efcdf262bf58007ec8
+  data.tar.gz: 4fa56e690b3715e7179c863afa41e2fd85946b570ad0d1f2a2df6fba64883b25a0b2c102a5c5a9fd9503dd33e3f43b69c15ec08d140d5e0e839627de4e2619ac

data/README.md

@@ -31,20 +31,25 @@ The file which describes how `syslogstash` will operate is a fairly simple
 YAML file.  It consists of two sections, `sockets` and `servers`, which list
 the UNIX sockets to listen for syslog messages on, and the URLs of logstash
 servers to send the resulting log entries to.  Optionally, you can specify
-additional tags to insert into every message received from each syslog
+additional fields to insert into every message received from each syslog
 socket.
 
 It looks like this:
 
     sockets:
-      # These sockets have no additional tags
+      # These sockets have no additional fields
       /tmp/sock1:
       /tmp/sock2:
 
-      # This socket will have its messages tagged
-      /tmp/taggedsock:
-        foo: bar
-        baz: wombat
+      # This socket will have some fields added to its messages, and will
+      # send all messages to a couple of other sockets, too
+      /tmp/supersock:
+        add_fields:
+          foo: bar
+          baz: wombat
+        relay_to:
+          - /tmp/relaysock1
+          - /tmp/relaysock2
 
     # Every log entry received will be sent to *exactly* one of these
     # servers.  This provides high availability for your log messages.
@@ -54,6 +59,23 @@ It looks like this:
       - tcp://10.0.0.2:5151
 
 
+### Socket configuration
+
+Each socket has a configuration associated with it.  Using this
+configuration, you can add logstash fields to each entry, and configure
+socket relaying.
+
+The following keys are available under each socket's path:
+
+* `add_fields` -- A hash of additional fields to add to every log entry that
+  is received on this socket, before it is passed on to logstash.
+
+* `relay_to` -- A list of sockets to send all received messages to.  This is
+  useful in a very limited range of circumstances, when (for instance) you
+  have another syslog socket consumer that wants to get in on the act, like
+  a legacy syslogd.
+
+
 ## Logstash server configuration
 
 You'll need to setup a TCP input, with the `json_lines` codec, for

data/lib/syslogstash.rb

@@ -12,7 +12,7 @@ class Syslogstash
 
 		@writer = LogstashWriter.new(servers, backlog, @metrics)
 
-		@readers = sockets.map { |f, tags| SyslogReader.new(f, tags, @writer, @metrics) }
+		@readers = sockets.map { |f, cfg| SyslogReader.new(f, cfg, @writer, @metrics) }
 	end
 
 	def run
@@ -25,19 +25,19 @@ class Syslogstash
 		dead_thread = tw.next_wait
 
 		if dead_thread == @writer.thread
-			$stderr.puts "Writer thread crashed."
+			$stderr.puts "[Syslogstash] Writer thread crashed."
 		elsif dead_thread == @metrics.thread
-			$stderr.puts "Metrics exporter thread crashed."
+			$stderr.puts "[Syslogstash] Metrics exporter thread crashed."
 		else
 			reader = @readers.find { |r| r.thread == dead_thread }
 
-			$stderr.puts "Reader thread for #{reader.file} crashed."
+			$stderr.puts "[Syslogstash] Reader thread for #{reader.file} crashed."
 		end
 
 		begin
 			dead_thread.join
 		rescue Exception => ex
-			$stderr.puts "Exception in thread was: #{ex.message} (#{ex.class})"
+			$stderr.puts "[Syslogstash] Exception in thread was: #{ex.message} (#{ex.class})"
 			$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
 		end
 

data/lib/syslogstash/logstash_writer.rb

@@ -62,7 +62,7 @@ class Syslogstash::LogstashWriter
 					# to put the entry back on the queue in the ensure block
 					entry = nil
 				rescue StandardError => ex
-					$stderr.puts "Unhandled exception: #{ex.message} (#{ex.class})"
+					log { "Unhandled exception: #{ex.message} (#{ex.class})" }
 					$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
 				ensure
 					@entries_mutex.synchronize { @entries.unshift if entry }

data/lib/syslogstash/prometheus_exporter.rb

@@ -1,6 +1,7 @@
 require 'prometheus/client/rack/exporter'
 require 'rack'
-require 'rack/handler/puma'
+require 'rack/handler/webrick'
+require 'logger'
 
 class Syslogstash::PrometheusExporter
 	attr_reader :thread
@@ -40,7 +41,11 @@ class Syslogstash::PrometheusExporter
 			app.use Prometheus::Client::Rack::Exporter
 			app.run ->(env) { [404, {'Content-Type' => 'text/plain'}, ['Nope']] }
 
-			Rack::Handler::Puma.run app, Host: '[::]', Port: 9159
+			logger = Logger.new($stderr)
+			logger.level = Logger::INFO
+			logger.formatter = proc { |s, t, p, m| "[Syslogstash::PrometheusExporter::WEBrick] #{m}\n" }
+
+			Rack::Handler::WEBrick.run app, BindAddress: '::', Port: 9159, Logger: logger, AccessLog: []
 		end
 	end
 

data/lib/syslogstash/syslog_reader.rb

@@ -7,10 +7,24 @@ class Syslogstash::SyslogReader
 
 	attr_reader :file
 
-	def initialize(file, tags, logstash, metrics)
-		@file, @tags, @logstash, @metrics = file, tags, logstash, metrics
+	def initialize(file, config, logstash, metrics)
+		@file, @logstash, @metrics = file, logstash, metrics
+		config ||= {}
 
-		log { "initializing syslog socket #{file} with tags #{tags.inspect}" }
+		@add_fields = config['add_fields'] || {}
+		@relay_to   = config['relay_to']   || []
+
+		unless @add_fields.is_a? Hash
+			raise ArgumentError,
+			      "add_fields parameter to socket #{file} must be a hash"
+		end
+
+		unless @relay_to.is_a? Array
+			raise ArgumentError,
+			      "relay_to parameter to socket #{file} must be an array"
+		end
+
+		log { "initialized syslog socket #{file} with config #{config.inspect}" }
 	end
 
 	# Start reading from the socket file, parsing entries, and flinging
@@ -28,7 +42,7 @@ class Syslogstash::SyslogReader
 			File.unlink(@file) rescue nil
 			retry
 		rescue SystemCallError
-			$stderr.puts "Error while trying to bind to #{@file}"
+			log { "Error while trying to bind to #{@file}" }
 			raise
 		end
 
@@ -39,6 +53,7 @@ class Syslogstash::SyslogReader
 					debug { "Message received: #{msg.inspect}" }
 					@metrics.received(@file, Time.now)
 					process_message msg.first.chomp
+					relay_message msg.first
 				end
 			ensure
 				socket.close
@@ -87,7 +102,7 @@ class Syslogstash::SyslogReader
 
 			@logstash.send_entry(log_entry)
 		else
-			$stderr.puts "Unparseable message: #{msg}"
+			log { "Unparseable message: #{msg}" }
 		end
 	end
 
@@ -100,13 +115,46 @@ class Syslogstash::SyslogReader
 			h[:severity_name] = SEVERITIES[h[:severity]]
 
 			e.merge!(h.delete_if { |k,v| v.nil? })
-
-			e.merge!(@tags) if @tags.is_a? Hash
+			e.merge!(@add_fields)
 
 			debug { "Log entry is: #{e.inspect}" }
 		end
 	end
 
+	def relay_message(msg)
+		@currently_failed ||= {}
+
+		@relay_to.each do |f|
+			s = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM, 0)
+			begin
+				s.connect(Socket.pack_sockaddr_un(f))
+			rescue Errno::ENOENT
+				# Socket doesn't exist; we don't care enough about this to bother
+				# reporting it.  People will figure it out themselves soon enough.
+			rescue StandardError => ex
+				log { "Error while connecting to relay socket #{f}: #{ex.message} (#{ex.class})" }
+				next
+			end
+
+			begin
+				s.sendmsg_nonblock(msg)
+				if @currently_failed[f]
+					log { "Backlog on socket #{f} has cleared; messages are being delivered again" }
+					@currently_failed[f] = false
+				end
+			rescue Errno::ENOTCONN
+				# Socket isn't being listened to.  Not our problem.
+			rescue IO::EAGAINWaitWritable
+				unless @currently_failed[f]
+					log { "Socket #{f} is backlogged; messages to this socket from socket #{@file} are being discarded undelivered" }
+					@currently_failed = true
+				end
+			rescue StandardError => ex
+				log { "Failed to relay message to socket #{f} from #{@file}: #{ex.message} (#{ex.class})" }
+			end
+		end
+	end
+
 	FACILITIES = %w{
 		kern
 		user

data/lib/syslogstash/worker.rb

@@ -23,12 +23,12 @@ module Syslogstash::Worker
 	private
 
 	def log
-		$stderr.puts "#{Time.now.strftime("%F %T.%L")} #{self.class} #{yield.to_s}"
+		$stderr.puts "[#{self.class}] #{yield.to_s}"
 	end
 
 	def debug
 		if ENV['DEBUG_SYSLOGSTASH']
-			$stderr.puts "#{Time.now.strftime("%F %T.%L")} #{self.class} #{yield.to_s}"
+			$stderr.puts "[#{self.class}] #{yield.to_s}"
 		end
 	end
 end

data/syslogstash.gemspec

@@ -24,7 +24,6 @@ Gem::Specification.new do |s|
 	s.required_ruby_version = ">= 2.1.0"
 
 	s.add_runtime_dependency 'prometheus-client'
-	s.add_runtime_dependency 'puma'
 	s.add_runtime_dependency 'rack'
 
 	s.add_development_dependency 'bundler'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: syslogstash
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-02 00:00:00.000000000 Z
+date: 2016-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: prometheus-client
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: puma
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -210,7 +196,6 @@ files:
 - LICENCE
 - README.md
 - bin/syslogstash
-- lib/.gitkeep
 - lib/syslogstash.rb
 - lib/syslogstash/logstash_writer.rb
 - lib/syslogstash/prometheus_exporter.rb