syslogstash 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e2eed91a10f0ce2e5adede3c8639b37fc4becf3
-  data.tar.gz: b673ee85fcb7bfb6504aadf1941f27f1293dc1af
+  metadata.gz: f013a89558632b9ac7402191a3a1bf7e9399d9af
+  data.tar.gz: 5e56866a48fa92bbc4d3decdd6662d130380f409
 SHA512:
-  metadata.gz: d02d4b1013baebd1aaccc6b11e1fcaf5f1afb4735e8fa272aef9246ec58c39e13ee963911795696264d5e3deaa5f2d5f0d80c9189f1275ba5bebbc6d2cc04fa1
-  data.tar.gz: b5f8d87359e5cf32a564db2d0966392deadb2038bbdc8bc55dc93921ab595b834947a114ba1390288ce7847c506af70b7ecd66ec47e396fc686bf1867d5de85c
+  metadata.gz: 8adb890310f7fbb3e3c0b388361d9f746b7676cc6ab85730728431df938d48b61f911ac86f50223c4b59d1bd4b6e87d0a71ad5e9d6fe26064864d20a0d3e5319
+  data.tar.gz: e67f439643eb30a7af9d7abae8e1abba1d64a0006330d887ebaf128e1377d95d7afd42b121d6149d4669eacb8515d714fb726c65418001e6a6d054b0570d7e89

data/.gitignore

@@ -2,3 +2,4 @@ Gemfile.lock
 /pkg
 /doc
 /.yardoc
+/.bundle

data/lib/syslogstash.rb

@@ -8,22 +8,26 @@ require 'thwait'
 #
 class Syslogstash
 	def initialize(sockets, servers, backlog)
-		@writer = LogstashWriter.new(servers, backlog)
+		@metrics = PrometheusExporter.new
 
-		@readers = sockets.map { |f, tags| SyslogReader.new(f, tags, @writer) }
+		@writer = LogstashWriter.new(servers, backlog, @metrics)
+
+		@readers = sockets.map { |f, tags| SyslogReader.new(f, tags, @writer, @metrics) }
 	end
 
 	def run
+		@metrics.run
 		@writer.run
 		@readers.each { |w| w.run }
 
-		tw = ThreadsWait.new(@writer.thread, *(@readers.map { |r| r.thread }))
+		tw = ThreadsWait.new(@metrics.thread, @writer.thread, *(@readers.map { |r| r.thread }))
 
 		dead_thread = tw.next_wait
 
 		if dead_thread == @writer.thread
 			$stderr.puts "Writer thread crashed."
-			exit 1
+		elsif dead_thread == @metrics.thread
+			$stderr.puts "Metrics exporter thread crashed."
 		else
 			reader = @readers.find { |r| r.thread == dead_thread }
 
@@ -33,7 +37,7 @@ class Syslogstash
 		begin
 			dead_thread.join
 		rescue Exception => ex
-			$stderr.puts "Exception in worker thread was: #{ex.message} (#{ex.class})"
+			$stderr.puts "Exception in thread was: #{ex.message} (#{ex.class})"
 			$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
 		end
 
@@ -43,4 +47,4 @@ end
 
 require_relative 'syslogstash/syslog_reader'
 require_relative 'syslogstash/logstash_writer'
-
+require_relative 'syslogstash/prometheus_exporter'

data/lib/syslogstash/logstash_writer.rb

@@ -10,8 +10,8 @@ class Syslogstash::LogstashWriter
 	# Give it a list of servers, and your writer will be ready to go.
 	# No messages will actually be *delivered*, though, until you call #run.
 	#
-	def initialize(servers, backlog)
-		@servers, @backlog = servers.map { |s| URI(s) }, backlog
+	def initialize(servers, backlog, metrics)
+		@servers, @backlog, @metrics = servers.map { |s| URI(s) }, backlog, metrics
 
 		unless @servers.all? { |url| url.scheme == 'tcp' }
 			raise ArgumentError,
@@ -28,7 +28,7 @@ class Syslogstash::LogstashWriter
 	#
 	def send_entry(e)
 		@entries_mutex.synchronize do
-			@entries << e
+			@entries << { content: e, arrival_timestamp: Time.now }
 			@entries.shift while @entries.length > @backlog
 		end
 		@worker.run if @worker
@@ -53,9 +53,11 @@ class Syslogstash::LogstashWriter
 					entry = @entries_mutex.synchronize { @entries.shift }
 
 					current_server do |s|
-						s.puts entry
+						s.puts entry[:content]
 					end
 
+					@metrics.sent(@servers.last, entry[:arrival_timestamp])
+
 					# If we got here, we sent successfully, so we don't want
 					# to put the entry back on the queue in the ensure block
 					entry = nil
@@ -75,7 +77,7 @@ class Syslogstash::LogstashWriter
 	#
 	# The yielding is very deliberate: it allows us to centralise all
 	# error detection and handling within this one method, and retry
-	# sending just be calling `yield` again when we've connected to
+	# sending just by calling `yield` again when we've connected to
 	# another server.
 	#
 	def current_server

data/lib/syslogstash/prometheus_exporter.rb

@@ -0,0 +1,58 @@
+require 'prometheus/client/rack/exporter'
+require 'rack'
+require 'rack/handler/puma'
+
+class Syslogstash::PrometheusExporter
+	attr_reader :thread
+
+	def initialize
+		@msg_in  = prom.counter(:syslogstash_messages_received, "The number of syslog messages received from each log socket")
+		@msg_out = prom.counter(:syslogstash_messages_sent, "The number of logstash messages sent to each logstash server")
+		@lag     = prom.gauge(:syslogstash_lag_ms, "How far behind we are in relaying messages")
+		@queue   = prom.gauge(:syslogstash_queue_size, "How many messages are queued to be sent")
+	end
+
+	def received(socket, stamp)
+		@msg_in.increment(socket_path: socket)
+		@queue.set({}, (@queue.get({}) || 0) + 1)
+
+		if @most_recent_received.nil? || @most_recent_received < stamp
+			@most_recent_received = stamp
+
+			refresh_lag
+		end
+	end
+
+	def sent(server, stamp)
+		@msg_out.increment(logstash_server: server)
+		@queue.set({}, @queue.get({}) - 1)
+
+		if @most_recent_sent.nil? || @most_recent_sent < stamp
+			@most_recent_sent = stamp
+
+			refresh_lag
+		end
+	end
+
+	def run
+		@thread = Thread.new do
+			app = Rack::Builder.new
+			app.use Prometheus::Client::Rack::Exporter
+			app.run ->(env) { [404, {'Content-Type' => 'text/plain'}, ['Nope']] }
+
+			Rack::Handler::Puma.run app, Host: '[::]', Port: 9159
+		end
+	end
+
+	private
+
+	def prom
+		Prometheus::Client.registry
+	end
+
+	def refresh_lag
+		if @most_recent_received && @most_recent_sent
+			@lag.set({}, ((@most_recent_received.to_f - @most_recent_sent.to_f) * 1000).to_i)
+		end
+	end
+end

data/lib/syslogstash/syslog_reader.rb

@@ -7,8 +7,8 @@ class Syslogstash::SyslogReader
 
 	attr_reader :file
 
-	def initialize(file, tags, logstash)
-		@file, @tags, @logstash = file, tags, logstash
+	def initialize(file, tags, logstash, metrics)
+		@file, @tags, @logstash, @metrics = file, tags, logstash, metrics
 
 		log { "initializing syslog socket #{file} with tags #{tags.inspect}" }
 	end
@@ -37,6 +37,7 @@ class Syslogstash::SyslogReader
 				loop do
 					msg = socket.recvmsg
 					debug { "Message received: #{msg.inspect}" }
+					@metrics.received(@file, Time.now)
 					process_message msg.first.chomp
 				end
 			ensure

data/syslogstash.gemspec

@@ -21,7 +21,11 @@ Gem::Specification.new do |s|
 	s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Rakefile)/ }
 	s.executables = ["syslogstash"]
 
-	s.required_ruby_version = ">= 2.0.0"
+	s.required_ruby_version = ">= 2.1.0"
+
+	s.add_runtime_dependency 'prometheus-client'
+	s.add_runtime_dependency 'puma'
+	s.add_runtime_dependency 'rack'
 
 	s.add_development_dependency 'bundler'
 	s.add_development_dependency 'github-release'

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: syslogstash
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-24 00:00:00.000000000 Z
+date: 2016-07-27 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puma
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +213,7 @@ files:
 - lib/.gitkeep
 - lib/syslogstash.rb
 - lib/syslogstash/logstash_writer.rb
+- lib/syslogstash/prometheus_exporter.rb
 - lib/syslogstash/syslog_reader.rb
 - lib/syslogstash/worker.rb
 - syslogstash.gemspec
@@ -185,7 +228,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="