syslogstash 2.2.0 → 3.0.0

data/syslogstash.gemspec

@@ -1,40 +1,41 @@
 begin
-	require 'git-version-bump'
+  require 'git-version-bump'
 rescue LoadError
-	nil
+  nil
 end
 
 Gem::Specification.new do |s|
-	s.name = "syslogstash"
+  s.name = "syslogstash"
 
-	s.version = GVB.version rescue "0.0.0.1.NOGVB"
-	s.date    = GVB.date    rescue Time.now.strftime("%Y-%m-%d")
+  s.version = GVB.version rescue "0.0.0.1.NOGVB"
+  s.date    = GVB.date    rescue Time.now.strftime("%Y-%m-%d")
 
-	s.platform = Gem::Platform::RUBY
+  s.platform = Gem::Platform::RUBY
 
-	s.summary  = "Send messages from syslog UNIX sockets to logstash"
+  s.summary  = "Send messages from syslog UNIX sockets to logstash"
 
-	s.authors  = ["Matt Palmer"]
-	s.email    = ["matt.palmer@discourse.org"]
-	s.homepage = "https://github.com/discourse/syslogstash"
+  s.authors  = ["Matt Palmer"]
+  s.email    = ["matt.palmer@discourse.org"]
+  s.homepage = "https://github.com/discourse/syslogstash"
 
-	s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Rakefile)/ }
-	s.executables = ["syslogstash"]
+  s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Rakefile)/ }
+  s.executables = ["syslogstash"]
 
-	s.required_ruby_version = ">= 2.1.0"
+  s.required_ruby_version = ">= 2.4.0"
 
-	s.add_runtime_dependency 'frankenstein'
-	s.add_runtime_dependency 'rack'
+  s.add_runtime_dependency 'config_skeleton'
+  s.add_runtime_dependency 'frankenstein'
+  s.add_runtime_dependency 'logstash_writer'
+  s.add_runtime_dependency 'rack'
 
-	s.add_development_dependency 'bundler'
-	s.add_development_dependency 'github-release'
-	s.add_development_dependency 'guard-spork'
-	s.add_development_dependency 'guard-rspec'
-	s.add_development_dependency 'rake', '~> 10.4', '>= 10.4.2'
-	# Needed for guard
-	s.add_development_dependency 'rb-inotify', '~> 0.9'
-	s.add_development_dependency 'redcarpet'
-	s.add_development_dependency 'rspec'
-	s.add_development_dependency 'webmock'
-	s.add_development_dependency 'yard'
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'github-release'
+  s.add_development_dependency 'guard-rspec'
+  s.add_development_dependency 'rake', '~> 10.4', '>= 10.4.2'
+  # Needed for guard
+  s.add_development_dependency 'rb-inotify', '~> 0.9'
+  s.add_development_dependency 'redcarpet'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'webmock'
+  s.add_development_dependency 'yard'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: syslogstash
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-10 00:00:00.000000000 Z
+date: 2019-03-18 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: config_skeleton
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: frankenstein
   requirement: !ruby/object:Gem::Requirement
@@ -25,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rack
+  name: logstash_writer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,13 +53,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -53,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: github-release
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: guard-spork
+  name: github-release
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -192,16 +206,14 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".gitignore"
+- ".travis.yml"
 - Dockerfile
 - LICENCE
-- Makefile
 - README.md
 - bin/syslogstash
 - lib/syslogstash.rb
-- lib/syslogstash/config.rb
-- lib/syslogstash/logstash_writer.rb
-- lib/syslogstash/prometheus_exporter.rb
 - lib/syslogstash/syslog_reader.rb
 - syslogstash.gemspec
 homepage: https://github.com/discourse/syslogstash
@@ -215,15 +227,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.1.0
+      version: 2.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Send messages from syslog UNIX sockets to logstash

data/Makefile

@@ -1,14 +0,0 @@
-IMAGE := discourse/syslogstash
-TAG := $(shell date -u +%Y%m%d.%H%M%S)
-
-.PHONY: default
-default: push
-	@printf "${IMAGE}:${TAG} ready\n"
-
-.PHONY: push
-push: build
-	docker push ${IMAGE}:${TAG}
-
-.PHONY: build
-build:
-	docker build --build-arg=http_proxy=${http_proxy} -t ${IMAGE}:${TAG} .

data/lib/syslogstash/config.rb

@@ -1,118 +0,0 @@
-require 'logger'
-
-class Syslogstash::Config
-	class ConfigurationError < StandardError; end
-
-	# Raised if any problems were found with the config
-	class InvalidEnvironmentError < StandardError; end
-
-	attr_reader :logstash_server,
-	            :syslog_socket,
-	            :backlog_size,
-	            :stats_server,
-	            :add_fields,
-	            :relay_sockets
-
-	attr_reader :logger
-
-	attr_accessor :relay_to_stdout
-
-	# Create a new syslogstash config based on environment variables.
-	#
-	# Examines the environment passed in, and then creates a new config
-	# object if all is well.
-	#
-	# @param env [Hash] the set of environment variables to use.
-	#
-	# @param logger [Logger] the logger to which all diagnostic and error
-	#   data will be sent.
-	#
-	# @raise [ConfigurationError] if any problems are detected with the
-	#   environment variables found.
-	#
-	def initialize(env, logger:)
-		@logger = logger
-
-		parse_env(env)
-	end
-
-	private
-
-	def parse_env(env)
-		@logger.info("config") { "Parsing environment:\n" + env.map { |k, v| "#{k}=#{v.inspect}" }.join("\n") }
-
-		@logstash_server = pluck_string(env, "LOGSTASH_SERVER")
-		@syslog_socket   = pluck_string(env, "SYSLOG_SOCKET")
-		@relay_to_stdout = pluck_boolean(env, "RELAY_TO_STDOUT", default: false)
-		@stats_server    = pluck_boolean(env, "STATS_SERVER", default: false)
-		@backlog_size    = pluck_integer(env, "BACKLOG_SIZE", valid_range: 0..(2**31 - 1), default: 1_000_000)
-		@add_fields      = pluck_prefix_list(env, "ADD_FIELD_")
-		@relay_sockets   = pluck_path_list(env, "RELAY_SOCKETS", default: [])
-	end
-
-	def pluck_string(env, key, default: nil)
-		maybe_default(env, key, default) { env[key] }
-	end
-
-	def pluck_boolean(env, key, default: nil)
-		maybe_default(env, key, default) do
-			case env[key]
-			when /\A(no|off|0|false)\z/
-				false
-			when /\A(yes|on|1|true)\z/
-				true
-			else
-				raise ConfigurationError,
-				      "Value for #{key} (#{env[key].inspect}) is not a valid boolean"
-			end
-		end
-	end
-
-	def pluck_integer(env, key, valid_range: nil, default: nil)
-		maybe_default(env, key, default) do
-			if env[key] !~ /\A\d+\z/
-				raise InvalidEnvironmentError,
-				      "Value for #{key} (#{env[key].inspect}) is not an integer"
-			end
-
-			env[key].to_i.tap do |v|
-				unless valid_range.nil? || !valid_range.include?(v)
-					raise InvalidEnvironmentError,
-					      "Value for #{key} (#{env[key]}) out of range (must be between #{valid_range.first} and #{valid_range.last} inclusive)"
-				end
-			end
-		end
-	end
-
-	def pluck_prefix_list(env, prefix)
-		{}.tap do |list|
-			env.each do |k, v|
-				next unless k.start_with? prefix
-				key = k.sub(prefix, '')
-				list[key] = v
-			end
-
-			@logger.debug("config") { "Prefix list for #{prefix.inspect} is #{list.inspect}" }
-		end
-	end
-
-	def pluck_path_list(env, key, default: nil)
-		maybe_default(env, key, default) do
-			env[key].split(":")
-		end
-	end
-
-	def maybe_default(env, key, default)
-		if env[key].nil? || env[key].empty?
-			if default.nil?
-				raise ConfigurationError,
-				      "Required environment variable #{key} not specified"
-			else
-				@logger.debug("config") { "Using default value #{default.inspect} for config parameter #{key}" }
-				default
-			end
-		else
-			yield.tap { |v| @logger.debug("config") { "Using plucked value #{v.inspect} for config parameter #{key}" } }
-		end
-	end
-end

data/lib/syslogstash/logstash_writer.rb

@@ -1,202 +0,0 @@
-require 'resolv'
-require 'ipaddr'
-
-# Write messages to a logstash server.
-#
-class Syslogstash::LogstashWriter
-	Target = Struct.new(:hostname, :port)
-
-	attr_reader :thread
-
-	# Create a new logstash writer.
-	#
-	# Once the object is created, you're ready to give it messages by
-	# calling #send_entry.  No messages will actually be *delivered* to
-	# logstash, though, until you call #run.
-	#
-	def initialize(cfg, stats)
-		@server_name, @logger, @backlog, @stats = cfg.logstash_server, cfg.logger, cfg.backlog_size, stats
-
-		@entries = []
-		@entries_mutex = Mutex.new
-		@cs_mutex = Mutex.new
-	end
-
-	# Add an entry to the list of messages to be sent to logstash.  Actual
-	# message delivery will happen in a worker thread that is started with
-	# #run.
-	#
-	def send_entry(e)
-		@entries_mutex.synchronize do
-			@entries << { content: e, arrival_timestamp: Time.now }
-			while @entries.length > @backlog
-				@entries.shift
-				@stats.dropped
-			end
-		end
-
-		@thread.run if @thread
-	end
-
-	# Start sending messages to logstash servers.  This method will return
-	# almost immediately, and actual message sending will occur in a
-	# separate thread.
-	#
-	def run
-		@thread = Thread.new { send_messages }
-	end
-
-	# Cause the writer to disconnect from the currently-active server.
-	#
-	def force_disconnect!
-		@cs_mutex.synchronize do
-			@logger.info("writer") { "Forced disconnect from #{server_id(@current_server) }" }
-			@current_server.close if @current_server
-			@current_server = nil
-		end
-	end
-
-	private
-
-	def send_messages
-		loop do
-			if @entries_mutex.synchronize { @entries.empty? }
-				sleep 1
-			else
-				begin
-					entry = @entries_mutex.synchronize { @entries.shift }
-
-					current_server do |s|
-						s.puts entry[:content]
-						@stats.sent(server_id(s), entry[:arrival_timestamp])
-					end
-
-					# If we got here, we sent successfully, so we don't want
-					# to put the entry back on the queue in the ensure block
-					entry = nil
-				rescue StandardError => ex
-					@logger.error("writer") { (["Unhandled exception while writing entry: #{ex.message} (#{ex.class})"] + ex.backtrace).join("\n  ") }
-				ensure
-					@entries_mutex.synchronize { @entries.unshift if entry }
-				end
-			end
-		end
-	end
-
-	# *Yield* a TCPSocket connected to the server we currently believe to
-	# be accepting log entries, so that something can send log entries to
-	# it.
-	#
-	# The yielding is very deliberate: it allows us to centralise all
-	# error detection and handling within this one method, and retry
-	# sending just by calling `yield` again when we've connected to
-	# another server.
-	#
-	def current_server
-		# I could handle this more cleanly with recursion, but I don't want
-		# to fill the stack if we have to retry a lot of times
-		done = false
-
-		until done
-			@cs_mutex.synchronize do
-				if @current_server
-					begin
-						@logger.debug("writer") { "Using current server #{server_id(@current_server)}" }
-						yield @current_server
-						done = true
-					rescue SystemCallError => ex
-						# Something went wrong during the send; disconnect from this
-						# server and recycle
-						@logger.debug("writer") { "Error while writing to current server: #{ex.message} (#{ex.class})" }
-						@current_server.close
-						@current_server = nil
-						sleep 0.1
-					end
-				else
-					candidates = resolve_server_name
-					@logger.debug("writer") { "Server candidates: #{candidates.inspect}" }
-
-					begin
-						next_server = candidates.shift
-
-						if next_server
-							@logger.debug("writer") { "Trying to connect to #{next_server.to_s}" }
-							@current_server = TCPSocket.new(next_server.hostname, next_server.port)
-						else
-							@logger.debug("writer") { "Could not connect to any server; pausing before trying again" }
-							@current_server = nil
-							sleep 5
-						end
-					rescue SystemCallError => ex
-						# Connection failed for any number of reasons; try the next one in the list
-						@logger.warn("writer") { "Failed to connect to #{next_server.to_s}: #{ex.message} (#{ex.class})" }
-						sleep 0.1
-						retry
-					end
-				end
-			end
-		end
-	end
-
-	def server_id(s)
-		pa = s.peeraddr
-		if pa[0] == "AF_INET6"
-			"[#{pa[3]}]:#{pa[1]}"
-		else
-			"#{pa[3]}:#{pa[1]}"
-		end
-	end
-
-	def resolve_server_name
-		return [static_target] if static_target
-
-		# The IPv6 literal case should have been taken care of by
-		# static_target, so the only two cases we have to deal with
-		# here are specified-port (assume A/AAAA) or no port (assume SRV).
-		if @server_name =~ /:/
-			host, port = @server_name.split(":", 2)
-			addrs = Resolv::DNS.new.getaddresses(host)
-			if addrs.empty?
-				@logger.warn("writer") { "No addresses resolved for server_name #{host.inspect}" }
-			end
-			addrs.sort_by { rand }.map { |a| Target.new(a.to_s, port.to_i) }
-		else
-			# SRV records ftw
-			[].tap do |list|
-				left = Resolv::DNS.new.getresources(@server_name, Resolv::DNS::Resource::IN::SRV)
-				if left.empty?
-					@logger.warn("writer") { "No SRV records found for server_name #{@server_name.inspect}" }
-				end
-				until left.empty?
-					prio = left.map { |rr| rr.priority }.uniq.min
-					candidates = left.select { |rr| rr.priority == prio }
-					left -= candidates
-					candidates.sort_by! { |rr| [rr.weight, rr.target.to_s] }
-					until candidates.empty?
-						selector = rand(candidates.inject(1) { |n, rr| n + rr.weight })
-						chosen = candidates.inject(0) do |n, rr|
-							break rr if n + rr.weight >= selector
-							n + rr.weight
-						end
-						candidates.delete(chosen)
-						list << Target.new(chosen.target.to_s, chosen.port)
-					end
-				end
-			end
-		end
-	end
-
-	def static_target
-		@static_target ||= begin
-			if @server_name =~ /\A(.*):(\d+)\z/
-				begin
-					Target.new(IPAddr.new($1).to_s, $2.to_i)
-				rescue ArgumentError
-					# Whatever is on the LHS isn't a recognisable address;
-					# assume hostname and continue
-					nil
-				end
-			end
-		end
-	end
-end