postfix-exporter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4b3403736d6ac37fd2733f6db98f566a36e426de634fe971b96a332dd13527dc
+  data.tar.gz: b20157262dd52237db592744c80713c260ea6eb224e2feb5eac7a60d5665ca40
+SHA512:
+  metadata.gz: 8d99493844dfb43b7b15a3742430f50db2c70ec3506b0c1bd93557a4ca4aca45d22c4eca5a95010571eac486519740858e415f9bae2329ff2a630962056bd10d
+  data.tar.gz: a10120c7f4b0b7a31f36b4fd215bb44ea14868e8281491c473e26812e8b2cfbad0352c87666437087875fb3e095c4036e8b0316461ae82b0077567e938040846

data/.gitignore

@@ -0,0 +1,5 @@
+Gemfile.lock
+/pkg
+/doc
+/.yardoc
+/.bundle

data/README.md

@@ -0,0 +1,50 @@
+This is a simple exporter for various statistics relating to a Postfix
+server.
+
+
+# Features
+
+* Examines mail queue periodically and exports `postfix_mail_queue_size`;
+
+* Reads syslog entries as they happen, and exports disposition status
+  counters (`postfix_delivery_delays_count`) per DSN, as well as delay summaries per
+  DSN (`postfix_delivery_delays{quantile="..."}`);
+
+* Total number of SMTP connections (`postfix_smtpd_connections_total`) and
+  currently-active connections (`postfix_smtpd_active_connections`);
+
+* Count how many delivery attempts were received
+  (`postfix_incoming_delivery_attempts_total`), split out by whether we
+  accepted or rejected the message (`status`) and the exact DSN provided to
+  the client (`dsn`);
+
+* Whether or not the Postfix `master` process is running (`postfix_up`);
+
+* Drinks from the syslog stream directly.
+
+
+# Deployment
+
+The default way of rolling out this exporter is as a Docker container.  For
+that, you'll want to setup a volume for the postfix queue (making it
+available in the container as `/var/spool/postfix`), and some env
+vars.  You can also run it directly (via the gem), with the same env vars,
+and with the expectation that `/var/spool/postfix` is in the usual place.
+
+If you configure the postfix-exporter to run in the same PID namespace as
+whatever it is that's running Postfix itself (either `--pid=host` or in the
+same NS namespace, a la k8s pods), then the `postfix_up` metric will be
+valid, otherwise it'll be random (but unlikely to be correct).
+
+
+## Environment Variables
+
+* `SYSLOG_SOCKET` -- where to read raw syslog events from.  If you're
+  running Postfix in an isolated container, you can probably hook up
+  `/dev/log` directly between containers with a bit of shenanigans,
+  otherwise configure your syslog daemon to write out a copy of all `mail`
+  facility messages to a custom socket.  Alternately, if you use logstash,
+  take a look at the [`syslogstash`
+  gem](https://rubygems.org/gems/syslogstash) and
+  [container](https://hub.docker.com/r/discourse/syslogstash/), and go forth
+  and sin no more.

data/bin/postfix-exporter

@@ -0,0 +1,183 @@
+#!/usr/bin/env ruby
+
+require 'rack'
+require 'prometheus/middleware/exporter'
+require 'socket'
+require 'rack/handler/webrick'
+require 'logger'
+
+prometheus = Prometheus::Client.registry
+
+prometheus.gauge(:postfix_exporter_start_time_seconds, "When this process started up").set({}, Time.now.to_f)
+
+oldest = prometheus.gauge(:postfix_oldest_message_timestamp_seconds, "Queue time of the oldest message")
+mailq  = prometheus.gauge(:postfix_queue_size, "Number of messages in the mail queue")
+q_err  = prometheus.counter(:postfix_queue_processing_error_total, "Exceptions raised whilst scanning the Postfix queue")
+up     = prometheus.gauge(:postfix_up, "Whether the master process is running or not")
+
+Thread.abort_on_exception = true
+
+Thread.new do
+	loop do
+		begin
+			%w{incoming active corrupt hold}.each do |q|
+				mailq.set({ queue: q }, Dir["/var/spool/postfix/#{q}/*"].size)
+			end
+
+			# deferred is special, because it's often hueg it gets sharded into
+			# multiple subdirectories
+			mailq.set({ queue: 'deferred' }, Dir["/var/spool/postfix/deferred/*/*"].size)
+		rescue StandardError => ex
+			$stderr.puts "Error while monitoring queue sizes: #{ex.message} (#{ex.class})"
+			$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
+			q_err.increment(class: ex.class.to_s, phase: "scan")
+		end
+
+		begin
+			master_pid = File.read("/var/spool/postfix/pid/master.pid").to_i
+
+			if master_pid > 1
+				Process.kill(0, master_pid)
+				# If we get here, then the process exists, and
+				# that'll do for our purposes
+				up.set({}, 1)
+			else
+				up.set({}, 0)
+			end
+		rescue Errno::ENOENT, Errno::ESRCH, Errno::EACCES
+			up.set({}, 0)
+		rescue Errno::EPERM
+			# Ironically, we don't need to be able to *actually*
+			# signal the process; EPERM means it exists and is running
+			# as someone more privileged than us, which is enough
+			# for our purposes
+			up.set({}, 1)
+		rescue StandardError => ex
+			$stderr.puts "Error while checking master process: #{ex.message} (#{ex.class})"
+			$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
+			q_err.increment(class: ex.class.to_s, phase: "up")
+		end
+
+		sleep 5
+
+	end
+end
+
+Thread.new do
+	earliest_ctime = ->(glob) do
+		# There is seemingly no way to unset or remove a gauge metric in the Ruby
+		# implementation of the prom exporter.  As a hack, we return the current
+		# time in cases where there is nothing to sample.
+		now = Time.now.to_i
+
+		Dir[glob].lazy.map do |n|
+			begin
+				File.stat(n).ctime.to_i
+			rescue Errno::ENOENT
+				now
+			end
+		end.min || now
+	end
+
+	loop do
+		begin
+			%w{incoming active corrupt hold}.each do |q|
+				oldest.set({ queue: q }, earliest_ctime["/var/spool/postfix/#{q}/*"])
+			end
+			oldest.set({ queue: 'deferred' }, earliest_ctime["/var/spool/postfix/deferred/*/*"])
+		rescue StandardError => ex
+			$stderr.puts "Error while sampling message ages: #{ex.message} (#{ex.class})"
+			$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
+			q_err.increment(class: ex.class.to_s, phase: "stat")
+		end
+
+		# stat()ing all the files in a large queue could potentially be quite
+		# expensive, so we sample this data less frequently.
+		sleep 60
+
+	end
+end
+
+if ENV["SYSLOG_SOCKET"]
+	delays     = prometheus.summary(:postfix_delivery_delays, "Distribution of time taken to deliver (or bounce) messages")
+	connects   = prometheus.counter(:postfix_smtpd_connections_total, "Connections to smtpd")
+	active     = prometheus.gauge(:postfix_smtpd_active_connections, "Current connections to smtpd")
+	incoming   = prometheus.counter(:postfix_incoming_delivery_attempts_total, "Delivery attempts, labelled by dsn and status")
+	messages   = prometheus.counter(:postfix_log_messages_total, "Syslog messages received, labelled by how it was handled")
+	log_errors = prometheus.counter(:postfix_log_processing_error_total, "Exceptions raised whilst processing log messages")
+
+	Thread.new do
+		begin
+			s = Socket.new(Socket::AF_UNIX, Socket::SOCK_DGRAM, 0)
+			s.bind(Socket.pack_sockaddr_un(ENV["SYSLOG_SOCKET"]))
+		rescue Errno::EEXIST, Errno::EADDRINUSE
+			File.unlink ENV["SYSLOG_SOCKET"]
+			retry
+		end
+
+		loop do
+			begin
+				msg = s.recvmsg.first
+				if msg =~ %r{postfix/.* delay=(\d+(\.\d+)?), .* dsn=(\d+\.\d+\.\d+), status=(\w+)}
+					delay  = $1.to_f
+					dsn    = $3
+					status = $4
+
+					if status == "bounced" or status == "sent"
+						delays.observe({dsn: dsn, status: status}, delay)
+					end
+
+					messages.increment(type: "delay")
+				elsif msg =~ %r{postfix/smtpd\[\d+\]: connect from }
+					connects.increment({})
+					active.send(:synchronize) { active.set({}, active.get({}) || 0 + 1) }
+					messages.increment(type: "connect")
+				elsif msg =~ %r{postfix/smtpd\[\d+\]: disconnect from }
+					active.send(:synchronize) do
+					  new = (active.get({}) || 0) - 1
+					  # If we start running mid-stream,
+					  # we might end up seeing more
+					  # disconnects than connections,
+					  # which would be confusing
+					  new = 0 if new < 0
+					  active.set({}, new)
+					end
+					messages.increment(type: "disconnect")
+				elsif msg =~ %r{postfix/smtpd\[\d+\]: [A-F0-9]+: client=}
+					incoming.increment(dsn: "2.0.0", status: "queued")
+					messages.increment(type: "queued")
+				elsif msg =~ %r{postfix/smtpd\[\d+\]: NOQUEUE: reject: RCPT from \S+: \d{3} (\d+\.\d+\.\d+) }
+					incoming.increment(dsn: $1, status: "rejected")
+					messages.increment(type: "noqueue")
+				else
+					messages.increment(type: "ignored")
+				end
+			rescue StandardError => ex
+				$stderr.puts "Error while receiving postfix logs: #{ex.message} (#{ex.class})"
+				$stderr.puts ex.backtrace.map { |l| "  #{l}" }.join("\n")
+				log_errors.increment(class: ex.class.to_s)
+				sleep 1
+			end
+		end
+	end
+end
+
+app = Rack::Builder.new
+app.use Rack::Deflater, if: ->(_, _, _, body) { body.any? && body[0].length > 512 }
+app.use Prometheus::Middleware::Exporter
+app.run ->(env) { [404, {'Content-Type' => 'text/plain'}, ['NOPE NOPE NOPE NOPE']] }
+
+logger = Logger.new($stderr)
+logger.level = Logger::INFO
+logger.formatter = proc { |s, t, p, m| "WEBrick: #{m}\n" }
+
+# This is the only way to get the Rack-mediated webrick to listen on both
+# INADDR_ANY and IN6ADDR_ANY on libcs that don't support getaddrinfo("*")
+# (ie musl-libc).  Setting `Host: '*'` barfs on the above-mentioned buggy(?)
+# libcs, `Host: '::'` fails on newer rubies (because they use
+# setsockopt(V6ONLY) by default), and with RACK_ENV at its default of
+# "development", it only listens on localhost.  And even *this* only works
+# on Rack 2, because before that the non-development default listen address
+# was "0.0.0.0"!
+ENV['RACK_ENV'] = "none"
+Rack::Handler::WEBrick.run app, Port: 9154, Logger: logger, AccessLog: []

data/postfix-exporter.gemspec

@@ -0,0 +1,32 @@
+begin
+	require 'git-version-bump'
+rescue LoadError
+	nil
+end
+
+Gem::Specification.new do |s|
+	s.name = "postfix-exporter"
+
+	s.version = GVB.version rescue "0.0.0.1.NOGVB"
+	s.date    = GVB.date    rescue Time.now.strftime("%Y-%m-%d")
+
+	s.platform = Gem::Platform::RUBY
+
+	s.summary  = "Export Prometheus statistics for a Postfix server"
+
+	s.authors  = ["Matt Palmer"]
+	s.email    = ["matt.palmer@discourse.org"]
+	s.homepage = "https://github.com/discourse/postfix-exporter"
+
+	s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(G|spec|Dockerfile|Rakefile)/ }
+	s.executables = ["postfix-exporter"]
+
+	s.required_ruby_version = ">= 2.1.0"
+
+	s.add_runtime_dependency 'prometheus-client', '~> 0.7', '< 0.10'
+	s.add_runtime_dependency 'rack', '~> 2.0'
+
+	s.add_development_dependency 'bundler'
+	s.add_development_dependency 'github-release'
+	s.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: postfix-exporter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Matt Palmer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-10-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: prometheus-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: github-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- matt.palmer@discourse.org
+executables:
+- postfix-exporter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- README.md
+- bin/postfix-exporter
+- postfix-exporter.gemspec
+homepage: https://github.com/discourse/postfix-exporter
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Export Prometheus statistics for a Postfix server
+test_files: []