raindrops 0.4.1 → 0.5.0

data/lib/raindrops/linux.rb

@@ -1,6 +1,14 @@
 # -*- encoding: binary -*-
-class Raindrops
-module Linux
+
+# For reporting TCP ListenStats, users of older \Linux kernels need to ensure
+# that the the "inet_diag" and "tcp_diag" kernel modules are loaded as they do
+# not autoload correctly.  The inet_diag facilities of \Raindrops is useful
+# for periodic snapshot reporting of listen queue sizes.
+#
+# Instead of snapshotting, Raindrops::Aggregate::LastDataRecv may be used
+# to aggregate statistics from +all+ accepted sockets as they arrive
+# based on the +last_data_recv+ field in Raindrops::TCP_Info
+module Raindrops::Linux
 
   # The standard proc path for active UNIX domain sockets, feel free to call
   # String#replace on this if your /proc is mounted in a non-standard location
@@ -11,7 +19,7 @@ module Linux
   # Get ListenStats from an array of +paths+
   #
   # Socket state mapping from integer => symbol, based on socket_state
-  # enum from include/linux/net.h in the Linux kernel:
+  # enum from include/linux/net.h in the \Linux kernel:
   #     typedef enum {
   #             SS_FREE = 0,              /* not allocated                */
   #             SS_UNCONNECTED,           /* unconnected to any socket    */
@@ -26,20 +34,24 @@ module Linux
   # counterpart due to the latter being able to use inet_diag via netlink.
   # This parses /proc/net/unix as there is no other (known) way
   # to expose Unix domain socket statistics over netlink.
-  def unix_listener_stats(paths)
-    rv = Hash.new { |h,k| h[k.freeze] = ListenStats.new(0, 0) }
-    paths = paths.map do |path|
-      path = path.dup
-      path.force_encoding(Encoding::BINARY) if defined?(Encoding)
-      rv[path]
-      Regexp.escape(path)
+  def unix_listener_stats(paths = nil)
+    rv = Hash.new { |h,k| h[k.freeze] = Raindrops::ListenStats.new(0, 0) }
+    if nil == paths
+      paths = [ '[^\n]+' ]
+    else
+      paths = paths.map do |path|
+        path = path.dup
+        path.force_encoding(Encoding::BINARY) if defined?(Encoding)
+        rv[path]
+        Regexp.escape(path)
+      end
     end
-    paths = / 00000000 \d+ (\d+)\s+\d+ (#{paths.join('|')})$/n
+    paths = /^\w+: \d+ \d+ 00000000 \d+ (\d+)\s+\d+ (#{paths.join('|')})$/n
 
     # no point in pread since we can't stat for size on this file
     File.read(*PROC_NET_UNIX_ARGS).scan(paths) do |s|
-      path = s.last
-      case s.first.to_i
+      path = s[-1]
+      case s[0].to_i
       when 2 then rv[path].queued += 1
       when 3 then rv[path].active += 1
       end
@@ -47,8 +59,6 @@ module Linux
 
     rv
   end
-
   module_function :unix_listener_stats
 
-end # Linux
-end # Raindrops
+end # Raindrops::Linux

data/lib/raindrops/middleware.rb

@@ -1,71 +1,144 @@
 # -*- encoding: binary -*-
 require 'raindrops'
 
-# Raindrops middleware should be loaded at the top of Rack
-# middleware stack before other middlewares for maximum accuracy.
-class Raindrops
-class Middleware < ::Struct.new(:app, :stats, :path, :tcp, :unix)
+# Raindrops::Middleware is Rack middleware that allows snapshotting
+# current activity from an HTTP request.  For all operating systems,
+# it returns at least the following fields:
+#
+# * calling - the number of application dispatchers on your machine
+# * writing - the number of clients being written to on your machine
+#
+# Additional fields are available for \Linux users.
+#
+# It should be loaded at the top of Rack middleware stack before other
+# middlewares for maximum accuracy.
+#
+# === Usage (Rainbows!/Unicorn preload_app=false)
+#
+# If you're using preload_app=false (the default) in your Rainbows!/Unicorn
+# config file, you'll need to create the global Stats object before
+# forking.
+#
+#    require 'raindrops'
+#    $stats ||= Raindrops::Middleware::Stats.new
+#
+# In your Rack config.ru:
+#
+#    use Raindrops::Middleware, :stats => $stats
+#
+# === Usage (Rainbows!/Unicorn preload_app=true)
+#
+# If you're using preload_app=true in your Rainbows!/Unicorn
+# config file, just add the middleware to your stack:
+#
+# In your Rack config.ru:
+#
+#    use Raindrops::Middleware
+#
+# === Linux-only extras!
+#
+# To get bound listener statistics under \Linux, you need to specify the
+# listener names for your server.  You can even include listen sockets for
+# *other* servers on the same machine.  This can be handy for monitoring
+# your nginx proxy as well.
+#
+# In your Rack config.ru, just pass the :listeners argument as an array of
+# strings (along with any other arguments).  You can specify any
+# combination of TCP or Unix domain socket names:
+#
+#    use Raindrops::Middleware, :listeners => %w(0.0.0.0:80 /tmp/.sock)
+#
+# If you're running Unicorn 0.98.0 or later, you don't have to pass in
+# the :listeners array, Raindrops will automatically detect the listeners
+# used by Unicorn master process.  This does not detect listeners in
+# different processes, of course.
+#
+# The response body includes the following stats for each listener
+# (see also Raindrops::ListenStats):
+#
+# * active - total number of active clients on that listener
+# * queued - total number of queued (pre-accept()) clients on that listener
+#
+# = Demo Server
+#
+# There is a server running this middleware (and Watcher) at
+#  http://raindrops-demo.bogomips.org/_raindrops
+#
+# Also check out the Watcher demo at http://raindrops-demo.bogomips.org/
+#
+# The demo server is only limited to 30 users, so be sure not to abuse it
+# by using the /tail/ endpoint too much.
+#
+class Raindrops::Middleware
+  attr_accessor :app, :stats, :path, :tcp, :unix # :nodoc:
+
+  # A Raindrops::Struct used to count the number of :calling and :writing
+  # clients.  This struct is intended to be shared across multiple processes
+  # and both counters are updated atomically.
+  #
+  # This is supported on all operating systems supported by Raindrops
+  class Stats < Raindrops::Struct.new(:calling, :writing)
+  end
 
   # :stopdoc:
-  Stats = Raindrops::Struct.new(:calling, :writing)
   PATH_INFO = "PATH_INFO"
+  require "raindrops/middleware/proxy"
   # :startdoc:
 
+  # +app+ may be any Rack application, this middleware wraps it.
+  # +opts+ is a hash that understands the following members:
+  #
+  # * :stats - Raindrops::Middleware::Stats struct (default: Stats.new)
+  # * :path - HTTP endpoint used for reading the stats (default: "/_raindrops")
+  # * :listeners - array of host:port or socket paths (default: from Unicorn)
   def initialize(app, opts = {})
-    super(app, opts[:stats] || Stats.new, opts[:path] || "/_raindrops")
+    @app = app
+    @stats = opts[:stats] || Stats.new
+    @path = opts[:path] || "/_raindrops"
     tmp = opts[:listeners]
     if tmp.nil? && defined?(Unicorn) && Unicorn.respond_to?(:listener_names)
       tmp = Unicorn.listener_names
     end
+    @tcp = @unix = nil
 
     if tmp
-      self.tcp = tmp.grep(/\A[^:]+:\d+\z/)
-      self.unix = tmp.grep(%r{\A/})
-      self.tcp = nil if tcp.empty?
-      self.unix = nil if unix.empty?
+      @tcp = tmp.grep(/\A.+:\d+\z/)
+      @unix = tmp.grep(%r{\A/})
+      @tcp = nil if @tcp.empty?
+      @unix = nil if @unix.empty?
     end
   end
 
   # standard Rack endpoint
-  def call(env)
-    env[PATH_INFO] == path ? stats_response : dup._call(env)
-  end
+  def call(env) # :nodoc:
+    env[PATH_INFO] == @path and return stats_response
+    begin
+      @stats.incr_calling
 
-  def _call(env)
-    stats.incr_calling
-    status, headers, self.app = app.call(env)
+      status, headers, body = @app.call(env)
+      rv = [ status, headers, Proxy.new(body, @stats) ]
 
-    # the Rack server will start writing headers soon after this method
-    stats.incr_writing
-    [ status, headers, self ]
+      # the Rack server will start writing headers soon after this method
+      @stats.incr_writing
+      rv
     ensure
-      stats.decr_calling
-  end
-
-  # yield to the Rack server here for writing
-  def each(&block)
-    app.each(&block)
-  end
-
-  # the Rack server should call this after #each (usually ensure-d)
-  def close
-    stats.decr_writing
-    app.close if app.respond_to?(:close)
+      @stats.decr_calling
+    end
   end
 
-  def stats_response
-    body = "calling: #{stats.calling}\n" \
-           "writing: #{stats.writing}\n"
+  def stats_response  # :nodoc:
+    body = "calling: #{@stats.calling}\n" \
+           "writing: #{@stats.writing}\n"
 
-    if defined?(Linux)
-      Linux.tcp_listener_stats(tcp).each do |addr,stats|
+    if defined?(Raindrops::Linux)
+      Raindrops::Linux.tcp_listener_stats(@tcp).each do |addr,stats|
         body << "#{addr} active: #{stats.active}\n" \
                 "#{addr} queued: #{stats.queued}\n"
-      end if tcp
-      Linux.unix_listener_stats(unix).each do |addr,stats|
+      end if @tcp
+      Raindrops::Linux.unix_listener_stats(@unix).each do |addr,stats|
         body << "#{addr} active: #{stats.active}\n" \
                 "#{addr} queued: #{stats.queued}\n"
-      end if unix
+      end if @unix
     end
 
     headers = {
@@ -74,6 +147,4 @@ class Middleware < ::Struct.new(:app, :stats, :path, :tcp, :unix)
     }
     [ 200, headers, [ body ] ]
   end
-
-end
 end

data/lib/raindrops/middleware/proxy.rb

@@ -0,0 +1,34 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# This class is by Raindrops::Middleware to proxy application response
+# bodies.  There should be no need to use it directly.
+class Raindrops::Middleware::Proxy
+  def initialize(body, stats)
+    @body, @stats = body, stats
+  end
+
+  # yield to the Rack server here for writing
+  def each
+    @body.each { |x| yield x }
+  end
+
+  # the Rack server should call this after #each (usually ensure-d)
+  def close
+    @stats.decr_writing
+    @body.close if @body.respond_to?(:close)
+  end
+
+  # Some Rack servers can optimize response processing if it responds
+  # to +to_path+ via the sendfile(2) system call, we proxy +to_path+
+  # to the underlying body if possible.
+  def to_path
+    @body.to_path
+  end
+
+  # Rack servers use +respond_to?+ to check for the presence of +close+
+  # and +to_path+ methods.
+  def respond_to?(m)
+    m = m.to_sym
+    :close == m || @body.respond_to?(m)
+  end
+end

data/lib/raindrops/struct.rb

@@ -1,7 +1,22 @@
 # -*- encoding: binary -*-
 
+# This is a wrapper around Raindrops objects much like the core Ruby
+# \Struct can be seen as a wrapper around the core \Array class.
+# It's usage is similar to the core \Struct class, except its fields
+# may only be used to house unsigned long integers.
+#
+#   class Foo < Raindrops::Struct.new(:readers, :writers)
+#   end
+#
+#   foo = Foo.new 0, 0
+#
+#   foo.incr_writers    -> 1
+#   foo.incr_readers    -> 1
+#
 class Raindrops::Struct
 
+  # returns a new class derived from Raindrops::Struct and supporting
+  # the given +members+ as fields, just like \Struct.new in core Ruby.
   def self.new(*members)
     members = members.map { |x| x.to_sym }.freeze
     str = <<EOS

data/lib/raindrops/watcher.rb

@@ -0,0 +1,362 @@
+# -*- encoding: binary -*-
+require "thread"
+require "time"
+require "socket"
+require "rack"
+require "aggregate"
+
+# Raindrops::Watcher is a stand-alone Rack application for watching
+# any number of TCP and UNIX listeners (all of them by default).
+#
+# It depends on the {Aggregate RubyGem}[http://rubygems.org/gems/aggregate]
+#
+# In your Rack config.ru:
+#
+#    run Raindrops::Watcher(options = {})
+#
+# It takes the following options hash:
+#
+# - :listeners - an array of listener names, (e.g. %w(0.0.0.0:80 /tmp/sock))
+# - :delay - interval between stats updates in seconds (default: 1)
+#
+# Raindrops::Watcher is compatible any thread-safe/thread-aware Rack
+# middleware.  It does not work well with multi-process web servers
+# but can be used to monitor them.  It consumes minimal resources
+# with the default :delay.
+#
+# == HTTP endpoints
+#
+# === GET /
+#
+# Returns an HTML summary listing of all listen interfaces watched on
+#
+# === GET /active/$LISTENER.txt
+#
+# Returns a plain text summary + histogram with X-* HTTP headers for
+# active connections.
+#
+# e.g.: curl http://example.com/active/0.0.0.0%3A80.txt
+#
+# === GET /active/$LISTENER.html
+#
+# Returns an HTML summary + histogram with X-* HTTP headers for
+# active connections.
+#
+# e.g.: curl http://example.com/active/0.0.0.0%3A80.html
+#
+# === GET /queued/$LISTENER.txt
+#
+# Returns a plain text summary + histogram with X-* HTTP headers for
+# queued connections.
+#
+# e.g.: curl http://example.com/queued/0.0.0.0%3A80.txt
+#
+# === GET /queued/$LISTENER.html
+#
+# Returns an HTML summary + histogram with X-* HTTP headers for
+# queued connections.
+#
+# e.g.: curl http://example.com/queued/0.0.0.0%3A80.html
+#
+# === GET /tail/$LISTENER.txt?active_min=1&queued_min=1
+#
+# Streams chunked a response to the client.
+# Interval is the preconfigured +:delay+ of the application (default 1 second)
+#
+# The response is plain text in the following format:
+#
+#   ISO8601_TIMESTAMP LISTENER_NAME ACTIVE_COUNT QUEUED_COUNT LINEFEED
+#
+# Query parameters:
+#
+# - active_min - do not stream a line until this active count is reached
+# - queued_min - do not stream a line until this queued count is reached
+#
+# == Response headers (mostly the same as Raindrops::LastDataRecv)
+#
+# - X-Count   - number of requests received
+# - X-Last-Reset - date since the last reset
+#
+# The following headers are only present if X-Count is greater than one.
+#
+# - X-Min     - lowest last_data_recv time recorded (in milliseconds)
+# - X-Max     - highest last_data_recv time recorded (in milliseconds)
+# - X-Mean    - mean last_data_recv time recorded (rounded, in milliseconds)
+# - X-Std-Dev - standard deviation of last_data_recv times
+# - X-Outliers-Low - number of low outliers (hopefully many!)
+# - X-Outliers-High - number of high outliers (hopefully zero!)
+#
+# = Demo Server
+#
+# There is a server running this app at http://raindrops-demo.bogomips.org/
+# The Raindrops::Middleware demo is also accessible at
+#   http://raindrops-demo.bogomips.org/_raindrops
+#
+# The demo server is only limited to 30 users, so be sure not to abuse it
+# by using the /tail/ endpoint too much.
+class Raindrops::Watcher
+  # :stopdoc:
+  attr_reader :snapshot
+  include Rack::Utils
+  include Raindrops::Linux
+
+  def initialize(opts = {})
+    @tcp_listeners = @unix_listeners = nil
+    if l = opts[:listeners]
+      tcp, unix = [], []
+      Array(l).each { |addr| (addr =~ %r{\A/} ? unix : tcp) << addr }
+      unless tcp.empty? && unix.empty?
+        @tcp_listeners = tcp
+        @unix_listeners = unix
+      end
+    end
+
+    agg_class = opts[:agg_class] || Aggregate
+    start = Time.now.utc
+    @active = Hash.new { |h,k| h[k] = agg_class.new }
+    @queued = Hash.new { |h,k| h[k] = agg_class.new }
+    @resets = Hash.new { |h,k| h[k] = start }
+    @snapshot = [ start, {} ]
+    @delay = opts[:delay] || 1
+    @lock = Mutex.new
+    @start = Mutex.new
+    @cond = ConditionVariable.new
+    @thr = nil
+  end
+
+  def hostname
+    Socket.gethostname
+  end
+
+  # rack endpoint
+  def call(env)
+    @start.synchronize { @thr ||= aggregator_thread(env["rack.logger"]) }
+    case env["REQUEST_METHOD"]
+    when "HEAD", "GET"
+      get env
+    when "POST"
+      post env
+    else
+      Rack::Response.new(["Method Not Allowed"], 405).finish
+    end
+  end
+
+  def aggregator_thread(logger) # :nodoc:
+    @socket = sock = Raindrops::InetDiagSocket.new
+    thr = Thread.new do
+      begin
+        combined = tcp_listener_stats(@tcp_listeners, sock)
+        combined.merge!(unix_listener_stats(@unix_listeners))
+        @lock.synchronize do
+          combined.each do |addr,stats|
+            @active[addr] << stats.active
+            @queued[addr] << stats.queued
+          end
+          @snapshot = [ Time.now.utc, combined ]
+          @cond.broadcast
+        end
+      rescue => e
+        logger.error "#{e.class} #{e.inspect}"
+      end while sleep(@delay) && @socket
+      sock.close
+    end
+    wait_snapshot
+    thr
+  end
+
+  def active_stats(addr) # :nodoc:
+    @lock.synchronize do
+      tmp = @active[addr] or return
+      [ @resets[addr], tmp.dup ]
+    end
+  end
+
+  def queued_stats(addr) # :nodoc:
+    @lock.synchronize do
+      tmp = @queued[addr] or return
+      [ @resets[addr], tmp.dup ]
+    end
+  end
+
+  def wait_snapshot
+    @lock.synchronize do
+      @cond.wait @lock
+      @snapshot
+    end
+  end
+
+  def agg_to_hash(reset_at, agg)
+    {
+      "X-Count" => agg.count.to_s,
+      "X-Min" => agg.min.to_s,
+      "X-Max" => agg.max.to_s,
+      "X-Mean" => agg.mean.to_s,
+      "X-Std-Dev" => agg.std_dev.to_s,
+      "X-Outliers-Low" => agg.outliers_low.to_s,
+      "X-Outliers-High" => agg.outliers_high.to_s,
+      "X-Last-Reset" => reset_at.httpdate,
+    }
+  end
+
+  def histogram_txt(agg)
+    reset_at, agg = *agg
+    headers = agg_to_hash(reset_at, agg)
+    body = agg.to_s
+    headers["Content-Type"] = "text/plain"
+    headers["Content-Length"] = bytesize(body).to_s
+    [ 200, headers, [ body ] ]
+  end
+
+  def histogram_html(agg, addr)
+    reset_at, agg = *agg
+    headers = agg_to_hash(reset_at, agg)
+    body = "<html>" \
+      "<head><title>#{hostname} - #{escape_html addr}</title></head>" \
+      "<body><table>" <<
+      headers.map { |k,v|
+        "<tr><td>#{k.gsub(/^X-/, '')}</td><td>#{v}</td></tr>"
+      }.join << "</table><pre>#{escape_html agg}</pre>" \
+      "<form action='/reset/#{escape addr}' method='post'>" \
+      "<input type='submit' name='x' value='reset' /></form>" \
+      "</body>"
+    headers["Content-Type"] = "text/html"
+    headers["Content-Length"] = bytesize(body).to_s
+    [ 200, headers, [ body ] ]
+  end
+
+  def get(env)
+    case env["PATH_INFO"]
+    when "/"
+      index
+    when %r{\A/active/(.+)\.txt\z}
+      histogram_txt(active_stats(unescape($1)))
+    when %r{\A/active/(.+)\.html\z}
+      addr = unescape $1
+      histogram_html(active_stats(addr), addr)
+    when %r{\A/queued/(.+)\.txt\z}
+      histogram_txt(queued_stats(unescape($1)))
+    when %r{\A/queued/(.+)\.html\z}
+      addr = unescape $1
+      histogram_html(queued_stats(addr), addr)
+    when %r{\A/tail/(.+)\.txt\z}
+      tail(unescape($1), env)
+    else
+      not_found
+    end
+    rescue Errno::EDOM
+      raise if defined?(retried)
+      retried = true
+      wait_snapshot
+      retry
+  end
+
+  def not_found
+    Rack::Response.new(["Not Found"], 404).finish
+  end
+
+  def post(env)
+    case env["PATH_INFO"]
+    when %r{\A/reset/(.+)\z}
+      reset!(env, unescape($1))
+    else
+      Rack::Response.new(["Not Found"], 404).finish
+    end
+  end
+
+  def reset!(env, addr)
+    @lock.synchronize do
+      @active.include?(addr) or return not_found
+      @active.delete addr
+      @queued.delete addr
+      @resets[addr] = Time.now.utc
+      @cond.wait @lock
+    end
+    req = Rack::Request.new(env)
+    res = Rack::Response.new
+    url = req.referer || "#{req.host_with_port}/"
+    res.redirect(url)
+    res.content_type.replace "text/plain"
+    res.write "Redirecting to #{url}"
+    res.finish
+  end
+
+  def index
+    updated_at, all = snapshot
+    headers = {
+      "Content-Type" => "text/html",
+      "Last-Modified" => updated_at.httpdate,
+    }
+    body = "<html><head>" \
+      "<title>#{hostname} - all interfaces</title>" \
+      "</head><body><h3>Updated at #{updated_at.iso8601}</h3>" \
+      "<table><tr>" \
+        "<th>address</th><th>active</th><th>queued</th><th>reset</th>" \
+      "</tr>" <<
+      all.map do |addr,stats|
+        e_addr = escape addr
+        "<tr>" \
+          "<td><a href='/tail/#{e_addr}.txt'>#{escape_html addr}</a></td>" \
+          "<td><a href='/active/#{e_addr}.html'>#{stats.active}</a></td>" \
+          "<td><a href='/queued/#{e_addr}.html'>#{stats.queued}</a></td>" \
+          "<td><form action='/reset/#{e_addr}' method='post'>" \
+            "<input type='submit' name='x' value='x' /></form></td>" \
+        "</tr>" \
+      end.join << "</table></body></html>"
+    headers["Content-Length"] = bytesize(body).to_s
+    [ 200, headers, [ body ] ]
+  end
+
+  def tail(addr, env)
+    Tailer.new(self, addr, env).finish
+  end
+  # :startdoc:
+
+  # This is the response body returned for "/tail/$ADDRESS.txt".  This
+  # must use a multi-threaded Rack server with streaming response support.
+  # It is an internal class and not expected to be used directly
+  class Tailer
+    def initialize(rdmon, addr, env) # :nodoc:
+      @rdmon = rdmon
+      @addr = addr
+      q = Rack::Utils.parse_query env["QUERY_STRING"]
+      @active_min = q["active_min"].to_i
+      @queued_min = q["queued_min"].to_i
+      len = Rack::Utils.bytesize(addr)
+      len = 35 if len > 35
+      @fmt = "%20s % #{len}s % 10u % 10u\n"
+      case env["HTTP_VERSION"]
+      when "HTTP/1.0", nil
+        @chunk = false
+      else
+        @chunk = true
+      end
+    end
+
+    def finish
+      headers = { "Content-Type" => "text/plain" }
+      headers["Transfer-Encoding"] = "chunked" if @chunk
+      [ 200, headers, self ]
+    end
+
+    # called by the Rack server
+    def each # :nodoc:
+      begin
+        time, all = @rdmon.wait_snapshot
+        stats = all[@addr] or next
+        stats.queued >= @queued_min or next
+        stats.active >= @active_min or next
+        body = sprintf(@fmt, time.iso8601, @addr, stats.active, stats.queued)
+        body = "#{body.size.to_s(16)}\r\n#{body}\r\n" if @chunk
+        yield body
+      end while true
+      yield "0\r\n\r\n" if @chunk
+    end
+  end
+
+  # shuts down the background thread
+  def shutdown
+    @socket = nil
+    @thr.join if @thr
+    @thr = nil
+  end
+end