raindrops 0.16.0 → 0.20.0

data/lib/raindrops/aggregate/pmq.rb

@@ -3,10 +3,10 @@ require "tempfile"
 require "aggregate"
 require "posix_mq"
 require "fcntl"
-require "io/extra"
 require "thread"
+require "stringio"
 
-# \Aggregate + POSIX message queues support for Ruby 1.9 and \Linux
+# \Aggregate + POSIX message queues support for Ruby 1.9+ and \Linux
 #
 # This class is duck-type compatible with \Aggregate and allows us to
 # aggregate and share statistics from multiple processes/threads aided
@@ -14,12 +14,11 @@ require "thread"
 # Raindrops::LastDataRecv Rack application, but can be used independently
 # on compatible Runtimes.
 #
-# Unlike the core of raindrops, this is only supported on Ruby 1.9 and
-# Linux 2.6.  Using this class requires the following additional RubyGems
+# Unlike the core of raindrops, this is only supported on Ruby 1.9+ and
+# Linux 2.6+.  Using this class requires the following additional RubyGems
 # or libraries:
 #
 # * aggregate (tested with 0.2.2)
-# * io-extra  (tested with 1.2.3)
 # * posix_mq  (tested with 1.0.0)
 #
 # == Design
@@ -39,9 +38,9 @@ class Raindrops::Aggregate::PMQ
   # :stopdoc:
   # These constants are for Linux.  This is designed for aggregating
   # TCP_INFO.
-  RDLOCK = [ Fcntl::F_RDLCK ].pack("s @256")
-  WRLOCK = [ Fcntl::F_WRLCK ].pack("s @256")
-  UNLOCK = [ Fcntl::F_UNLCK ].pack("s @256")
+  RDLOCK = [ Fcntl::F_RDLCK ].pack("s @256".freeze).freeze
+  WRLOCK = [ Fcntl::F_WRLCK ].pack("s @256".freeze).freeze
+  UNLOCK = [ Fcntl::F_UNLCK ].pack("s @256".freeze).freeze
   # :startdoc:
 
   # returns the number of dropped messages sent to a POSIX message
@@ -84,6 +83,7 @@ class Raindrops::Aggregate::PMQ
       @wr = File.open(t.path, "wb")
       @rd = File.open(t.path, "rb")
     end
+    @wr.sync = true
     @cached_aggregate = @aggregate
     flush_master
     @mq_send = if opts[:lossy]
@@ -142,8 +142,8 @@ class Raindrops::Aggregate::PMQ
       warn "Unhandled exception in #{__FILE__}:#{__LINE__}: #{e}"
       break
     end while true
-    ensure
-      flush_master
+  ensure
+    flush_master
   end
 
   # Loads the last shared \Aggregate from the master thread/process
@@ -151,7 +151,10 @@ class Raindrops::Aggregate::PMQ
     @cached_aggregate ||= begin
       flush
       Marshal.load(synchronize(@rd, RDLOCK) do |rd|
-        IO.pread rd.fileno, rd.stat.size, 0
+        dst = StringIO.new
+        dst.binmode
+        IO.copy_stream(rd, dst, rd.size, 0)
+        dst.string
       end)
     end
   end
@@ -163,7 +166,8 @@ class Raindrops::Aggregate::PMQ
     dump = Marshal.dump @aggregate
     synchronize(@wr, WRLOCK) do |wr|
       wr.truncate 0
-      IO.pwrite wr.fileno, dump, 0
+      wr.rewind
+      wr.write(dump)
     end
   end
 
@@ -171,24 +175,26 @@ class Raindrops::Aggregate::PMQ
   # worker thread or process
   def stop_master_loop
     sleep 0.1 until mq_send(false)
-    rescue Errno::EINTR
-      retry
+  rescue Errno::EINTR
+    retry
   end
 
   def lock! io, type # :nodoc:
     io.fcntl Fcntl::F_SETLKW, type
-    rescue Errno::EINTR
-      retry
+  rescue Errno::EINTR
+    retry
   end
 
   # we use both a mutex for thread-safety and fcntl lock for process-safety
   def synchronize io, type # :nodoc:
     @mutex.synchronize do
       begin
+        type = type.dup
         lock! io, type
         yield io
       ensure
-        lock! io, UNLOCK
+        lock! io, type.replace(UNLOCK)
+        type.clear
       end
     end
   end

data/lib/raindrops/aggregate.rb

@@ -1,6 +1,6 @@
 # -*- encoding: binary -*-
 #
-# raindrops may use the {aggregate}[http://github.com/josephruscio/aggregate]
+# raindrops may use the {aggregate}[https://github.com/josephruscio/aggregate]
 # RubyGem to aggregate statistics from TCP_Info lookups.
 module Raindrops::Aggregate
   autoload :PMQ, "raindrops/aggregate/pmq"

data/lib/raindrops/linux.rb

@@ -14,8 +14,7 @@ module Raindrops::Linux
   # The standard proc path for active UNIX domain sockets, feel free to call
   # String#replace on this if your /proc is mounted in a non-standard location
   # for whatever reason
-  PROC_NET_UNIX_ARGS = %w(/proc/net/unix)
-  defined?(::Encoding) and PROC_NET_UNIX_ARGS.push({ :encoding => "binary" })
+  PROC_NET_UNIX_ARGS = [ '/proc/net/unix', { encoding: "binary" }]
 
   # Get ListenStats from an array of +paths+
   #
@@ -42,11 +41,11 @@ module Raindrops::Linux
     else
       paths = paths.map do |path|
         path = path.dup
-        path.force_encoding(Encoding::BINARY) if defined?(Encoding)
+        path.force_encoding(Encoding::BINARY)
         if File.symlink?(path)
           link = path
           path = File.readlink(link)
-          path.force_encoding(Encoding::BINARY) if defined?(Encoding)
+          path.force_encoding(Encoding::BINARY)
           rv[link] = rv[path] # vivify ListenerStats
         else
           rv[path] # vivify ListenerStats
@@ -57,7 +56,7 @@ module Raindrops::Linux
     paths = /^\w+: \d+ \d+ (\d+) \d+ (\d+)\s+\d+ (#{paths.join('|')})$/n
 
     # no point in pread since we can't stat for size on this file
-    File.read(*PROC_NET_UNIX_ARGS).scan(paths) do |s|
+    File.read(PROC_NET_UNIX_ARGS[0], encoding: 'binary').scan(paths) do |s|
       path = s[-1]
       case s[0]
       when "00000000" # client sockets

data/lib/raindrops/middleware/proxy.rb

@@ -27,9 +27,9 @@ class Raindrops::Middleware::Proxy
 
   # Rack servers use +respond_to?+ to check for the presence of +close+
   # and +to_path+ methods.
-  def respond_to?(m)
+  def respond_to?(m, include_all = false)
     m = m.to_sym
-    :close == m || @body.respond_to?(m)
+    :close == m || @body.respond_to?(m, include_all)
   end
 
   # Avoid breaking users of non-standard extensions (e.g. #body)

data/lib/raindrops/middleware.rb

@@ -62,9 +62,9 @@ require 'raindrops'
 # = Demo Server
 #
 # There is a server running this middleware (and Watcher) at
-#  http://raindrops-demo.bogomips.org/_raindrops
+#  https://yhbt.net/raindrops-demo/_raindrops
 #
-# Also check out the Watcher demo at http://raindrops-demo.bogomips.org/
+# Also check out the Watcher demo at https://yhbt.net/raindrops-demo/
 #
 # The demo server is only limited to 30 users, so be sure not to abuse it
 # by using the /tail/ endpoint too much.

data/lib/raindrops/watcher.rb

@@ -8,7 +8,7 @@ require "aggregate"
 # Raindrops::Watcher is a stand-alone Rack application for watching
 # any number of TCP and UNIX listeners (all of them by default).
 #
-# It depends on the {Aggregate RubyGem}[http://rubygems.org/gems/aggregate]
+# It depends on the {Aggregate RubyGem}[https://rubygems.org/gems/aggregate]
 #
 # In your Rack config.ru:
 #
@@ -35,28 +35,28 @@ require "aggregate"
 # Returns a plain text summary + histogram with X-* HTTP headers for
 # active connections.
 #
-# e.g.: curl http://raindrops-demo.bogomips.org/active/0.0.0.0%3A80.txt
+# e.g.: curl https://yhbt.net/raindrops-demo/active/0.0.0.0%3A80.txt
 #
 # === GET /active/$LISTENER.html
 #
 # Returns an HTML summary + histogram with X-* HTTP headers for
 # active connections.
 #
-# e.g.: curl http://raindrops-demo.bogomips.org/active/0.0.0.0%3A80.html
+# e.g.: curl https://yhbt.net/raindrops-demo/active/0.0.0.0%3A80.html
 #
 # === GET /queued/$LISTENER.txt
 #
 # Returns a plain text summary + histogram with X-* HTTP headers for
 # queued connections.
 #
-# e.g.: curl http://raindrops-demo.bogomips.org/queued/0.0.0.0%3A80.txt
+# e.g.: curl https://yhbt.net/raindrops-demo/queued/0.0.0.0%3A80.txt
 #
 # === GET /queued/$LISTENER.html
 #
 # Returns an HTML summary + histogram with X-* HTTP headers for
 # queued connections.
 #
-# e.g.: curl http://raindrops-demo.bogomips.org/queued/0.0.0.0%3A80.html
+# e.g.: curl https://yhbt.net/raindrops-demo/queued/0.0.0.0%3A80.html
 #
 # === POST /reset/$LISTENER
 #
@@ -95,9 +95,9 @@ require "aggregate"
 #
 # = Demo Server
 #
-# There is a server running this app at http://raindrops-demo.bogomips.org/
+# There is a server running this app at https://yhbt.net/raindrops-demo/
 # The Raindrops::Middleware demo is also accessible at
-# http://raindrops-demo.bogomips.org/_raindrops
+# https://yhbt.net/raindrops-demo/_raindrops
 #
 # The demo server is only limited to 30 users, so be sure not to abuse it
 # by using the /tail/ endpoint too much.
@@ -106,7 +106,7 @@ class Raindrops::Watcher
   attr_reader :snapshot
   include Rack::Utils
   include Raindrops::Linux
-  DOC_URL = "http://raindrops.bogomips.org/Raindrops/Watcher.html"
+  DOC_URL = "https://yhbt.net/raindrops/Raindrops/Watcher.html"
   Peak = Struct.new(:first, :last)
 
   def initialize(opts = {})
@@ -244,10 +244,10 @@ class Raindrops::Watcher
   def histogram_txt(agg)
     updated_at, reset_at, agg, current, peak = *agg
     headers = agg_to_hash(reset_at, agg, current, peak)
-    body = agg.to_s
+    body = agg.to_s # 7-bit ASCII-clean
     headers["Content-Type"] = "text/plain"
     headers["Expires"] = (updated_at + @delay).httpdate
-    headers["Content-Length"] = bytesize(body).to_s
+    headers["Content-Length"] = body.size.to_s
     [ 200, headers, [ body ] ]
   end
 
@@ -265,7 +265,7 @@ class Raindrops::Watcher
       "</body>"
     headers["Content-Type"] = "text/html"
     headers["Expires"] = (updated_at + @delay).httpdate
-    headers["Content-Length"] = bytesize(body).to_s
+    headers["Content-Length"] = body.size.to_s
     [ 200, headers, [ body ] ]
   end
 
@@ -364,7 +364,7 @@ class Raindrops::Watcher
         "for more information and options." \
       "</p>" \
       "</body></html>"
-    headers["Content-Length"] = bytesize(body).to_s
+    headers["Content-Length"] = body.size.to_s
     [ 200, headers, [ body ] ]
   end
 
@@ -382,7 +382,7 @@ class Raindrops::Watcher
       q = Rack::Utils.parse_query env["QUERY_STRING"]
       @active_min = q["active_min"].to_i
       @queued_min = q["queued_min"].to_i
-      len = Rack::Utils.bytesize(addr)
+      len = addr.size
       len = 35 if len > 35
       @fmt = "%20s % #{len}s % 10u % 10u\n"
       case env["HTTP_VERSION"]

data/lib/raindrops.rb

@@ -12,7 +12,7 @@
 # Unlike many classes in this package, the core Raindrops class is
 # intended to be portable to all reasonably modern *nix systems
 # supporting mmap().  Please let us know if you have portability
-# issues, patches or pull requests at mailto:raindrops-public@bogomips.org
+# issues, patches or pull requests at mailto:raindrops-public@yhbt.net
 class Raindrops
 
   # Used to represent the number of +active+ and +queued+ sockets for
@@ -36,6 +36,30 @@ class Raindrops
     def total
       active + queued
     end
+  end unless defined? ListenStats
+
+  # call-seq:
+  #	Raindrops.new(size, io: nil)	-> raindrops object
+  #
+  # Initializes a Raindrops object to hold +size+ counters.  +size+ is
+  # only a hint and the actual number of counters the object has is
+  # dependent on the CPU model, number of cores, and page size of
+  # the machine.  The actual size of the object will always be equal
+  # or greater than the specified +size+.
+  # If +io+ is provided, then the Raindrops memory will be backed by
+  # the specified file; otherwise, it will allocate anonymous memory.
+  # The IO object must respond to +truncate+, as this is used to set
+  # the size of the file.
+  # If +zero+ is provided, then the memory region is zeroed prior to
+  # returning. This is only meaningful if +io+ is also provided; in
+  # that case it controls whether any existing counter values in +io+
+  # are retained (false) or whether it is entirely zeroed (true).
+  def initialize(size, io: nil, zero: false)
+    # This ruby wrapper exists to handle the keyword-argument handling,
+    # which is otherwise kind of awkward in C. We delegate the keyword
+    # arguments to the _actual_ initialize implementation as positional
+    # args.
+    initialize_cimpl(size, io, zero)
   end
 
   autoload :Linux, 'raindrops/linux'

data/pkg.mk

@@ -60,7 +60,7 @@ doc:: .document .olddoc.yml $(pkg_extra) $(PLACEHOLDERS)
 	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
 	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
 	$(RM) -r doc
-	$(RDOC) -f oldweb
+	$(RDOC) -f dark216
 	$(OLDDOC) merge
 	install -m644 COPYING doc/COPYING
 	install -m644 NEWS doc/NEWS
@@ -127,7 +127,8 @@ publish_doc:
 	-git set-file-times
 	$(MAKE) doc
 	$(MAKE) doc_gz
-	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/ \
+		--exclude index.html* --exclude created.rid*
 	git ls-files | xargs touch
 endif
 

data/raindrops.gemspec

@@ -1,30 +1,26 @@
 # -*- encoding: binary -*-
-ENV["VERSION"] or abort "VERSION= must be specified"
-manifest = File.readlines('.manifest').map! { |x| x.chomp! }
+manifest = File.exist?('.manifest') ?
+  IO.readlines('.manifest').map!(&:chomp!) : `git ls-files`.split("\n")
 test_files = manifest.grep(%r{\Atest/test_.*\.rb\z})
-require 'olddoc'
-extend Olddoc::Gemspec
-name, summary, title = readme_metadata
 
 Gem::Specification.new do |s|
   s.name = %q{raindrops}
-  s.version = ENV["VERSION"].dup
-
+  s.version = (ENV["VERSION"] ||= '0.18.0').dup
   s.authors = ["raindrops hackers"]
-  s.description = readme_description
-  s.email = %q{raindrops@bogomips.org}
+  s.description = File.read('README').split("\n\n")[1]
+  s.email = %q{raindrops-public@yhbt.net}
   s.extensions = %w(ext/raindrops/extconf.rb)
-  s.extra_rdoc_files = extra_rdoc_files(manifest)
+  s.extra_rdoc_files = IO.readlines('.document').map!(&:chomp!).keep_if do |f|
+    File.exist?(f)
+  end
   s.files = manifest
-  s.homepage = Olddoc.config['rdoc_url']
-  s.summary = summary
+  s.homepage = 'https://yhbt.net/raindrops/'
+  s.summary = 'real-time stats for preforking Rack servers'
+  s.required_ruby_version = '>= 1.9.3'
   s.test_files = test_files
   s.add_development_dependency('aggregate', '~> 0.2')
   s.add_development_dependency('test-unit', '~> 3.0')
-  s.add_development_dependency('io-extra', [ '~> 1.2', '>= 1.2.3'])
   s.add_development_dependency('posix_mq', '~> 2.0')
-  s.add_development_dependency('rack', '~> 1.2')
-  s.add_development_dependency('olddoc', '~> 1.0')
-
+  s.add_development_dependency('rack', [ '>= 1.2', '< 3.0' ])
   s.licenses = %w(LGPL-2.1+)
 end

data/test/ipv6_enabled.rb

@@ -2,8 +2,8 @@ def ipv6_enabled?
   tmp = TCPServer.new(ENV["TEST_HOST6"] || '::1', 0)
   tmp.close
   true
-  rescue => e
-    warn "skipping IPv6 tests, host does not seem to be IPv6 enabled:"
-    warn "  #{e.class}: #{e}"
-    false
+rescue => e
+  warn "skipping IPv6 tests, host does not seem to be IPv6 enabled:"
+  warn "  #{e.class}: #{e}"
+  false
 end

data/test/test_aggregate_pmq.rb

@@ -3,7 +3,7 @@ require "raindrops"
 pmq = begin
   Raindrops::Aggregate::PMQ
 rescue LoadError => e
-  warn "W: #{e} skipping test"
+  warn "W: #{e} skipping #{__FILE__}"
   false
 end
 if RUBY_VERSION.to_f < 1.9

data/test/test_inet_diag_socket.rb

@@ -8,7 +8,7 @@ class TestInetDiagSocket < Test::Unit::TestCase
   def test_new
     sock = Raindrops::InetDiagSocket.new
     assert_kind_of Socket, sock
-    assert_kind_of Fixnum, sock.fileno
+    assert_kind_of Integer, sock.fileno
     flags = sock.fcntl(Fcntl::F_GETFD)
     assert_equal Fcntl::FD_CLOEXEC, flags & Fcntl::FD_CLOEXEC
     assert_nil sock.close

data/test/test_last_data_recv_unicorn.rb

@@ -7,7 +7,7 @@ $stderr.sync = $stdout.sync = true
 pmq = begin
   Raindrops::Aggregate::PMQ
 rescue LoadError => e
-  warn "W: #{e} skipping test"
+  warn "W: #{e} skipping #{__FILE__}"
   false
 end
 if RUBY_VERSION.to_f < 1.9

data/test/test_linux.rb

@@ -76,6 +76,7 @@ class TestLinux < Test::Unit::TestCase
 
     assert_equal 0, stats[tmp.path].active
     assert_equal 0, stats[tmp.path].queued
+    us.close
   end
 
   def test_unix_resolves_symlinks
@@ -151,8 +152,8 @@ class TestLinux < Test::Unit::TestCase
     assert_equal 1, stats.size
     assert_equal 0, stats[addr].queued
     assert_equal 1, stats[addr].active
-    ensure
-      nlsock.close
+  ensure
+    nlsock.close
   end
 
   def test_tcp_multi

data/test/test_linux_all_tcp_listen_stats_leak.rb

@@ -37,7 +37,7 @@ class TestLinuxAllTcpListenStatsLeak < Test::Unit::TestCase
     end
     cur_kb = rss_kb
     p [ :cur_kb, cur_kb ]
-    ensure
-      s.close
+  ensure
+    s.close
   end
 end if ENV["STRESS"].to_i != 0

data/test/test_raindrops.rb

@@ -1,6 +1,7 @@
 # -*- encoding: binary -*-
 require 'test/unit'
 require 'raindrops'
+require 'tempfile'
 
 class TestRaindrops < Test::Unit::TestCase
 
@@ -134,7 +135,7 @@ class TestRaindrops < Test::Unit::TestCase
     assert_equal 0, rd[rd.capa - 1]
     assert_equal 1, rd.incr(rd.capa - 1)
     assert_raises(ArgumentError) { rd[rd.capa] }
-    rescue RangeError
+  rescue RangeError
   end # if RUBY_PLATFORM =~ /linux/
 
   def test_evaporate
@@ -162,4 +163,45 @@ class TestRaindrops < Test::Unit::TestCase
     assert status.success?
     assert_equal [ 1, 2 ], tmp.to_ary
   end
+
+  def test_io_backed
+    file = Tempfile.new('test_io_backed')
+    rd = Raindrops.new(4, io: file, zero: true)
+    rd[0] = 123
+    rd[1] = 456
+
+    assert_equal 123, rd[0]
+    assert_equal 456, rd[1]
+
+    rd.evaporate!
+
+    file.rewind
+    data = file.read
+    assert_equal 123, data.unpack('L!')[0]
+    assert_equal 456, data[Raindrops::SIZE..data.size].unpack('L!')[0]
+  end
+
+  def test_io_backed_reuse
+    file = Tempfile.new('test_io_backed')
+    rd = Raindrops.new(4, io: file, zero: true)
+    rd[0] = 123
+    rd[1] = 456
+    rd.evaporate!
+
+    rd = Raindrops.new(4, io: file, zero: false)
+    assert_equal 123, rd[0]
+    assert_equal 456, rd[1]
+  end
+
+  def test_iobacked_noreuse
+    file = Tempfile.new('test_io_backed')
+    rd = Raindrops.new(4, io: file, zero: true)
+    rd[0] = 123
+    rd[1] = 456
+    rd.evaporate!
+
+    rd = Raindrops.new(4, io: file, zero: true)
+    assert_equal 0, rd[0]
+    assert_equal 0, rd[1]
+  end
 end

data/test/{test_linux_tcp_info.rb → test_tcp_info.rb}

@@ -5,15 +5,15 @@ require 'raindrops'
 require 'socket'
 require 'pp'
 $stderr.sync = $stdout.sync = true
-class TestLinuxTCP_Info < Test::Unit::TestCase
+class TestTCP_Info < Test::Unit::TestCase
 
   TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
 
   # Linux kernel commit 5ee3afba88f5a79d0bff07ddd87af45919259f91
   TCP_INFO_useful_listenq = `uname -r`.strip >= '2.6.24'
 
-
-  def test_tcp_server
+  def test_tcp_server_unacked
+    return if RUBY_PLATFORM !~ /linux/ # unacked not implemented on others...
     s = TCPServer.new(TEST_ADDR, 0)
     rv = Raindrops::TCP_Info.new s
     c = TCPSocket.new TEST_ADDR, s.addr[1]
@@ -29,10 +29,8 @@ class TestLinuxTCP_Info < Test::Unit::TestCase
     tmp.get!(s)
     assert_equal before, tmp.object_id
 
-    ensure
-      c.close if c
-      a.close if a
-      s.close
+  ensure
+    [ c, a, s ].compact.each(&:close)
   end
 
   def test_accessors
@@ -42,12 +40,14 @@ class TestLinuxTCP_Info < Test::Unit::TestCase
     assert tcp_info_methods.size >= 32
     tcp_info_methods.each do |m|
       next if m.to_sym == :get!
+      next if ! tmp.respond_to?(m)
       val = tmp.__send__ m
       assert_kind_of Integer, val
       assert val >= 0
     end
-    ensure
-      s.close
+    assert tmp.respond_to?(:state), 'every OS knows about TCP state, right?'
+  ensure
+    s.close
   end
 
   def test_tcp_server_delayed
@@ -60,9 +60,29 @@ class TestLinuxTCP_Info < Test::Unit::TestCase
     a = s.accept
     i = Raindrops::TCP_Info.new(a)
     assert i.last_data_recv >= delay_ms, "#{i.last_data_recv} < #{delay_ms}"
-    ensure
-      c.close if c
-      a.close if a
-      s.close
+  ensure
+    c.close if c
+    a.close if a
+    s.close
+  end
+
+  def test_tcp_server_state_closed
+    s = TCPServer.new(TEST_ADDR, 0)
+    c = TCPSocket.new(TEST_ADDR, s.addr[1])
+    i = Raindrops::TCP_Info.allocate
+    a = s.accept
+    i.get!(a)
+    state = i.state
+    if Raindrops.const_defined?(:TCP)
+      assert_equal state, Raindrops::TCP[:ESTABLISHED]
+    end
+    c = c.close
+    sleep(0.01) # wait for kernel to update state
+    i.get!(a)
+    assert_not_equal state, i.state
+  ensure
+    s.close if s
+    c.close if c
+    a.close if a
   end
-end if RUBY_PLATFORM =~ /linux/
+end if defined? Raindrops::TCP_Info