raindrops-maintained 0.21.0

data/test/ipv6_enabled.rb

@@ -0,0 +1,9 @@
+def ipv6_enabled?
+  tmp = TCPServer.new(ENV["TEST_HOST6"] || '::1', 0)
+  tmp.close
+  true
+rescue => e
+  warn "skipping IPv6 tests, host does not seem to be IPv6 enabled:"
+  warn "  #{e.class}: #{e}"
+  false
+end

data/test/rack_unicorn.rb

@@ -0,0 +1,11 @@
+# -*- encoding: binary -*-
+require "test/unit"
+require "raindrops"
+require "open-uri"
+begin
+  require "unicorn"
+  require "rack"
+  require "rack/lobster"
+rescue LoadError => e
+  warn "W: #{e} skipping test since Rack or Unicorn was not found"
+end

data/test/test_aggregate_pmq.rb

@@ -0,0 +1,65 @@
+require "test/unit"
+require "raindrops"
+pmq = begin
+  Raindrops::Aggregate::PMQ
+rescue LoadError => e
+  warn "W: #{e} skipping #{__FILE__}"
+  false
+end
+if RUBY_VERSION.to_f < 1.9
+  pmq = false
+  warn "W: skipping #{__FILE__}, only Ruby 1.9 supported for now"
+end
+
+Thread.abort_on_exception = true
+
+class TestAggregatePMQ < Test::Unit::TestCase
+
+  def setup
+    @queue = "/test.#{rand}"
+  end
+
+  def teardown
+    POSIX_MQ.unlink @queue
+  end
+
+  def test_run
+    pmq = Raindrops::Aggregate::PMQ.new :queue => @queue
+    thr = Thread.new { pmq.master_loop }
+    agg = Aggregate.new
+    (1..10).each { |i| pmq << i; agg << i }
+    pmq.stop_master_loop
+    assert thr.join
+    assert_equal agg.count, pmq.count
+    assert_equal agg.mean, pmq.mean
+    assert_equal agg.std_dev, pmq.std_dev
+    assert_equal agg.min, pmq.min
+    assert_equal agg.max, pmq.max
+    assert_equal agg.to_s, pmq.to_s
+  end
+
+  def test_multi_process
+    nr_workers = 4
+    nr = 100
+    pmq = Raindrops::Aggregate::PMQ.new :queue => @queue
+    pid = fork { pmq.master_loop }
+    workers = (1..nr_workers).map {
+      fork {
+        (1..nr).each { |i| pmq << i }
+        pmq.flush
+      }
+    }
+    workers.each { |wpid| assert Process.waitpid2(wpid).last.success? }
+    pmq.stop_master_loop
+    assert Process.waitpid2(pid).last.success?
+    assert_equal 400, pmq.count
+    agg = Aggregate.new
+    (1..nr_workers).map { (1..nr).each { |i| agg << i } }
+    assert_equal agg.to_s, pmq.to_s
+    assert_equal agg.mean, pmq.mean
+    assert_equal agg.std_dev, pmq.std_dev
+    assert_equal agg.min, pmq.min
+    assert_equal agg.max, pmq.max
+    assert_equal agg.to_s, pmq.to_s
+  end
+end if pmq

data/test/test_inet_diag_socket.rb

@@ -0,0 +1,16 @@
+# -*- encoding: binary -*-
+require 'test/unit'
+require 'raindrops'
+require 'fcntl'
+$stderr.sync = $stdout.sync = true
+
+class TestInetDiagSocket < Test::Unit::TestCase
+  def test_new
+    sock = Raindrops::InetDiagSocket.new
+    assert_kind_of Socket, sock
+    assert_kind_of Integer, sock.fileno
+    flags = sock.fcntl(Fcntl::F_GETFD)
+    assert_equal Fcntl::FD_CLOEXEC, flags & Fcntl::FD_CLOEXEC
+    assert_nil sock.close
+  end
+end if RUBY_PLATFORM =~ /linux/

data/test/test_last_data_recv.rb

@@ -0,0 +1,57 @@
+begin
+  require 'aggregate'
+  have_aggregate = true
+rescue LoadError => e
+  warn "W: #{e} skipping #{__FILE__}"
+end
+require 'test/unit'
+require 'raindrops'
+require 'io/wait'
+
+class TestLastDataRecv < Test::Unit::TestCase
+  def setup
+    Raindrops::Aggregate::LastDataRecv.default_aggregate = []
+  end
+
+  def teardown
+    Raindrops::Aggregate::LastDataRecv.default_aggregate = nil
+  end
+
+  def test_accept_nonblock_agg
+    s = Socket.new(:INET, :STREAM, 0)
+    s.listen(128)
+    addr = s.connect_address
+    s.extend(Raindrops::Aggregate::LastDataRecv)
+    s.raindrops_aggregate = []
+    c = Socket.new(:INET, :STREAM, 0)
+    c.connect(addr)
+    c.write '.' # for TCP_DEFER_ACCEPT
+    client, ai = s.accept_nonblock(exception: false)
+    assert client.kind_of?(Socket)
+    assert ai.kind_of?(Addrinfo)
+    assert_equal 1, s.raindrops_aggregate.size
+    assert s.raindrops_aggregate[0].instance_of?(Integer)
+    client, ai = s.accept_nonblock(exception: false)
+    assert_equal :wait_readable, client
+    assert_nil ai
+    assert_equal 1, s.raindrops_aggregate.size
+    assert_raise(IO::WaitReadable) { s.accept_nonblock }
+  end
+
+  def test_accept_nonblock_one
+    s = TCPServer.new('127.0.0.1', 0)
+    s.extend(Raindrops::Aggregate::LastDataRecv)
+    s.raindrops_aggregate = []
+    addr = s.addr
+    c = TCPSocket.new(addr[3], addr[1])
+    c.write '.' # for TCP_DEFER_ACCEPT
+    client = s.accept_nonblock(exception: false)
+    assert client.kind_of?(TCPSocket)
+    assert_equal 1, s.raindrops_aggregate.size
+    assert s.raindrops_aggregate[0].instance_of?(Integer)
+    client = s.accept_nonblock(exception: false)
+    assert_equal :wait_readable, client
+    assert_equal 1, s.raindrops_aggregate.size
+    assert_raise(IO::WaitReadable) { s.accept_nonblock }
+  end
+end if RUBY_PLATFORM =~ /linux/ && have_aggregate

data/test/test_last_data_recv_unicorn.rb

@@ -0,0 +1,69 @@
+# -*- encoding: binary -*-
+require "./test/rack_unicorn"
+require "tempfile"
+require "net/http"
+
+$stderr.sync = $stdout.sync = true
+pmq = begin
+  Raindrops::Aggregate::PMQ
+rescue LoadError => e
+  warn "W: #{e} skipping #{__FILE__}"
+  false
+end
+if RUBY_VERSION.to_f < 1.9
+  pmq = false
+  warn "W: skipping test=#{__FILE__}, only Ruby 1.9 supported for now"
+end
+
+class TestLastDataRecvUnicorn < Test::Unit::TestCase
+  def setup
+    @queue = "/test.#{rand}"
+    @host = ENV["UNICORN_TEST_ADDR"] || "127.0.0.1"
+    @sock = TCPServer.new @host, 0
+    @port = @sock.addr[1]
+    ENV["UNICORN_FD"] = @sock.fileno.to_s
+    @host_with_port = "#@host:#@port"
+    @cfg = Tempfile.new 'unicorn_config_file'
+    @cfg.puts "require 'raindrops'"
+    @cfg.puts "preload_app true"
+      ENV['RAINDROPS_MQUEUE'] = @queue
+    # @cfg.puts "worker_processes 4"
+    @opts = { :listeners => [ @host_with_port ], :config_file => @cfg.path }
+  end
+
+  def test_auto_listener
+    @srv = fork {
+      Thread.abort_on_exception = true
+      app = %q!Rack::Builder.new do
+        map("/ldr") { run Raindrops::LastDataRecv.new }
+        map("/") { run Rack::Lobster.new }
+      end.to_app!
+      def app.arity; 0; end
+      def app.call; eval self; end
+      Unicorn::HttpServer.new(app, @opts).start.join
+    }
+    400.times { assert_kind_of Net::HTTPSuccess, get("/") }
+    resp = get("/ldr")
+    # # p(resp.methods - Object.methods)
+    # resp.each_header { |k,v| p [k, "=" , v] }
+    assert resp.header["x-count"]
+    assert resp.header["x-min"]
+    assert resp.header["x-max"]
+    assert resp.header["x-mean"]
+    assert resp.header["x-std-dev"]
+    assert resp.header["x-outliers-low"]
+    assert resp.header["x-outliers-high"]
+    assert resp.body.size > 0
+  end
+
+  def get(path)
+    Net::HTTP.start(@host, @port) { |http| http.get path }
+  end
+
+  def teardown
+    Process.kill :QUIT, @srv
+    _, status = Process.waitpid2 @srv
+    assert status.success?
+    POSIX_MQ.unlink @queue
+  end
+end if defined?(Unicorn) && RUBY_PLATFORM =~ /linux/ && pmq

data/test/test_linux.rb

@@ -0,0 +1,281 @@
+# -*- encoding: binary -*-
+require 'test/unit'
+require 'tempfile'
+require 'raindrops'
+require 'socket'
+require 'pp'
+$stderr.sync = $stdout.sync = true
+
+class TestLinux < Test::Unit::TestCase
+  include Raindrops::Linux
+
+  TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+
+  def setup
+    @to_close = []
+  end
+
+  def teardown
+    @to_close.each { |io| io.close unless io.closed? }
+  end
+
+  def test_unix
+    tmp = Tempfile.new("\xde\xad\xbe\xef") # valid path, really :)
+    File.unlink(tmp.path)
+    us = UNIXServer.new(tmp.path)
+    stats = unix_listener_stats([tmp.path])
+    assert_equal 1, stats.size
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 0, stats[tmp.path].queued
+
+    @to_close << UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats([tmp.path])
+    assert_equal 1, stats.size
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+
+    @to_close << UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats([tmp.path])
+    assert_equal 1, stats.size
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 2, stats[tmp.path].queued
+
+    @to_close << us.accept
+    stats = unix_listener_stats([tmp.path])
+    assert_equal 1, stats.size
+    assert_equal 1, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+  end
+
+  def test_unix_all
+    tmp = Tempfile.new("\xde\xad\xbe\xef") # valid path, really :)
+    File.unlink(tmp.path)
+    us = UNIXServer.new(tmp.path)
+    @to_close << UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+
+    @to_close << UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 2, stats[tmp.path].queued
+
+    @to_close << us.accept
+    stats = unix_listener_stats
+    assert_equal 1, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+  end
+
+  def test_unix_all_unused
+    tmp = Tempfile.new("\xde\xad\xbe\xef") # valid path, really :)
+    File.unlink(tmp.path)
+    us = UNIXServer.new(tmp.path)
+    stats = unix_listener_stats
+    assert stats.keys.include?(tmp.path), stats.inspect
+
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 0, stats[tmp.path].queued
+    us.close
+  end
+
+  def test_unix_resolves_symlinks
+    tmp = Tempfile.new("\xde\xad\xbe\xef") # valid path, really :)
+    File.unlink(tmp.path)
+    us = UNIXServer.new(tmp.path)
+
+    # Create a symlink
+    link = Tempfile.new("somethingelse")
+    File.unlink(link.path) # We need an available name, not an actual file
+    File.symlink(tmp.path, link.path)
+
+    @to_close << UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+
+    @to_close << UNIXSocket.new(link.path)
+    stats = unix_listener_stats([link.path])
+    assert_equal 0, stats[link.path].active
+    assert_equal 2, stats[link.path].queued
+
+    assert_equal stats[link.path].object_id, stats[tmp.path].object_id
+
+    @to_close << us.accept
+    stats = unix_listener_stats
+    assert_equal 1, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+  end
+
+  def test_tcp
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
+    addr = "#{TEST_ADDR}:#{port}"
+    addrs = [ addr ]
+    stats = tcp_listener_stats(addrs)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    @to_close << TCPSocket.new(TEST_ADDR, port)
+    stats = tcp_listener_stats(addrs)
+    assert_equal 1, stats.size
+    assert_equal 1, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    @to_close << s.accept
+    stats = tcp_listener_stats(addrs)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 1, stats[addr].active
+  end
+
+  def test_tcp_reuse_sock
+    nlsock = Raindrops::InetDiagSocket.new
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
+    addr = "#{TEST_ADDR}:#{port}"
+    addrs = [ addr ]
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    @to_close << TCPSocket.new(TEST_ADDR, port)
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 1, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    @to_close << s.accept
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 1, stats[addr].active
+  ensure
+    nlsock.close
+  end
+
+  def test_tcp_multi
+    s1 = TCPServer.new(TEST_ADDR, 0)
+    s2 = TCPServer.new(TEST_ADDR, 0)
+    port1, port2 = s1.addr[1], s2.addr[1]
+    addr1, addr2 = "#{TEST_ADDR}:#{port1}", "#{TEST_ADDR}:#{port2}"
+    addrs = [ addr1, addr2 ]
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 0, stats[addr1].queued
+    assert_equal 0, stats[addr1].active
+    assert_equal 0, stats[addr2].queued
+    assert_equal 0, stats[addr2].active
+
+    @to_close << TCPSocket.new(TEST_ADDR, port1)
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 1, stats[addr1].queued
+    assert_equal 0, stats[addr1].active
+    assert_equal 0, stats[addr2].queued
+    assert_equal 0, stats[addr2].active
+
+    sc1 = s1.accept
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 0, stats[addr1].queued
+    assert_equal 1, stats[addr1].active
+    assert_equal 0, stats[addr2].queued
+    assert_equal 0, stats[addr2].active
+
+    @to_close << TCPSocket.new(TEST_ADDR, port2)
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 0, stats[addr1].queued
+    assert_equal 1, stats[addr1].active
+    assert_equal 1, stats[addr2].queued
+    assert_equal 0, stats[addr2].active
+
+    @to_close << TCPSocket.new(TEST_ADDR, port2)
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 0, stats[addr1].queued
+    assert_equal 1, stats[addr1].active
+    assert_equal 2, stats[addr2].queued
+    assert_equal 0, stats[addr2].active
+
+    @to_close << s2.accept
+    stats = tcp_listener_stats(addrs)
+    assert_equal 2, stats.size
+    assert_equal 0, stats[addr1].queued
+    assert_equal 1, stats[addr1].active
+    assert_equal 1, stats[addr2].queued
+    assert_equal 1, stats[addr2].active
+
+    sc1.close
+    stats = tcp_listener_stats(addrs)
+    assert_equal 0, stats[addr1].queued
+    assert_equal 0, stats[addr1].active
+    assert_equal 1, stats[addr2].queued
+    assert_equal 1, stats[addr2].active
+
+    # make sure we don't leave "true" placeholders in results if a
+    # listener becomes invalid (even momentarily).
+    s2.close
+    stats = tcp_listener_stats(addrs)
+    assert stats.values.all? { |x| x.instance_of?(Raindrops::ListenStats) },
+      "placeholders left: #{stats.inspect}"
+  end
+
+  # tries to overflow buffers
+  def test_tcp_stress_test
+    nr_proc = 32
+    nr_sock = 500
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
+    addr = "#{TEST_ADDR}:#{port}"
+    addrs = [ addr ]
+    rda, wra = IO.pipe
+    rdb, wrb = IO.pipe
+
+    nr_proc.times do
+      fork do
+        rda.close
+        wrb.close
+        @to_close.concat((1..nr_sock).map { s.accept })
+        wra.syswrite('.')
+        wra.close
+        rdb.sysread(1) # wait for parent to nuke us
+      end
+    end
+
+    nr_proc.times do
+      fork do
+        rda.close
+        wrb.close
+        @to_close.concat((1..nr_sock).map { TCPSocket.new(TEST_ADDR, port) })
+        wra.syswrite('.')
+        wra.close
+        rdb.sysread(1) # wait for parent to nuke us
+      end
+    end
+
+    assert_equal('.' * (nr_proc * 2), rda.read(nr_proc * 2))
+
+    rda.close
+    stats = tcp_listener_stats(addrs)
+    expect = { addr => Raindrops::ListenStats[nr_sock * nr_proc, 0] }
+    assert_equal expect, stats
+
+    @to_close << TCPSocket.new(TEST_ADDR, port)
+    stats = tcp_listener_stats(addrs)
+    expect = { addr => Raindrops::ListenStats[nr_sock * nr_proc, 1] }
+    assert_equal expect, stats
+
+    if ENV["BENCHMARK"].to_i != 0
+      require 'benchmark'
+      puts(Benchmark.measure{1000.times { tcp_listener_stats(addrs) }})
+    end
+
+    wrb.syswrite('.' * (nr_proc * 2)) # broadcast a wakeup
+    statuses = Process.waitall
+    statuses.each { |(_,status)| assert status.success?, status.inspect }
+  end if ENV["STRESS"].to_i != 0
+end if RUBY_PLATFORM =~ /linux/

data/test/test_linux_all_tcp_listen_stats.rb

@@ -0,0 +1,66 @@
+# -*- encoding: binary -*-
+require 'test/unit'
+require 'socket'
+require 'raindrops'
+require 'pp'
+$stderr.sync = $stdout.sync = true
+
+class TestLinuxAllTcpListenStats < Test::Unit::TestCase
+  include Raindrops::Linux
+  TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+
+  def test_print_all
+    puts "EVERYTHING"
+    pp Raindrops::Linux.tcp_listener_stats
+    puts("-" * 72)
+  end if $stdout.tty?
+
+  def setup
+    @socks = []
+  end
+
+  def teardown
+    @socks.each { |io| io.closed? or io.close }
+  end
+
+  def new_server
+    s = TCPServer.new TEST_ADDR, 0
+    @socks << s
+    [ s, s.addr[1] ]
+  end
+
+  def new_client(port)
+    s = TCPSocket.new("127.0.0.1", port)
+    @socks << s
+    s
+  end
+
+  def new_accept(srv)
+    c = srv.accept
+    @socks << c
+    c
+  end
+
+  def test_all_ports
+    srv, port = new_server
+    addr = "#{TEST_ADDR}:#{port}"
+    all = Raindrops::Linux.tcp_listener_stats
+    assert_equal [0,0], all[addr].to_a
+
+    new_client(port)
+    all = Raindrops::Linux.tcp_listener_stats
+    assert_equal [0,1], all[addr].to_a
+
+    new_client(port)
+    all = Raindrops::Linux.tcp_listener_stats
+    assert_equal [0,2], all[addr].to_a
+
+    new_accept(srv)
+    all = Raindrops::Linux.tcp_listener_stats
+    assert_equal [1,1], all[addr].to_a
+
+    new_accept(srv)
+    all = Raindrops::Linux.tcp_listener_stats
+    assert_equal [2,0], all[addr].to_a
+  end
+end if RUBY_PLATFORM =~ /linux/

data/test/test_linux_all_tcp_listen_stats_leak.rb

@@ -0,0 +1,43 @@
+# -*- encoding: binary -*-
+require 'test/unit'
+require 'raindrops'
+require 'socket'
+require 'benchmark'
+$stderr.sync = $stdout.sync = true
+
+class TestLinuxAllTcpListenStatsLeak < Test::Unit::TestCase
+
+  TEST_ADDR = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+
+
+  def rss_kb
+    File.readlines("/proc/#$$/status").grep(/VmRSS:/)[0].split(/\s+/)[1].to_i
+  end
+  def test_leak
+    s = TCPServer.new(TEST_ADDR, 0)
+    start_kb = rss_kb
+    p [ :start_kb, start_kb ]
+    assert_nothing_raised do
+      p(Benchmark.measure {
+        1000.times { Raindrops::Linux.all_tcp_listener_stats }
+      })
+    end
+    cur_kb = rss_kb
+    p [ :cur_kb, cur_kb ]
+    now = Time.now.to_i
+    fin = now + 60
+    assert_nothing_raised do
+      1000000000.times { |i|
+        if (i % 1024) == 0
+          now = Time.now.to_i
+          break if now > fin
+        end
+        Raindrops::Linux.all_tcp_listener_stats
+      }
+    end
+    cur_kb = rss_kb
+    p [ :cur_kb, cur_kb ]
+  ensure
+    s.close
+  end
+end if ENV["STRESS"].to_i != 0