raindrops 0.4.1 → 0.5.0

data/pkg.mk

@@ -0,0 +1,171 @@
+RUBY = ruby
+RAKE = rake
+RSYNC = rsync
+WRONGDOC = wrongdoc
+
+GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
+	@./GIT-VERSION-GEN
+-include GIT-VERSION-FILE
+-include local.mk
+DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts RbConfig::CONFIG["DLEXT"]')
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+lib := lib
+
+ifeq ($(shell test -f script/isolate_for_tests && echo t),t)
+isolate_libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/isolate.mk
+$(isolate_libs): script/isolate_for_tests
+	@$(RUBY) script/isolate_for_tests
+-include $(isolate_libs)
+lib := $(lib):$(ISOLATE_LIBS)
+endif
+
+ext := $(firstword $(wildcard ext/*))
+ifneq ($(ext),)
+ext_pfx := tmp/ext/$(RUBY_ENGINE)-$(RUBY_VERSION)
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(wildcard $(ext)/*.c $(ext_h))
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_d := $(ext_pfx)/$(ext)/.d
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>> $@
+$(ext_d):
+	@mkdir -p $(@D)
+	@> $@
+$(ext_pfx)/$(ext)/%: $(ext)/% $(ext_d)
+	install -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_d) $(ext_h)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	@echo $^ == $@
+	$(MAKE) -C $(@D)
+lib := $(lib):$(ext_pfx)/$(ext)
+build: $(ext_dl)
+else
+build:
+endif
+
+pkg_extra += GIT-VERSION-FILE NEWS ChangeLog LATEST
+ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
+	$(WRONGDOC) prepare
+NEWS LATEST: ChangeLog
+
+manifest:
+	$(RM) .manifest
+	$(MAKE) .manifest
+
+.manifest: $(pkg_extra)
+	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
+		LC_ALL=C sort > $@+
+	cmp $@+ $@ || mv $@+ $@
+	$(RM) $@+
+
+doc:: .document .wrongdoc.yml $(pkg_extra)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(WRONGDOC) all
+	install -m644 COPYING doc/COPYING
+	install -m644 $(shell grep '^[A-Z]' .document) doc/
+
+ifneq ($(VERSION),)
+pkggem := pkg/$(rfpackage)-$(VERSION).gem
+pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
+release_notes := release_notes-$(VERSION)
+release_changes := release_changes-$(VERSION)
+
+release-notes: $(release_notes)
+release-changes: $(release_changes)
+$(release_changes):
+	$(WRONGDOC) release_changes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+$(release_notes):
+	$(WRONGDOC) release_notes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+
+# ensures we're actually on the tagged $(VERSION), only used for release
+verify:
+	test x"$(shell umask)" = x0022
+	git rev-parse --verify refs/tags/v$(VERSION)^{}
+	git diff-index --quiet HEAD^0
+	test $$(git rev-parse --verify HEAD^0) = \
+	     $$(git rev-parse --verify refs/tags/v$(VERSION)^{})
+
+fix-perms:
+	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
+	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
+
+gem: $(pkggem)
+
+install-gem: $(pkggem)
+	gem install $(CURDIR)/$<
+
+$(pkggem): manifest fix-perms
+	gem build $(rfpackage).gemspec
+	mkdir -p pkg
+	mv $(@F) $@
+
+$(pkgtgz): distdir = $(basename $@)
+$(pkgtgz): HEAD = v$(VERSION)
+$(pkgtgz): manifest fix-perms
+	@test -n "$(distdir)"
+	$(RM) -r $(distdir)
+	mkdir -p $(distdir)
+	tar cf - $$(cat .manifest) | (cd $(distdir) && tar xf -)
+	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
+	mv $@+ $@
+
+package: $(pkgtgz) $(pkggem)
+
+test-release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	@echo rubyforge add_release -f \
+	  -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	@echo gem push $(pkggem)
+	@echo rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	# push gem to RubyGems.org
+	gem push $(pkggem)
+	# in case of gem downloads from RubyForge releases page
+	rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+else
+gem install-gem: GIT-VERSION-FILE
+	$(MAKE) $@ VERSION=$(GIT_VERSION)
+endif
+
+all:: test
+test_units := $(wildcard test/test_*.rb)
+test: test-unit
+test-unit: $(test_units)
+$(test_units): build
+	$(RUBY) -I $(lib) $@ $(RUBY_TEST_OPTS)
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	find doc/images -type f | \
+		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+
+.PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest

data/raindrops.gemspec

@@ -1,38 +1,28 @@
 # -*- encoding: binary -*-
-
 ENV["VERSION"] or abort "VERSION= must be specified"
 manifest = File.readlines('.manifest').map! { |x| x.chomp! }
 test_files = manifest.grep(%r{\Atest/test_.*\.rb\z})
+require 'wrongdoc'
+extend Wrongdoc::Gemspec
+name, summary, title = readme_metadata
 
 Gem::Specification.new do |s|
   s.name = %q{raindrops}
-  s.version = ENV["VERSION"]
+  s.version = ENV["VERSION"].dup
 
   s.authors = ["raindrops hackers"]
   s.date = Time.now.utc.strftime('%Y-%m-%d')
-  s.description = File.read("README").split(/\n\n/)[1]
+  s.description = readme_description
   s.email = %q{raindrops@librelist.com}
   s.extensions = %w(ext/raindrops/extconf.rb)
-
-  s.extra_rdoc_files = File.readlines('.document').map! do |x|
-    x.chomp!
-    if File.directory?(x)
-      manifest.grep(%r{\A#{x}/})
-    elsif File.file?(x)
-      x
-    else
-      nil
-    end
-  end.flatten.compact
-
+  s.extra_rdoc_files = extra_rdoc_files(manifest)
   s.files = manifest
-  s.homepage = %q{http://raindrops.bogomips.org/}
-  s.summary = %q{real-time stats for preforking Rack servers}
-  s.rdoc_options = [ "-Na", "-t", "raindrops - #{s.summary}" ]
-  s.require_paths = %w(lib)
+  s.homepage = Wrongdoc.config[:rdoc_url]
+  s.summary = summary
+  s.rdoc_options = rdoc_options
   s.rubyforge_project = %q{rainbows}
-
   s.test_files = test_files
+  s.add_development_dependency('bundler', '~> 1.0.10')
 
   # s.licenses = %w(LGPLv3) # accessor not compatible with older RubyGems
 end

data/test/ipv6_enabled.rb

@@ -0,0 +1,10 @@
+def ipv6_enabled?
+  tmp = TCPServer.new(ENV["TEST_HOST6"] || '::1', 0)
+  tmp.close
+  true
+  rescue => e
+    warn "skipping IPv6 tests, host does not seem to be IPv6 enabled:"
+    warn "  #{e.class}: #{e}"
+    return false
+    ipv6_enabled = false
+end

data/test/rack_unicorn.rb

@@ -0,0 +1,12 @@
+# -*- encoding: binary -*-
+require "test/unit"
+require "raindrops"
+require "rack"
+require "rack/lobster"
+require "open-uri"
+begin
+  require "unicorn"
+  require "rack/lobster"
+rescue LoadError => e
+  warn "W: #{e} skipping test since Rack or Unicorn was not found"
+end

data/test/test_aggregate_pmq.rb

@@ -0,0 +1,65 @@
+require "test/unit"
+require "raindrops"
+pmq = begin
+  Raindrops::Aggregate::PMQ
+rescue => LoadError
+  warn "W: #{e} skipping test"
+  false
+end
+if RUBY_VERSION.to_f < 1.9
+  pmq = false
+  warn "W: skipping #{__FILE__}, only Ruby 1.9 supported for now"
+end
+
+Thread.abort_on_exception = true
+
+class TestAggregatePMQ < Test::Unit::TestCase
+
+  def setup
+    @queue = "/test.#{rand}"
+  end
+
+  def teardown
+    POSIX_MQ.unlink @queue
+  end
+
+  def test_run
+    pmq = Raindrops::Aggregate::PMQ.new :queue => @queue
+    thr = Thread.new { pmq.master_loop }
+    agg = Aggregate.new
+    (1..10).each { |i| pmq << i; agg << i }
+    pmq.stop_master_loop
+    assert thr.join
+    assert_equal agg.count, pmq.count
+    assert_equal agg.mean, pmq.mean
+    assert_equal agg.std_dev, pmq.std_dev
+    assert_equal agg.min, pmq.min
+    assert_equal agg.max, pmq.max
+    assert_equal agg.to_s, pmq.to_s
+  end
+
+  def test_multi_process
+    nr_workers = 4
+    nr = 100
+    pmq = Raindrops::Aggregate::PMQ.new :queue => @queue
+    pid = fork { pmq.master_loop }
+    workers = (1..nr_workers).map {
+      fork {
+        (1..nr).each { |i| pmq << i }
+        pmq.flush
+      }
+    }
+    workers.each { |pid| assert Process.waitpid2(pid).last.success? }
+    pmq.stop_master_loop
+    assert Process.waitpid2(pid).last.success?
+    assert_equal 400, pmq.count
+    agg = Aggregate.new
+    (1..nr_workers).map { (1..nr).each { |i| agg << i } }
+    assert_equal agg.to_s, pmq.to_s
+    assert_equal agg.mean, pmq.mean
+    assert_equal agg.std_dev, pmq.std_dev
+    assert_equal agg.min, pmq.min
+    assert_equal agg.max, pmq.max
+    assert_equal agg.to_s, pmq.to_s
+  end
+end if pmq

data/test/test_inet_diag_socket.rb

@@ -0,0 +1,13 @@
+# -*- encoding: binary -*-
+require 'test/unit'
+require 'raindrops'
+$stderr.sync = $stdout.sync = true
+
+class TestInetDiagSocket < Test::Unit::TestCase
+  def test_new
+    sock = Raindrops::InetDiagSocket.new
+    assert_kind_of Socket, sock
+    assert_kind_of Fixnum, sock.fileno
+    assert_nil sock.close
+  end
+end if RUBY_PLATFORM =~ /linux/

data/test/test_last_data_recv_unicorn.rb

@@ -0,0 +1,69 @@
+# -*- encoding: binary -*-
+require "./test/rack_unicorn"
+require "tempfile"
+require "net/http"
+
+$stderr.sync = $stdout.sync = true
+pmq = begin
+  Raindrops::Aggregate::PMQ
+rescue => LoadError
+  warn "W: #{e} skipping test"
+  false
+end
+if RUBY_VERSION.to_f < 1.9
+  pmq = false
+  warn "W: skipping test=#{__FILE__}, only Ruby 1.9 supported for now"
+end
+
+class TestLastDataRecvUnicorn < Test::Unit::TestCase
+  def setup
+    @queue = "/test.#{rand}"
+    @host = ENV["UNICORN_TEST_ADDR"] || "127.0.0.1"
+    @sock = TCPServer.new @host, 0
+    @port = @sock.addr[1]
+    ENV["UNICORN_FD"] = @sock.fileno.to_s
+    @host_with_port = "#@host:#@port"
+    @cfg = Tempfile.new 'unicorn_config_file'
+    @cfg.puts "require 'raindrops'"
+    @cfg.puts "preload_app true"
+      ENV['RAINDROPS_MQUEUE'] = @queue
+    # @cfg.puts "worker_processes 4"
+    @opts = { :listeners => [ @host_with_port ], :config_file => @cfg.path }
+  end
+
+  def test_auto_listener
+    @srv = fork {
+      Thread.abort_on_exception = true
+      app = %q!Rack::Builder.new do
+        map("/ldr") { run Raindrops::LastDataRecv.new }
+        map("/") { run Rack::Lobster.new }
+      end.to_app!
+      def app.arity; 0; end
+      def app.call; eval self; end
+      Unicorn.run(app, @opts)
+    }
+    400.times { assert_kind_of Net::HTTPSuccess, get("/") }
+    resp = get("/ldr")
+    # # p(resp.methods - Object.methods)
+    # resp.each_header { |k,v| p [k, "=" , v] }
+    assert resp.header["x-count"]
+    assert resp.header["x-min"]
+    assert resp.header["x-max"]
+    assert resp.header["x-mean"]
+    assert resp.header["x-std-dev"]
+    assert resp.header["x-outliers-low"]
+    assert resp.header["x-outliers-high"]
+    assert resp.body.size > 0
+  end
+
+  def get(path)
+    Net::HTTP.start(@host, @port) { |http| http.get path }
+  end
+
+  def teardown
+    Process.kill :QUIT, @srv
+    _, status = Process.waitpid2 @srv
+    assert status.success?
+    POSIX_MQ.unlink @queue
+  end
+end if defined?(Unicorn) && RUBY_PLATFORM =~ /linux/ && pmq

data/test/test_linux.rb

@@ -39,9 +39,29 @@ class TestLinux < Test::Unit::TestCase
     assert_equal 1, stats[tmp.path].queued
   end
 
+  def test_unix_all
+    tmp = Tempfile.new("\xde\xad\xbe\xef") # valid path, really :)
+    File.unlink(tmp.path)
+    us = UNIXServer.new(tmp.path)
+    uc0 = UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+
+    uc1 = UNIXSocket.new(tmp.path)
+    stats = unix_listener_stats
+    assert_equal 0, stats[tmp.path].active
+    assert_equal 2, stats[tmp.path].queued
+
+    ua0 = us.accept
+    stats = unix_listener_stats
+    assert_equal 1, stats[tmp.path].active
+    assert_equal 1, stats[tmp.path].queued
+  end
+
   def test_tcp
-    port = unused_port
-    s = TCPServer.new(TEST_ADDR, port)
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
     addr = "#{TEST_ADDR}:#{port}"
     addrs = [ addr ]
     stats = tcp_listener_stats(addrs)
@@ -62,10 +82,36 @@ class TestLinux < Test::Unit::TestCase
     assert_equal 1, stats[addr].active
   end
 
+  def test_tcp_reuse_sock
+    nlsock = Raindrops::InetDiagSocket.new
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
+    addr = "#{TEST_ADDR}:#{port}"
+    addrs = [ addr ]
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    c = TCPSocket.new(TEST_ADDR, port)
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 1, stats[addr].queued
+    assert_equal 0, stats[addr].active
+
+    sc = s.accept
+    stats = tcp_listener_stats(addrs, nlsock)
+    assert_equal 1, stats.size
+    assert_equal 0, stats[addr].queued
+    assert_equal 1, stats[addr].active
+    ensure
+      nlsock.close
+  end
+
   def test_tcp_multi
-    port1, port2 = unused_port, unused_port
-    s1 = TCPServer.new(TEST_ADDR, port1)
-    s2 = TCPServer.new(TEST_ADDR, port2)
+    s1 = TCPServer.new(TEST_ADDR, 0)
+    s2 = TCPServer.new(TEST_ADDR, 0)
+    port1, port2 = s1.addr[1], s2.addr[1]
     addr1, addr2 = "#{TEST_ADDR}:#{port1}", "#{TEST_ADDR}:#{port2}"
     addrs = [ addr1, addr2 ]
     stats = tcp_listener_stats(addrs)
@@ -127,10 +173,10 @@ class TestLinux < Test::Unit::TestCase
   def test_tcp_stress_test
     nr_proc = 32
     nr_sock = 500
-    port = unused_port
+    s = TCPServer.new(TEST_ADDR, 0)
+    port = s.addr[1]
     addr = "#{TEST_ADDR}:#{port}"
     addrs = [ addr ]
-    s = TCPServer.new(TEST_ADDR, port)
     rda, wra = IO.pipe
     rdb, wrb = IO.pipe
 
@@ -138,7 +184,7 @@ class TestLinux < Test::Unit::TestCase
       fork do
         rda.close
         wrb.close
-        socks = (1..nr_sock).times.map { s.accept }
+        socks = (1..nr_sock).map { s.accept }
         wra.syswrite('.')
         wra.close
         rdb.sysread(1) # wait for parent to nuke us
@@ -149,7 +195,7 @@ class TestLinux < Test::Unit::TestCase
       fork do
         rda.close
         wrb.close
-        socks = (1..nr_sock).times.map { TCPSocket.new(TEST_ADDR, port) }
+        socks = (1..nr_sock).map { TCPSocket.new(TEST_ADDR, port) }
         wra.syswrite('.')
         wra.close
         rdb.sysread(1) # wait for parent to nuke us
@@ -177,52 +223,4 @@ class TestLinux < Test::Unit::TestCase
     statuses = Process.waitall
     statuses.each { |(pid,status)| assert status.success?, status.inspect }
   end if ENV["STRESS"].to_i != 0
-
-private
-
-  # Stolen from Unicorn, also a version of this is used by the Rainbows!
-  # test suite.
-  # unused_port provides an unused port on +addr+ usable for TCP that is
-  # guaranteed to be unused across all compatible tests on that system.  It
-  # prevents race conditions by using a lock file other tests
-  # will see.  This is required if you perform several builds in parallel
-  # with a continuous integration system or run tests in parallel via
-  # gmake.  This is NOT guaranteed to be race-free if you run other
-  # systems that bind to random ports for testing (but the window
-  # for a race condition is very small).  You may also set UNICORN_TEST_ADDR
-  # to override the default test address (127.0.0.1).
-  def unused_port(addr = TEST_ADDR)
-    retries = 100
-    base = 5000
-    port = sock = nil
-    begin
-      begin
-        port = base + rand(32768 - base)
-        while port == 8080
-          port = base + rand(32768 - base)
-        end
-
-        sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
-        sock.bind(Socket.pack_sockaddr_in(port, addr))
-        sock.listen(5)
-      rescue Errno::EADDRINUSE, Errno::EACCES
-        sock.close rescue nil
-        retry if (retries -= 1) >= 0
-      end
-
-      # since we'll end up closing the random port we just got, there's a race
-      # condition could allow the random port we just chose to reselect itself
-      # when running tests in parallel with gmake.  Create a lock file while
-      # we have the port here to ensure that does not happen .
-      lock_path = "#{Dir::tmpdir}/unicorn_test.#{addr}:#{port}.lock"
-      lock = File.open(lock_path, File::WRONLY|File::CREAT|File::EXCL, 0600)
-      at_exit { File.unlink(lock_path) rescue nil }
-    rescue Errno::EEXIST
-      sock.close rescue nil
-      retry
-    end
-    sock.close rescue nil
-    port
-  end
-
 end if RUBY_PLATFORM =~ /linux/