regurgitator 0.0.0

data/t/.gitignore

@@ -0,0 +1,7 @@
+*.sqlite3
+/test-results-*
+/bin-*
+/random_blob
+/.dep+*
+/trash
+/tmp

data/t/GNUmakefile

@@ -0,0 +1,65 @@
+# we can run tests in parallel with GNU make
+
+all::
+
+pid := $(shell echo $$PPID)
+
+RUBY = ruby
+my_lib := $(shell cd ../lib && pwd)
+-include ../local.mk
+ifeq ($(RUBY_VERSION),)
+  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+endif
+ifeq ($(RUBY_VERSION),)
+  $(error unable to detect RUBY_VERSION)
+endif
+
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+export RUBY_VERSION RUBY_ENGINE
+ifeq ($(RUBYLIB),)
+  RUBYLIB := $(my_lib)
+else
+  RUBYLIB := $(my_lib):$(RUBYLIB)
+endif
+export RUBYLIB
+
+T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
+
+all:: $(T)
+
+# can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
+t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION)
+TEST_OPTS =
+# TRACER = strace -f -o $(t_pfx).strace -s 100000
+# TRACER = /usr/bin/time -o $(t_pfx).time
+
+ifdef V
+  ifeq ($(V),2)
+    TEST_OPTS += --trace
+  else
+    TEST_OPTS += --verbose
+  endif
+endif
+
+random_blob:
+	dd if=/dev/urandom bs=1M count=1 of=$@.$(pid)
+	mv $@.$(pid) $@
+
+$(T): random_blob
+
+dependencies := curl sqlite3
+deps := $(addprefix .dep+,$(dependencies))
+$(deps): dep_bin = $(lastword $(subst +, ,$@))
+$(deps):
+	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
+	@test -s $@.$(pid) || \
+	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
+	@mv $@.$(pid) $@
+
+$(T): $(deps)
+	  $(TRACER) $(SHELL) $(SH_TEST_OPTS) $@ $(TEST_OPTS)
+
+clean:
+	$(RM) -r trash/*.log trash/*.code test-bin-$(RUBY_VERSION)
+
+.PHONY: $(T) clean

data/t/bin/unused_listen

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+# -*- encoding: binary -*-
+# this is to remain compatible with the unused_port function in the
+# Unicorn test/test_helper.rb file
+require 'socket'
+require 'tmpdir'
+
+default_port = 8080
+addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+retries = 100
+base = 5000
+port = sock = lock_path = nil
+
+begin
+  begin
+    port = base + rand(32768 - base)
+    while port == default_port
+      port = base + rand(32768 - base)
+    end
+
+    sock = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+    sock.bind(Socket.pack_sockaddr_in(port, addr))
+    sock.listen(5)
+  rescue Errno::EADDRINUSE, Errno::EACCES
+    sock.close rescue nil
+    retry if (retries -= 1) >= 0
+  end
+
+  # since we'll end up closing the random port we just got, there's a race
+  # condition could allow the random port we just chose to reselect itself
+  # when running tests in parallel with gmake.  Create a lock file while
+  # we have the port here to ensure that does not happen.
+  lock_path = "#{Dir::tmpdir}/unicorn_test.#{addr}:#{port}.lock"
+  lock = File.open(lock_path, File::WRONLY|File::CREAT|File::EXCL, 0600)
+rescue Errno::EEXIST
+  sock.close rescue nil
+  retry
+end
+sock.close rescue nil
+puts %Q(listen=#{addr}:#{port} T_RM_LIST="$T_RM_LIST #{lock_path}")

data/t/domain_host.ru

@@ -0,0 +1,8 @@
+require 'regurgitator'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+o = {
+  :db => Sequel.connect(ENV['TEST_SEQUEL']),
+  :suffix => '.example.com',
+}
+run Regurgitator::DomainHost.new(lambda { |env| [ 404, [], [] ] }, o)

data/t/domain_path.ru

@@ -0,0 +1,5 @@
+require 'regurgitator'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+db = Sequel.connect(ENV['TEST_SEQUEL'])
+run Regurgitator::DomainPath.new(lambda { |env| [ 404, [], [] ] }, db)

data/t/file.ru

@@ -0,0 +1,2 @@
+use Rack::Head
+run Rack::File.new(ENV['TMPDIR'])

data/t/fixtures.sql

@@ -0,0 +1,59 @@
+CREATE TABLE domain (
+	dmid SMALLINT NOT NULL,
+	namespace VARCHAR(255),
+	PRIMARY KEY (dmid),
+	UNIQUE (namespace)
+);
+
+CREATE TABLE file (
+	fid INTEGER NOT NULL,
+	dmid SMALLINT NOT NULL,
+	dkey VARCHAR(255),
+	length INTEGER,
+	classid TINYINT NOT NULL,
+	devcount TINYINT NOT NULL,
+	UNIQUE (dmid, dkey),
+	PRIMARY KEY (fid)
+);
+
+CREATE TABLE host (
+	hostid MEDIUMINT NOT NULL PRIMARY KEY,
+	status VARCHAR(5),
+	http_port MEDIUMINT DEFAULT 7500,
+	http_get_port MEDIUMINT,
+	hostname VARCHAR(40),
+	hostip VARCHAR(15),
+	altip VARCHAR(15),
+	altmask VARCHAR(18),
+	UNIQUE (hostname),
+	UNIQUE (hostip),
+	UNIQUE (altip)
+);
+
+CREATE TABLE server_settings (
+	field VARCHAR(50) PRIMARY KEY,
+	value VARCHAR(255)
+);
+
+CREATE TABLE device (
+	devid MEDIUMINT NOT NULL,
+	hostid MEDIUMINT NOT NULL,
+	status VARCHAR(8),
+	weight MEDIUMINT DEFAULT 100,
+	mb_total MEDIUMINT,
+	mb_used MEDIUMINT,
+	mb_asof INTEGER,
+	PRIMARY KEY (devid)
+);
+
+CREATE TABLE file_on (
+	fid INTEGER NOT NULL,
+	devid MEDIUMINT NOT NULL,
+	PRIMARY KEY (fid, devid)
+);
+
+INSERT INTO host VALUES(1,'alive',7500,NULL,'localhost','127.0.0.1',NULL,NULL);
+INSERT INTO device VALUES(1,1,'alive',100,99999,0,1234567890);
+INSERT INTO domain VALUES(1,'d');
+INSERT INTO file VALUES(1,1,'blah',0,0,1);
+INSERT INTO file_on VALUES(1,1);

data/t/my-tap-lib.sh

@@ -0,0 +1,200 @@
+#!/bin/sh
+# Copyright (c) 2009 Eric Wong <normalperson@yhbt.net>
+#
+# TAP-producing shell library for POSIX-compliant Bourne shells We do
+# not _rely_ on Bourne Again features, though we will use "set -o
+# pipefail" from ksh93 or bash 3 if available
+#
+# Only generic, non-project/non-language-specific stuff goes here.  We
+# only have POSIX dependencies for the core tests (without --verbose),
+# though we'll enable useful non-POSIX things if they're available.
+#
+# This test library is intentionally unforgiving, it does not support
+# skipping tests nor continuing after any failure.  Any failures
+# immediately halt execution as do any references to undefined
+# variables.
+#
+# When --verbose is specified, we always prefix stdout/stderr
+# output with "#" to avoid confusing TAP consumers.  Otherwise
+# the normal stdout/stderr streams are redirected to /dev/null
+
+# dup normal stdout(fd=1) and stderr (fd=2) to fd=3 and fd=4 respectively
+# normal TAP output goes to fd=3, nothing should go to fd=4
+exec 3>&1 4>&2
+
+# ensure a sane environment
+TZ=UTC LC_ALL=C LANG=C
+export LANG LC_ALL TZ
+unset CDPATH
+
+# pipefail is non-POSIX, but very useful in ksh93/bash
+( set -o pipefail 2>/dev/null ) && set -o pipefail
+
+SED=${SED-sed}
+
+# Unlike other test frameworks, we are unforgiving and bail immediately
+# on any failures.  We do this because we're lazy about error handling
+# and also because we believe anything broken should not be allowed to
+# propagate throughout the rest of the test
+set -e
+set -u
+
+# name of our test
+T=${0##*/}
+
+t_expect_nr=-1
+t_nr=0
+t_current=
+t_complete=false
+
+# list of files to remove unconditionally on exit
+T_RM_LIST=
+
+# list of files to remove only on successful exit
+T_OK_RM_LIST=
+
+# emit output to stdout, it'll be parsed by the TAP consumer
+# so it must be TAP-compliant output
+t_echo () {
+	echo >&3 "$@"
+}
+
+# emits non-parsed information to stdout, it will be prefixed with a '#'
+# to not throw off TAP consumers
+t_info () {
+	t_echo '#' "$@"
+}
+
+# exit with an error and print a diagnostic
+die () {
+	echo >&2 "$@"
+	exit 1
+}
+
+# our at_exit handler, it'll fire for all exits except SIGKILL (unavoidable)
+t_at_exit () {
+	code=$?
+	set +e
+	if test $code -eq 0
+	then
+		$t_complete || {
+			t_info "t_done not called"
+			code=1
+		}
+	elif test -n "$t_current"
+	then
+		t_echo "not ok $t_nr - $t_current"
+	fi
+	if test $t_expect_nr -ne -1
+	then
+		test $t_expect_nr -eq $t_nr || {
+			t_info "planned $t_expect_nr tests but ran $t_nr"
+			test $code -ne 0 || code=1
+		}
+	fi
+	$t_complete || {
+		t_info "unexpected test failure"
+		test $code -ne 0 || code=1
+	}
+	rm -f $T_RM_LIST
+	test $code -eq 0 && rm -f $T_OK_RM_LIST
+	set +x
+	exec >&3 2>&4
+	t_close_fds
+	exit $code
+}
+
+# close test-specific extra file descriptors
+t_close_fds () {
+	exec 3>&- 4>&-
+}
+
+# call this at the start of your test to specify the number of tests
+# you plan to run
+t_plan () {
+	test "$1" -ge 1 || die "must plan at least one test"
+	test $t_expect_nr -eq -1 || die "tried to plan twice in one test"
+	t_expect_nr=$1
+	shift
+	t_echo 1..$t_expect_nr "#" "$@"
+	trap t_at_exit EXIT
+}
+
+_t_checkup () {
+	test $t_expect_nr -le 0 && die "no tests planned"
+	test -n "$t_current" && t_echo "ok $t_nr - $t_current"
+	true
+}
+
+# finalizes any previously test and starts a new one
+t_begin () {
+	_t_checkup
+	t_nr=$(( $t_nr + 1 ))
+	t_current="$1"
+
+	# just in case somebody wanted to cheat us:
+	set -e
+}
+
+# finalizes the current test without starting a new one
+t_end () {
+	_t_checkup
+	t_current=
+}
+
+# run this to signify the end of your test
+t_done () {
+	_t_checkup
+	t_current=
+	t_complete=true
+	test $t_expect_nr -eq $t_nr || exit 1
+	exit 0
+}
+
+# create and assign named-pipes to variable _names_ passed to this function
+t_fifos () {
+	for _id in "$@"
+	do
+		_name=$_id
+		_tmp=$(mktemp -t $T.$$.$_id.XXXXXXXX)
+		eval "$_id=$_tmp"
+		rm -f $_tmp
+		mkfifo $_tmp
+		T_RM_LIST="$T_RM_LIST $_tmp"
+	done
+}
+
+t_verbose=false t_trace=false
+
+while test "$#" -ne 0
+do
+	arg="$1"
+	shift
+	case $arg in
+	-v|--verbose) t_verbose=true ;;
+	--trace) t_trace=true t_verbose=true ;;
+	*) die "Unknown option: $arg" ;;
+	esac
+done
+
+# we always only setup stdout, nothing should end up in the "real" stderr
+if $t_verbose
+then
+	if test x"$(which mktemp 2>/dev/null)" = x
+	then
+		die "mktemp(1) not available for --verbose"
+	fi
+	t_fifos t_stdout t_stderr
+
+	(
+		# use a subshell so seds are not waitable
+		$SED -e 's/^/#: /' $t_stdout &
+		$SED -e 's/^/#! /' $t_stderr &
+	) &
+	exec > $t_stdout 2> $t_stderr
+else
+	exec > /dev/null 2> /dev/null
+fi
+
+$t_trace && set -x
+true

data/t/one_domain.ru

@@ -0,0 +1,8 @@
+require 'regurgitator'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+o = {
+  :db => Sequel.connect(ENV['TEST_SEQUEL']),
+  :domain => 'd',
+}
+run Regurgitator::OneDomain.new(lambda { |env| [ 404, [], [] ] }, o)

data/t/t0001-domain_path.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 8 "DomainPath tests"
+
+t_begin "setup" && {
+	setup domain_path.ru
+}
+
+t_begin "retrieve empty file" && {
+	test 0 -eq $(curl -sSvf 2> $curl_err http://$listen/d/blah | wc -c)
+	grep 'E[tT]ag: ' $curl_err && die "ETag not expected"
+	grep 'Last-Modified: ' $curl_err && die "Last-Modified not expected"
+}
+
+t_begin "retrieve missing file" && {
+	sqlite3 $db <<EOF
+UPDATE file SET length = $random_blob_size;
+EOF
+	ok=$(curl -sSvf 2> $curl_err http://$listen/d/blah || echo ok)
+	test ok = "$ok"
+	grep '\<404\>' $curl_err
+}
+
+t_begin "retrieve existing file" && {
+	dir=$TMPDIR/dev1/0/000/000
+	mkdir -p $dir
+	ln random_blob $dir/0000000001.fid
+	cksum="$(curl -sSvf 2> $curl_err http://$listen/d/blah | cksum)"
+	test "$cksum" = "$random_blob_cksum"
+	dbgcat curl_err
+	grep 'E[tT]ag: "1"' $curl_err
+}
+
+t_begin "reproxy existing file" && {
+	curl -sSvf -H 'X-Reproxy-Path: /reproxy' 2> $curl_err \
+	  http://$listen/d/blah >/dev/null
+	dbgcat curl_err
+	uri=http://$rack_file_listen/dev1/0/000/000/0000000001.fid
+	grep '^< X-Accel-Redirect: /reproxy' $curl_err
+	grep '^< X-Reproxy-Content-Type: application/octet-stream' $curl_err
+	grep "^< Location: $uri" $curl_err
+	grep '^< X-Reproxy-Last-Modified: ' $curl_err
+	grep '^< Content-Length: 0' $curl_err
+	grep '^< Etag: "1"' $curl_err
+}
+
+t_begin "HEAD requests work" && {
+	curl -vsSfI > $curl_err http://$listen/d/blah
+	grep '^E[Tt]ag: "1"' $curl_err
+	grep "^Content-Length: $random_blob_size" $curl_err
+}
+
+t_begin "HEAD requests work with reproxy" && {
+	curl -vsSfI -H X-Reproxy-Path:/z > $curl_err http://$listen/d/blah
+	grep '^E[Tt]ag: "1"' $curl_err
+	grep "^Content-Length: $random_blob_size" $curl_err
+	grep '^X-Accel-Redirect:' $curl_err && die "unexpected reproxy header"
+	grep '^X-Reproxy' $curl_err && die "unexpected reproxy header"
+	grep '^Location:' $curl_err && die "unexpected reproxy header"
+}
+
+t_begin "kill servers" && {
+	kill -INT $rack_file_pid
+	kill -INT $rack_server_pid
+}
+
+t_done