regurgitator 0.0.0

data/t/t0002-domain_host.sh

@@ -0,0 +1,71 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 7 "DomainHost tests"
+
+t_begin "setup" && {
+	setup domain_host.ru
+}
+
+t_begin "retrieve empty file" && {
+	test 0 -eq $(curl -sSvf 2> $curl_err \
+	     -H Host:d.example.com \
+	     http://$listen/blah | wc -c)
+	grep 'E[tT]ag: ' $curl_err && die "ETag not expected"
+	grep 'Last-Modified: ' $curl_err && die "Last-Modified not expected"
+}
+
+t_begin "retrieve missing file" && {
+	sqlite3 $db <<EOF
+UPDATE file SET length = $random_blob_size;
+EOF
+	ok=$(curl -sSvf \
+	     -H Host:d.example.com \
+	     2> $curl_err http://$listen/blah || echo ok)
+	test ok = "$ok"
+	grep '\<404\>' $curl_err
+}
+
+t_begin "retrieve existing file" && {
+	dir=$TMPDIR/dev1/0/000/000
+	mkdir -p $dir
+	ln random_blob $dir/0000000001.fid
+	cksum="$(curl -sSvf 2> $curl_err \
+	         -H Host:d.example.com http://$listen/blah | cksum)"
+	test "$cksum" = "$random_blob_cksum"
+	dbgcat curl_err
+	grep 'E[tT]ag: "1"' $curl_err
+}
+
+t_begin "reproxy existing file" && {
+	curl -sSvf -H 'X-Reproxy-Path: /reproxy' 2> $curl_err \
+	  -H Host:d.example.com \
+	  http://$listen/blah >/dev/null
+	dbgcat curl_err
+	uri=http://$rack_file_listen/dev1/0/000/000/0000000001.fid
+	grep '^< X-Accel-Redirect: /reproxy' $curl_err
+	grep '^< X-Reproxy-Content-Type: application/octet-stream' $curl_err
+	grep "^< Location: $uri" $curl_err
+	grep '^< X-Reproxy-Last-Modified: ' $curl_err
+	grep '^< Content-Length: 0' $curl_err
+	grep '^< Etag: "1"' $curl_err
+}
+
+t_begin "using CNAME host name" && {
+	sqlite3 $db <<EOF
+UPDATE domain SET namespace = 'i-have-a-cname.example.org'
+WHERE namespace = 'd';
+EOF
+	cksum="$(curl -sSvf 2> $curl_err \
+	         -H Host:i-have-a-cname.example.org \
+	         http://$listen/blah | cksum)"
+	test "$cksum" = "$random_blob_cksum"
+	dbgcat curl_err
+	grep 'E[tT]ag: "1"' $curl_err
+}
+
+t_begin "kill servers" && {
+	kill -INT $rack_file_pid
+	kill -INT $rack_server_pid
+}
+
+t_done

data/t/t0003-one_domain.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 6 "OneDomain tests"
+
+t_begin "setup" && {
+	setup one_domain.ru
+}
+
+t_begin "retrieve empty file" && {
+	test 0 -eq $(curl -sSvf 2> $curl_err http://$listen/blah | wc -c)
+	grep 'E[tT]ag: ' $curl_err && die "ETag not expected"
+	grep 'Last-Modified: ' $curl_err && die "Last-Modified not expected"
+}
+
+t_begin "retrieve missing file" && {
+	sqlite3 $db <<EOF
+UPDATE file SET length = $random_blob_size;
+EOF
+	ok=$(curl -sSvf 2> $curl_err http://$listen/blah || echo ok)
+	test ok = "$ok"
+	grep '\<404\>' $curl_err
+}
+
+t_begin "retrieve existing file" && {
+	dir=$TMPDIR/dev1/0/000/000
+	mkdir -p $dir
+	ln random_blob $dir/0000000001.fid
+	cksum="$(curl -sSvf 2> $curl_err http://$listen/blah | cksum)"
+	test "$cksum" = "$random_blob_cksum"
+	dbgcat curl_err
+	grep 'E[tT]ag: "1"' $curl_err
+}
+
+t_begin "reproxy existing file" && {
+	curl -sSvf -H 'X-Reproxy-Path: /reproxy' 2> $curl_err \
+	  http://$listen/blah >/dev/null
+	dbgcat curl_err
+	uri=http://$rack_file_listen/dev1/0/000/000/0000000001.fid
+	grep '^< X-Accel-Redirect: /reproxy' $curl_err
+	grep '^< X-Reproxy-Content-Type: application/octet-stream' $curl_err
+	grep "^< Location: $uri" $curl_err
+	grep '^< X-Reproxy-Last-Modified: ' $curl_err
+	grep '^< Content-Length: 0' $curl_err
+	grep '^< Etag: "1"' $curl_err
+}
+
+t_begin "kill servers" && {
+	kill -INT $rack_file_pid
+	kill -INT $rack_server_pid
+}
+
+t_done

data/t/test-lib.sh

@@ -0,0 +1,119 @@
+#!/bin/sh
+. ./my-tap-lib.sh
+set -e
+set -u
+
+# sometimes we rely on http_proxy to avoid wasting bandwidth with Isolate
+# and multiple Ruby versions
+NO_PROXY=${UNICORN_TEST_ADDR-127.0.0.1}
+export NO_PROXY
+
+RUBY="${RUBY-ruby}"
+RUBY_VERSION=${RUBY_VERSION-$($RUBY -e 'puts RUBY_VERSION')}
+t_pfx=$PWD/trash/$T-$RUBY_VERSION
+LOCKDIR=${TMPDIR-/tmp}
+export LOCKDIR
+TMPDIR=$t_pfx
+rm -rf $TMPDIR
+mkdir -p $TMPDIR
+export TMPDIR
+
+PATH=$PWD/bin:$PATH
+export PATH
+
+test -x $PWD/bin/unused_listen || die "must be run in 't' directory"
+
+# given a list of variable names, create temporary files and assign
+# the pathnames to those variables
+rtmpfiles () {
+	for id in "$@"
+	do
+		name=$id
+		_tmp=$t_pfx/$id
+		eval "$id=$_tmp"
+
+		case $name in
+		*fifo)
+			rm -f $_tmp
+			mkfifo $_tmp
+			T_RM_LIST="$T_RM_LIST $_tmp"
+			;;
+		*socket)
+			rm -f $_tmp
+			T_RM_LIST="$T_RM_LIST $_tmp"
+			;;
+		*)
+			rm -f $_tmp
+			> $_tmp
+			T_OK_RM_LIST="$T_OK_RM_LIST $_tmp"
+			;;
+		esac
+	done
+}
+
+dbgcat () {
+	id=$1
+	eval '_file=$'$id
+	echo "==> $id <=="
+	sed -e "s/^/$id:/" < $_file
+}
+
+check_stderr () {
+	set +u
+	_r_err=${1-${r_err}}
+	set -u
+	if grep -i Error $_r_err
+	then
+		die "Errors found in $_r_err"
+	elif grep SIGKILL $_r_err
+	then
+		die "SIGKILL found in $_r_err"
+	fi
+}
+
+setup () {
+	rtmpfiles r_err r_out r_pid db f_err f_out f_pid curl_err
+
+	random_blob_size=$(wc -c < random_blob | awk '{print $1}')
+	random_blob_cksum="$(cksum < random_blob)"
+	eval $(unused_listen)
+	rack_file_listen=$listen
+	port=$(expr $listen : '[^:]\+:\([0-9]\+\)')
+	rack_file_port=$port
+
+	( rackup -s webrick -P$f_pid -p$port file.ru 2>> $f_err >> $f_out & ) &
+
+	(
+		cat fixtures.sql
+		echo "UPDATE host SET http_get_port = $rack_file_port;"
+	) | sqlite3 $db
+
+	eval $(unused_listen)
+	port=$(expr $listen : '[^:]\+:\([0-9]\+\)')
+	TEST_SEQUEL=sqlite://$db
+	(
+		RACK_ENV=deployment
+		export TEST_SEQUEL RACK_ENV
+		rackup -E none \
+		  -s webrick -P$r_pid -p $port "$@" 2>> $r_err >> $r_out &
+	) &
+
+	wait_for_start $r_err
+	wait_for_start $f_err
+	wait
+	rack_server_pid=$(cat $r_pid)
+	test -n "$rack_server_pid" && kill -0 $rack_server_pid
+	rack_file_pid=$(cat $f_pid)
+	test -n "$rack_file_pid" && kill -0 $rack_file_pid
+}
+
+wait_for_start () {
+	log_err=$1
+	(
+		renice 19 $$
+		while ! grep 'HTTPServer#start:' $log_err
+		do
+			:
+		done
+	) >/dev/null 2>&1 &
+}

data/test/test_local.rb

@@ -0,0 +1,78 @@
+require "test/unit"
+require "regurgitator"
+require "tempfile"
+require "fileutils"
+
+class TestLocal < Test::Unit::TestCase
+  include Regurgitator::Local
+
+  def teardown
+    STORE_PATHS.clear
+    FileUtils.rm_rf @tmpdir
+    Regurgitator::Local.refresh_addrs!
+  end
+
+  def setup
+    @local_addrs = Regurgitator::Local.instance_variable_get(:@local_addrs)
+    @local_addrs["127.0.0.1"] = true if @local_addrs.empty?
+    @addr = @local_addrs.keys.first
+    tmp = Tempfile.new('test_local')
+    @tmpdir = tmp.path
+    tmp.close!
+    Dir.mkdir(@tmpdir)
+  end
+
+  def test_device_path_stat
+    port = 1234
+    uri = URI("http://#@addr:#{port}/dev10/0/004/689/0004689631.fid")
+    assert_nil device_path_stat(uri)
+
+    assert_raises(ArgumentError) {
+      Regurgitator::Local.register port, @tmpdir
+    }
+    Dir.mkdir "#@tmpdir/dev10"
+    devs = Regurgitator::Local.register port, @tmpdir
+    assert_equal [ "dev10" ], devs.to_a
+
+    Regurgitator::Local.register port, @tmpdir
+    expect = "#@tmpdir/dev10/0/004/689/0004689631.fid"
+    FileUtils.mkdir_p(File.dirname(expect))
+
+    File.open(expect, "wb") { |fp| fp.write '.' }
+    assert_equal expect, device_path_stat(uri)[0]
+  end
+
+  def test_device_path_stat_ambiguous
+    Dir.mkdir(@a = "#@tmpdir/a")
+    Dir.mkdir(@b = "#@tmpdir/b")
+    port = 1234
+    uri = URI("http://#@addr:#{port}/dev10/0/004/689/0004689631.fid")
+    assert_nil device_path_stat(uri)
+
+    apath = "#@a/dev10/0/004/689/0004689631.fid"
+    bpath = "#@b/dev10/0/004/689/0004689631.fid"
+    FileUtils.mkdir_p([apath, bpath].map! { |x| File.dirname(x) })
+    Regurgitator::Local.register port, @a
+    Regurgitator::Local.register port, @b
+
+    [ apath, bpath ].each do |path|
+      File.open(path, "wb") { |fp| fp.write '.' }
+    end
+    assert_nil device_path_stat(uri)
+  end
+
+  def test_trylocal
+    port = 1233
+    env = { "REQUEST_METHOD" => "GET" }
+    a = URI("http://#@addr:#{port}/dev10/0/004/689/0004689631.fid")
+    b = URI("http://#@addr:#{port + 6}/dev11/0/004/689/0004689631.fid")
+    assert_nil trylocal(env, [ [ a, b ] ])
+
+    path_a = "#@tmpdir#{a.path}"
+    FileUtils.mkdir_p(File.dirname(path_a))
+    Regurgitator::Local.register port, @tmpdir
+    File.open(path_a, "wb") { |fp| fp.write '.' }
+    res = trylocal(env, [ [ a, b ] ])
+    assert_kind_of Regurgitator::LocalFile, res
+  end
+end

data/test/test_server_settings.rb

@@ -0,0 +1,37 @@
+require 'test/unit'
+require 'regurgitator'
+
+class TestServerSettings < Test::Unit::TestCase
+  include Regurgitator::ServerSettings
+
+  attr_accessor :db
+
+  def setup
+    @db = Sequel.connect 'sqlite:///'
+    @db.create_table(:server_settings) do
+      String :field, :size => 50, :null => false, :primary_key => true
+      String :value, :size => 255
+    end
+  end
+
+  def test_zones
+    ds = @db[:server_settings]
+    [ %w(network_zones west,east,central),
+      %w(zone_west 10.1.0.0/16),
+      %w(zone_central 10.2.0.0/16),
+      %w(zone_east 10.3.0.0/16)
+    ].each { |(field,value)| ds.insert(:field => field, :value => value) }
+    assert_equal 'west', zone_for('10.1.0.8')
+    assert_equal 'central', zone_for('10.2.1.8')
+    assert_equal 'east', zone_for('10.3.1.8')
+    assert_nil zone_for('127.0.0.1')
+  end
+
+  def test_no_zones_defined
+    ds = @db[:server_settings]
+    assert_nil zone_for('10.1.0.8')
+    assert_nil zone_for('10.2.1.8')
+    assert_nil zone_for('10.3.1.8')
+    assert_nil zone_for('127.0.0.1')
+  end
+end

metadata

@@ -0,0 +1,191 @@
+--- !ruby/object:Gem::Specification 
+name: regurgitator
+version: !ruby/object:Gem::Version 
+  hash: 31
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 0
+  version: 0.0.0
+platform: ruby
+authors: 
+- Regurgitators
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 1
+        - 2
+        version: "1.2"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 41
+        segments: 
+        - 3
+        - 23
+        version: "3.23"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: http_spew
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 1
+        version: "0.1"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: wrongdoc
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 5
+        version: "1.5"
+  type: :development
+  version_requirements: *id004
+description: |-
+  \Regurgitator is an GPL-licensed library and Rack middleware for
+  serving files stored in MogileFS.  It can be embedded inside
+  any existing Rack application or be used as a standalone Rack app.
+email: barfs@librelist.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- ChangeLog
+- lib/regurgitator.rb
+- lib/regurgitator/device.rb
+- lib/regurgitator/domain.rb
+- lib/regurgitator/domain_host.rb
+- lib/regurgitator/domain_path.rb
+- lib/regurgitator/endpoint.rb
+- lib/regurgitator/file_info.rb
+- lib/regurgitator/file_request.rb
+- lib/regurgitator/local.rb
+- lib/regurgitator/local_file.rb
+- lib/regurgitator/one_domain.rb
+- lib/regurgitator/server_settings.rb
+- NEWS
+- README
+- LICENSE
+files: 
+- .document
+- .gitignore
+- .manifest
+- .wrongdoc.yml
+- COPYING
+- ChangeLog
+- GIT-VERSION-FILE
+- GIT-VERSION-GEN
+- GNUmakefile
+- LATEST
+- LICENSE
+- NEWS
+- README
+- examples/domain_host.ru
+- examples/domain_path.ru
+- examples/one_domain.ru
+- lib/regurgitator.rb
+- lib/regurgitator/device.rb
+- lib/regurgitator/domain.rb
+- lib/regurgitator/domain_host.rb
+- lib/regurgitator/domain_path.rb
+- lib/regurgitator/endpoint.rb
+- lib/regurgitator/file_info.rb
+- lib/regurgitator/file_request.rb
+- lib/regurgitator/local.rb
+- lib/regurgitator/local_file.rb
+- lib/regurgitator/one_domain.rb
+- lib/regurgitator/server_settings.rb
+- pkg.mk
+- regurgitator.gemspec
+- setup.rb
+- t/.gitignore
+- t/GNUmakefile
+- t/bin/unused_listen
+- t/domain_host.ru
+- t/domain_path.ru
+- t/file.ru
+- t/fixtures.sql
+- t/my-tap-lib.sh
+- t/one_domain.ru
+- t/t0001-domain_path.sh
+- t/t0002-domain_host.sh
+- t/t0003-one_domain.sh
+- t/test-lib.sh
+- test/test_local.rb
+- test/test_server_settings.rb
+homepage: http://bogomips.org/regurgitator/
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- -t
+- \Regurgitator - read-only Rack endpoints for MogileFS
+- -W
+- http://bogomips.org/regurgitator.git/tree/%s
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rainbows
+rubygems_version: 1.8.1
+signing_key: 
+specification_version: 3
+summary: read-only Rack endpoints for MogileFS
+test_files: 
+- test/test_local.rb
+- test/test_server_settings.rb