omgf 0.0.0.GIT

data/test/integration.rb

@@ -0,0 +1,201 @@
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+$stdout.sync = $stderr.sync = true
+Thread.abort_on_exception = true
+if ENV["COVERAGE"]
+  require "coverage"
+  Coverage.start
+  at_exit do
+    # Dirty little text formatter.  I tried simplecov but the default
+    # HTML+JS is unusable without a GUI (I hate GUIs :P) and it would've
+    # taken me longer to search the Internets to find a plain-text
+    # formatter I like...
+    res = Coverage.result
+    relevant = res.keys.grep(%r{/lib/omgf/\w+\.rb})
+    relevant.each do |file|
+      cov = res[file]
+      puts "==> #{file} <=="
+      File.readlines(file).each_with_index do |line, i|
+        n = cov[i]
+        if n == 0 # BAD
+          print("  *** 0 #{line}")
+        elsif n
+          printf("% 7u %s", n, line)
+        elsif line =~ /\S/ # probably a line with just "end" in it
+          print("        #{line}")
+        else # blank line
+          print "\n" # don't output trailing whitespace on blank lines
+        end
+      end
+    end
+  end
+end
+require 'test/unit'
+require 'net/http'
+require 'uri'
+require 'tempfile'
+require 'mogilefs'
+require 'stringio'
+require 'logger'
+
+module TestMogileFSIntegration
+  def x(*cmd)
+    out = Tempfile.new("out")
+    err = Tempfile.new("err")
+    puts cmd.join(' ') if $VERBOSE
+    pid = fork do
+      $stderr.reopen(err.path, "a")
+      $stdout.reopen(out.path, "a")
+      out.close
+      err.close
+      ObjectSpace.each_object(Tempfile) do |tmp|
+        next if tmp.closed?
+        ObjectSpace.undefine_finalizer(tmp)
+        tmp.close_on_exec = true if tmp.respond_to?(:close_on_exec=)
+      end
+      exec(*cmd)
+    end
+    _, status = Process.waitpid2(pid)
+    out.rewind
+    err.rewind
+    [ status, out, err ]
+  end
+
+  def x!(*cmd)
+    status, out, err = x(*cmd)
+    assert status.success?, "#{status.inspect} / #{out.read} / #{err.read}"
+    [ status, out, err ]
+  end
+
+  def setup_mogilefs(plugins = nil)
+    @test_host = "127.0.0.1"
+    setup_mogstored
+    @tracker = TCPServer.new(@test_host, 0)
+    @tracker_port = @tracker.addr[1]
+
+    @dbname = Tempfile.new(["mogfresh", ".sqlite3"])
+    @mogilefsd_conf = Tempfile.new(["mogilefsd", "conf"])
+    @mogilefsd_pid = Tempfile.new(["mogilefsd", "pid"])
+
+    cmd = %w(mogdbsetup --yes --type=SQLite --dbname) << @dbname.path
+    x!(*cmd)
+
+    @mogilefsd_conf.puts "db_dsn DBI:SQLite:#{@dbname.path}"
+    @mogilefsd_conf.write <<EOF
+conf_port #@tracker_port
+listen #@test_host
+pidfile #{@mogilefsd_pid.path}
+replicate_jobs 1
+fsck_jobs 1
+query_jobs 1
+mogstored_stream_port #@mogstored_mgmt_port
+node_timeout 10
+EOF
+    @mogilefsd_conf.flush
+
+    @trackers = @hosts = [ "#@test_host:#@tracker_port" ]
+    @tracker.close
+    x!("mogilefsd", "--daemon", "--config=#{@mogilefsd_conf.path}")
+    wait_for_port @tracker_port
+    @admin = MogileFS::Admin.new(:hosts => @hosts)
+    500.times do
+      break if File.size(@mogstored_pid.path) > 0
+      sleep 0.05
+    end
+
+    args = { :ip => @test_host, :port => @mogstored_http_port }
+    args[:status] = "alive"
+    @admin.create_host("me", args)
+    yield_for_monitor_update { @admin.get_hosts.empty? or break }
+
+    mogadm!("device", "add", "me", "dev1")
+    yield_for_monitor_update { @admin.get_devices.empty? or break }
+    wait_for_usage_file "dev1"
+    mogadm!("device", "add", "me", "dev2")
+    wait_for_usage_file "dev2"
+    yield_for_monitor_update { @admin.get_devices.size == 2 and break }
+  end
+
+  def mogadm(*args)
+    x("mogadm", "--trackers=#{@trackers.join(',')}", *args)
+  end
+
+  def mogadm!(*args)
+    status, out, err = mogadm(*args)
+    assert status.success?, "#{status.inspect} / #{out.read} / #{err.read}"
+    [ status, out, err ]
+  end
+
+  def yield_for_monitor_update
+    50.times do
+      yield
+      sleep 0.1
+    end
+  end
+
+
+  def wait_for_port(port)
+    tries = 500
+    begin
+      TCPSocket.new(@test_host, port).close
+      return
+    rescue
+      sleep 0.05
+    end while (tries -= 1) > 0
+    raise "#@test_host:#{port} never became ready"
+  end
+
+  def teardown_mogilefs
+    if @mogstored_pid
+      pid = File.read(@mogstored_pid.path).to_i
+      Process.kill(:TERM, pid) if pid > 0
+    end
+    if @mogilefsd_pid
+      s = TCPSocket.new(@test_host, @tracker_port)
+      s.write "!shutdown\r\n"
+      s.close
+    end
+    FileUtils.rmtree(@docroot)
+  end
+
+  def wait_for_usage_file(device)
+    uri = URI("http://#@test_host:#@mogstored_http_port/#{device}/usage")
+    res = nil
+    100.times do
+      res = Net::HTTP.get_response(uri)
+      if Net::HTTPOK === res
+        puts res.body if $DEBUG
+        return
+      end
+      puts res.inspect if $DEBUG
+      sleep 0.1
+    end
+    raise "#{uri} failed to appear: #{res.inspect}"
+  end
+
+  def setup_mogstored
+    @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    Dir.mkdir("#@docroot/dev1")
+    Dir.mkdir("#@docroot/dev2")
+    @mogstored_mgmt = TCPServer.new(@test_host, 0)
+    @mogstored_http = TCPServer.new(@test_host, 0)
+    @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
+    @mogstored_http_port = @mogstored_http.addr[1]
+    @mogstored_conf = Tempfile.new(["mogstored", "conf"])
+    @mogstored_pid = Tempfile.new(["mogstored", "pid"])
+    @mogstored_conf.write <<EOF
+pidfile = #{@mogstored_pid.path}
+maxconns = 500
+httplisten = #@test_host:#@mogstored_http_port
+mgmtlisten = #@test_host:#@mogstored_mgmt_port
+docroot = #@docroot
+EOF
+    @mogstored_conf.flush
+    @mogstored_mgmt.close
+    @mogstored_http.close
+
+    x!("mogstored", "--daemon", "--config=#{@mogstored_conf.path}")
+    wait_for_port @mogstored_mgmt_port
+    wait_for_port @mogstored_http_port
+  end
+end

data/test/test_hyst.rb

@@ -0,0 +1,78 @@
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+require 'omgf/hysterical_raisins'
+begin
+  require 'unicorn'
+rescue LoadError
+end
+
+class TestHystScript < Test::Unit::TestCase
+  include TestMogileFSIntegration
+  def setup
+    setup_mogilefs
+    @admin.create_domain("testdom")
+    srv = TCPServer.new(@test_host, 0)
+    @api_port = srv.addr[1]
+    ENV["UNICORN_FD"] = srv.fileno.to_s
+    api_addr = "#@test_host:#@api_port"
+    @app = OMGF::HystericalRaisins.new(:hosts => @hosts)
+    @out = Tempfile.new("out")
+    @err = Tempfile.new("err")
+    @unicorn_pid = fork do
+      $stdin.reopen("/dev/null")
+      $stdout.reopen(@out.path, "ab")
+      $stderr.reopen(@err.path, "ab")
+      # hopefully this Unicorn API doesn't change...
+      Unicorn::HttpServer.new(@app, :listeners => [api_addr]).start.join
+    end
+    srv.close
+    ENV["HYST_HOST"] = api_addr
+    ENV["MOG_DOMAIN"] = 'testdom'
+    @hyst = File.dirname(File.dirname(__FILE__)) + "/examples/hyst.bash"
+    assert File.executable?(@hyst)
+  end
+
+  def test_hyst
+    out = `#@hyst ls`
+    assert_equal $?, 0
+    assert_equal '', out
+
+    # tee is a little tricky, unicorn is one of the few Rack servers that
+    # handle chunked PUTs
+    out = `echo HI | #@hyst tee foo`
+    assert_equal 0, $?
+    assert_equal "HI\n", out
+    assert_equal "HI\n", `#@hyst cat foo`
+
+    # /dev/null optimization
+    out = `echo NULL | #@hyst tee bar >/dev/null`
+    assert_equal 0, $?
+    assert_equal "", out
+    assert_equal "NULL\n", `#@hyst cat bar`
+
+    # listings
+    assert_equal "foo\n", `#@hyst ls fo`
+    assert_equal "bar\n", `#@hyst ls b`
+    assert_equal "", `#@hyst ls z`
+
+    # remove a file once
+    assert(system("#@hyst rm bar"))
+
+    # really removed?
+    err = Tempfile.new('curl_err')
+    a = `#@hyst cat bar 2>#{err.path}`
+    assert_equal 22, $?.exitstatus
+    assert_equal '', a
+    assert_match(/\s+404\b/, err.read)
+    err.close!
+  end
+
+  def teardown
+    ENV.delete("HYST_HOST")
+    Process.kill :QUIT, @unicorn_pid
+    _, status = Process.waitpid2(@unicorn_pid)
+    assert status.success?, status.inspect
+    teardown_mogilefs
+  end
+end if defined?(Unicorn)

data/test/test_hysterical_raisins.rb

@@ -0,0 +1,238 @@
+# Copyright (C) 2008-2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+require 'rack/mock'
+require 'open-uri'
+require 'omgf/hysterical_raisins'
+
+class TestHystericalRaisins < Test::Unit::TestCase
+  include TestMogileFSIntegration
+  def setup
+    setup_mogilefs
+    @app = OMGF::HystericalRaisins.new(:hosts => @hosts)
+    @req = Rack::MockRequest.new(@app)
+    @err = StringIO.new
+    @opts = { "rack.logger" => Logger.new(@err) }
+    @admin.create_domain("testdom")
+  end
+
+  def test_all
+    resp = @req.get("/")
+    assert_equal 200, resp.status, "/ response code is 200"
+    assert_equal "", resp.body, "/ returns empty body"
+
+    # domain listing
+    resp = @req.get("/testdom", @opts)
+    assert_equal 200, resp.status, "domain listing response code is 200"
+    assert_equal "", resp.body, "domain listing is empty"
+    assert_equal("text/plain", resp.headers["Content-Type"],
+                 "content-type is text/plain")
+
+    json = { "HTTP_ACCEPT" => "application/json" }
+
+    # json domain listing
+    resp = @req.get("/testdom", @opts.merge(json))
+    assert_equal 200, resp.status, "domain listing response code is 200"
+    assert_equal "[]", resp.body, "domain listing is empty array"
+    assert_equal("application/json", resp.headers["Content-Type"],
+                 "content-type is application/json")
+
+    # missing domain
+    resp = @req.get("/non-existent", @opts)
+    assert_equal 0, @err.string.size
+    assert_equal 404, resp.status, "non-existent domain listing gives 404"
+
+    # missing domain (json)
+    resp = @req.get("/non-existent", @opts.merge(json))
+    assert_equal 0, @err.string.size
+    assert_equal 404, resp.status, "non-existent domain listing gives 404"
+
+    # PUT to domain listing fails
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    resp = @req.put("/non-existent", opts)
+    assert_equal 0, @err.string.size
+    assert_equal 404, resp.status, "non-existent domain listing PUT gives 404"
+
+    # DELETEs to bad domains fail
+    resp = @req.delete("/non-existent", opts)
+    assert_equal 0, @err.string.size
+    assert_equal 404, resp.status, "non-existent domain DELETE gives 404"
+    resp = @req.delete("/non-existent/key", opts)
+    assert_equal 0, @err.string.size
+    assert_equal 406, resp.status, "non-existent domain DELETE gives 406"
+
+    # PUT to bad domain fails
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    resp = @req.put("/non-existent/key", opts)
+    assert_equal 0, @err.string.size
+    assert_equal 406, resp.status, "bad domain PUT w/key: #{resp.inspect}"
+    assert_equal "Invalid domain: non-existent\n", resp.body
+
+    # PUT with bad key fails
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    resp = @req.put("/testdom/key%20space", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 406, resp.status, "bad key"
+
+    # PUT with long key fails
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    resp = @req.put("/testdom/key#{'a' * 128}", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 406, resp.status, "bad key"
+
+    # PUT to good domain succeeds
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    resp = @req.put("/testdom/key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 200, resp.status
+
+    # GET succeeds with redirect
+    resp = @req.get("/testdom/key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 302, resp.status
+    assert_equal "HELLO", open(resp["Location"]).read
+
+    # PUT to good domain and REMOTE_USER succeeds succeeds with 201
+    opts = @opts.merge(:input => StringIO.new("HELLO"))
+    opts["REMOTE_USER"] = "root"
+    resp = @req.put("/testdom/user_key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 201, resp.status
+
+    # plain-text key listing succeeds
+    resp = @req.get("/testdom", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 200, resp.status, resp.inspect
+    assert_equal 2, resp.body.split(/\n/).size
+    assert_equal("text/plain", resp.headers["Content-Type"],
+                 "content-type is text/plain")
+    keys = resp.body.split(/\n/)
+    assert_match(/\Akey\|5|[12]\z/ ,keys[0])
+    assert_match(/\Auser_key\|5|[12]\z/ ,keys[1])
+
+    # json key listing succeeds
+    resp = @req.get("/testdom", @opts.merge(json))
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 200, resp.status, resp.inspect
+    keys = JSON.parse(resp.body)
+    assert_equal 2, keys.size
+    assert_equal("application/json", resp.headers["Content-Type"],
+                 "content-type is application/json")
+    assert_equal "key", keys[0][0]
+    assert_equal 5, keys[0][1]
+    assert_operator keys[0][2], :>=, 1
+    assert_operator keys[0][2], :<=, 2
+    assert_equal "user_key", keys[1][0]
+    assert_equal 5, keys[1][1]
+    assert_operator keys[1][2], :>=, 1
+    assert_operator keys[1][2], :<=, 2
+
+    # HEAD of json listing succeeds
+    resp = @req.head("/testdom", @opts.merge(json))
+    assert_equal("application/json", resp.headers["Content-Type"],
+                 "content-type is application/json")
+    assert_equal 200, resp.status
+    assert_equal "", resp.body
+
+    # HEAD of text listing succeeds
+    resp = @req.head("/testdom", @opts)
+    assert_equal("text/plain", resp.headers["Content-Type"],
+                 "content-type is text/plain")
+    assert_equal 200, resp.status
+    assert_equal "", resp.body
+
+    # DELETE succeeds
+    resp = @req.delete("/testdom/user_key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 204, resp.status, resp.inspect
+
+    # DELETE again fails
+    resp = @req.delete("/testdom/user_key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 404, resp.status, resp.inspect
+
+    # unauthed PUT fails to overwrite
+    opts = @opts.merge(:input => StringIO.new("FAIL"))
+    resp = @req.put("/testdom/key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 403, resp.status, resp.inspect
+
+    # GET returns unchanged file
+    resp = @req.get("/testdom/key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 302, resp.status
+    assert_equal "HELLO", open(resp["Location"]).read
+
+    # PUT succeeds with overwrite
+    opts = @opts.merge(:input => StringIO.new("BLAH"))
+    opts["REMOTE_USER"] = "root"
+    opts["HTTP_X_OMGF_FORCE"] = "true"
+    resp = @req.put("/testdom/key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 204, resp.status, resp.inspect
+
+    # GET succeeds with redirect
+    resp = @req.get("/testdom/key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 302, resp.status
+    assert_equal "BLAH", open(resp["Location"]).read
+
+    # HEAD shows size and metadata
+    resp = @req.head("/testdom/key", @opts)
+    assert_equal 0, @err.string.size, @err.string
+    assert_equal 200, resp.status, resp.inspect
+    assert_equal "4", resp["Content-Length"]
+    assert_equal "BLAH", open(resp["X-URL-0"]).read
+
+    # wait for replication
+    if ENV["EXPENSIVE"]
+      200.times do
+        resp["X-URL-1"] and break
+        sleep 0.1
+        resp = @req.head("/testdom/key", @opts)
+      end
+      assert_kind_of String, resp["X-URL-1"]
+      assert_equal "BLAH", open(resp["X-URL-1"]).read
+    end
+
+    reproxy_test
+  end
+
+  def reproxy_test
+    @app.instance_variable_set(:@reproxy_path, "/reproxy")
+    opts = @opts.merge("HTTP_X_OMGF_REPROXY" => "1")
+    resp = @req.get("/testdom/key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    t = Time.parse(resp["X-Redirect-Last-Modified"])
+    assert_equal "\"#{t.to_i}\"", resp["ETag"]
+    assert_equal "application/octet-stream", resp["X-Redirect-Content-Type"]
+    assert_equal "BLAH", open(resp["Location"]).read
+    assert_equal "/reproxy", resp["X-Accel-Redirect"]
+    assert_nil resp.original_headers["Content-Type"], resp.inspect
+    assert_nil resp["Last-Modified"]
+
+    # no point in redirecting HEAD requests
+    resp = @req.head("/testdom/key", opts)
+    assert_equal 0, @err.string.size, @err.string
+    t = Time.parse(resp["Last-Modified"])
+    assert_equal "\"#{t.to_i}\"", resp["ETag"]
+    assert_nil resp["X-Accel-Redirect"]
+    assert_equal "4", resp["Content-Length"]
+    assert_equal "application/octet-stream",
+                  resp.original_headers["Content-Type"], resp.inspect
+
+    # explicit filename
+    resp = @req.head("/testdom/key?inline=foo.txt", opts)
+    assert_equal "inline; filename=foo.txt", resp["Content-Disposition"]
+    resp = @req.head("/testdom/key?attachment=foo.txt", opts)
+    assert_equal "attachment; filename=foo.txt", resp["Content-Disposition"]
+    resp = @req.head("/testdom/key?filename=foo.txt", opts)
+    assert_equal "attachment; filename=foo.txt", resp["Content-Disposition"]
+
+    @app.instance_variable_set(:@reproxy_path, nil)
+  end
+
+  def teardown
+    teardown_mogilefs
+  end
+end