omgdav 0.0.0

data/test/integration.rb

@@ -0,0 +1,232 @@
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+$stdout.sync = $stderr.sync = true
+Thread.abort_on_exception = true
+if ENV["COVERAGE"]
+  require "coverage"
+  Coverage.start
+  at_exit do
+    # Dirty little text formatter.  I tried simplecov but the default
+    # HTML+JS is unusable without a GUI (I hate GUIs :P) and it would've
+    # taken me longer to search the Internets to find a plain-text
+    # formatter I like...
+    res = Coverage.result
+    relevant = res.keys.grep(%r{/lib/omgdav/\w+\.rb})
+    relevant.each do |file|
+      cov = res[file]
+      puts "==> #{file} <=="
+      File.readlines(file).each_with_index do |line, i|
+        n = cov[i]
+        if n == 0 # BAD
+          print("  *** 0 #{line}")
+        elsif n
+          printf("% 7u %s", n, line)
+        elsif line =~ /\S/ # probably a line with just "end" in it
+          print("        #{line}")
+        else # blank line
+          print "\n" # don't output trailing whitespace on blank lines
+        end
+      end
+    end
+  end
+end
+
+require 'pp'
+require 'minitest/autorun'
+require 'net/http'
+require 'uri'
+require 'tempfile'
+require 'stringio'
+require 'logger'
+require 'mogilefs'
+require 'open-uri'
+require 'rack'
+require 'rack/mock'
+require 'omgdav/app'
+require 'nokogiri'
+require 'time'
+require 'json'
+Sequel.extension(:migration)
+
+module TestMogileFSIntegration
+  def x(*cmd)
+    out = Tempfile.new("out")
+    err = Tempfile.new("err")
+    puts cmd.join(' ') if $VERBOSE
+    pid = fork do
+      $stderr.reopen(err.path, "a")
+      $stdout.reopen(out.path, "a")
+      out.close
+      err.close
+      ObjectSpace.each_object(Tempfile) do |tmp|
+        next if tmp.closed?
+        ObjectSpace.undefine_finalizer(tmp)
+        tmp.close_on_exec = true if tmp.respond_to?(:close_on_exec=)
+      end
+      exec(*cmd)
+    end
+    _, status = Process.waitpid2(pid)
+    out.rewind
+    err.rewind
+    [ status, out, err ]
+  end
+
+  def x!(*cmd)
+    status, out, err = x(*cmd)
+    assert status.success?, "#{status.inspect} / #{out.read} / #{err.read}"
+    [ status, out, err ]
+  end
+
+  def setup_mogilefs(plugins = nil, mogstored = "mogstored")
+    @test_host = "127.0.0.1"
+    setup_mogstored(mogstored)
+    @tracker = TCPServer.new(@test_host, 0)
+    @tracker_port = @tracker.addr[1]
+
+    @dbname = Tempfile.new(["mogfresh", ".sqlite3"])
+    @mogilefsd_conf = Tempfile.new(["mogilefsd", "conf"])
+    @mogilefsd_pid = Tempfile.new(["mogilefsd", "pid"])
+
+    cmd = %w(mogdbsetup --yes --type=SQLite --dbname) << @dbname.path
+    x!(*cmd)
+
+    @mogilefsd_conf.puts "db_dsn DBI:SQLite:#{@dbname.path}"
+    @mogilefsd_conf.write <<EOF
+conf_port #@tracker_port
+listen #@test_host
+pidfile #{@mogilefsd_pid.path}
+replicate_jobs 1
+fsck_jobs 1
+query_jobs 1
+mogstored_stream_port #@mogstored_mgmt_port
+node_timeout 10
+EOF
+    @mogilefsd_conf.flush
+
+    @trackers = @hosts = [ "#@test_host:#@tracker_port" ]
+    @tracker.close
+    x!("mogilefsd", "--daemon", "--config=#{@mogilefsd_conf.path}")
+    wait_for_port @tracker_port
+    @admin = MogileFS::Admin.new(:hosts => @hosts)
+    500.times do
+      break if File.size(@mogstored_pid.path) > 0
+      sleep 0.05
+    end
+
+    args = { :ip => @test_host, :port => @mogstored_http_port }
+    args[:status] = "alive"
+    @admin.create_host("me", args)
+    yield_for_monitor_update { @admin.get_hosts.empty? or break }
+
+    wait_for_usage_file "dev1"
+    wait_for_usage_file "dev2"
+    @admin.create_device("me", 1)
+    @admin.create_device("me", 2)
+    @admin.create_domain("testdom")
+    yield_for_monitor_update { @admin.get_devices.size == 2 and break }
+    @mogc = MogileFS::MogileFS.new(:hosts => @hosts, :domain => "testdom")
+  end
+
+  def yield_for_monitor_update
+    50.times do
+      yield
+      sleep 0.1
+    end
+  end
+
+  def wait_for_port(port)
+    tries = 500
+    begin
+      TCPSocket.new(@test_host, port).close
+      return
+    rescue
+      sleep 0.05
+    end while (tries -= 1) > 0
+    raise "#@test_host:#{port} never became ready"
+  end
+
+  def teardown
+    if @mogstored_pid
+      pid = File.read(@mogstored_pid.path).to_i
+      Process.kill(:TERM, pid) if pid > 0
+    end
+    if @mogilefsd_pid
+      s = TCPSocket.new(@test_host, @tracker_port)
+      s.write "!shutdown\r\n"
+      s.close
+    end
+    FileUtils.rmtree(@docroot)
+  end
+
+  def wait_for_usage_file(device)
+    uri = URI("http://#@test_host:#@mogstored_http_port/#{device}/usage")
+    res = nil
+    100.times do
+      res = Net::HTTP.get_response(uri)
+      if Net::HTTPOK === res
+        puts res.body if $DEBUG
+        return
+      end
+      puts res.inspect if $DEBUG
+      sleep 0.1
+    end
+    raise "#{uri} failed to appear: #{res.inspect}"
+  end
+
+  def setup_mogstored(mogstored = "mogstored")
+    @docroot = Dir.mktmpdir(["mogfresh", "docroot"])
+    Dir.mkdir("#@docroot/dev1")
+    Dir.mkdir("#@docroot/dev2")
+    @mogstored_mgmt = TCPServer.new(@test_host, 0)
+    @mogstored_http = TCPServer.new(@test_host, 0)
+    @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
+    @mogstored_http_port = @mogstored_http.addr[1]
+    @mogstored_conf = Tempfile.new(["mogstored", "conf"])
+    @mogstored_pid = Tempfile.new(["mogstored", "pid"])
+    @mogstored_conf.write <<EOF
+pidfile = #{@mogstored_pid.path}
+maxconns = 500
+httplisten = #@test_host:#@mogstored_http_port
+mgmtlisten = #@test_host:#@mogstored_mgmt_port
+docroot = #@docroot
+EOF
+    @mogstored_conf.flush
+    @mogstored_mgmt.close
+    @mogstored_http.close
+
+    x!(mogstored, "--daemon", "--config=#{@mogstored_conf.path}")
+    wait_for_port @mogstored_mgmt_port
+    wait_for_port @mogstored_http_port
+  end
+
+  def setup
+    setup_mogilefs
+    @err = StringIO.new
+    logger = Logger.new(@err)
+    logger = Logger.new($stderr)
+    @opts = { "rack.logger" => logger }
+    @db_file = Tempfile.new(%w(omgdav .sqlite))
+    @db = Sequel.connect("sqlite://#{@db_file.path}")
+    @db.pragma_set(:synchronous, :off)
+    @migdir = File.dirname(File.dirname(__FILE__)) + "/lib/omgdav/migrations"
+    Sequel::Migrator.apply(@db, @migdir, nil)
+    @app = OMGDAV::App.new(@db, @mogc)
+    @req = Rack::MockRequest.new(@app)
+  end
+
+  def body_string(body)
+    rv = ""
+    body.each { |chunk| rv << chunk }
+    rv
+  end
+
+  # Rack::MockRequest doesn't always work since it converts the
+  # response into an array without dup-ing Strings, so response
+  # bodies which reuse a buffer (when they assume they're writing
+  # to a socket) fail
+  def req(method, uri, opts = {})
+    o = { method: method }.merge!(opts)
+    env = @req.class.env_for(uri, o)
+    @app.call(env)
+  end
+end

data/test/test_copy.rb

@@ -0,0 +1,121 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+class TestCopy < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def test_copy_file
+    # create a file
+    resp = @req.put("/hello", input: StringIO.new("HELLO"))
+    assert_equal 201, resp.status.to_i
+
+    resp = @req.put("/goodbye", input: StringIO.new("GOODBYE"))
+    assert_equal 201, resp.status.to_i
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/bar',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("COPY", "/hello", opts)
+    assert_equal 201, status.to_i
+
+    status, _, body = req("GET", "/bar")
+    assert_equal 200, status.to_i
+    assert_equal "HELLO", body_string(body)
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/bar',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("COPY", "/goodbye", opts)
+    assert_equal 204, status.to_i
+
+    status, _, body = req("GET", "/bar")
+    assert_equal 200, status.to_i
+    assert_equal "GOODBYE", body_string(body)
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/bar',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("COPY", "/bar", opts)
+    assert_equal 403, status.to_i
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/a/b',
+      "HTTP_HOST" => 'example.com:80',
+    }
+    status, _, _ = req("COPY", "/bar", opts)
+    assert_equal 409, status.to_i
+
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/a/b',
+      "HTTP_HOST" => 'example.com:80',
+    }
+    status, _, _ = req("COPY", "/bar", opts)
+    assert_equal 201, status.to_i
+    status, _, body = req("GET", "/a/b")
+    assert_equal 200, status.to_i
+    assert_equal "GOODBYE", body_string(body)
+
+    opts = {
+      'HTTP_DESTINATION' => 'http://example.com/a/b',
+      'HTTP_HOST' => 'example.com',
+      'HTTP_OVERWRITE' => 'F',
+    }
+    status, _, _ = req("COPY", "/hello", opts)
+    assert_equal 412, status.to_i
+    status, _, body = req("GET", "/a/b")
+    assert_equal 200, status.to_i
+    assert_equal "GOODBYE", body_string(body)
+
+    opts = {
+      'HTTP_DESTINATION' => 'http://www.example.com/a/b',
+      'HTTP_HOST' => 'example.com',
+    }
+    assert_equal 502, req("COPY", "/hello", opts)[0].to_i
+
+    opts = {
+      'HTTP_DESTINATION' => 'http://example.com/a/b',
+      'HTTP_HOST' => 'example.com:8080',
+    }
+    assert_equal 502, req("COPY", "/hello", opts)[0].to_i
+  end
+
+  def test_copy_collection_empty
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+
+    opts = {
+      'HTTP_DESTINATION' => 'http://example.com/b',
+      'HTTP_HOST' => 'example.com',
+    }
+    assert_equal 201, req("COPY", "/a", opts)[0].to_i
+    assert @db[:paths][name: "b"][:collection]
+
+    assert_equal 204, req("COPY", "/a", opts)[0].to_i
+  end
+
+  def test_copy_collection_full
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+    assert_equal 201, req("MKCOL", "/a/b")[0].to_i
+    assert_equal 201, @req.put("/a/c", input: StringIO.new("a")).status.to_i
+    assert_equal 201, @req.put("/a/b/c", input: StringIO.new("C")).status.to_i
+
+    opts = {
+      'HTTP_DESTINATION' => 'http://example.com/b',
+      'HTTP_HOST' => 'example.com',
+    }
+    assert_equal 201, req("COPY", "/a", opts)[0].to_i
+
+    status, _, body = req("GET", "/b/c")
+    assert_equal 200, status.to_i
+    assert_equal "a", body_string(body)
+
+    status, _, body = req("GET", "/b/b/c")
+    assert_equal 200, status.to_i
+    assert_equal "C", body_string(body)
+  end
+end

data/test/test_delete.rb

@@ -0,0 +1,15 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+class TestDelete < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def test_delete
+    status, _, _ = req("DELETE", "/non-existent")
+    assert_equal 404, status.to_i
+
+    status, _, _ = req("DELETE", "/")
+    assert_equal 403, status.to_i
+  end
+end

data/test/test_litmus.rb

@@ -0,0 +1,61 @@
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "./test/integration"
+require "unicorn"
+
+LITMUS_DIR = ENV["LITMUS_DIR"]
+class TestLitmus < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def setup
+    super
+    @uni_err = Tempfile.new(%w(stderr .log))
+    @ru = Tempfile.new(%w(omgdav .ru))
+    @ru.sync = true
+    @ru.write <<EOF
+require 'omgdav/app'
+db = Sequel.connect("sqlite://#{@db_file.path}")
+db.pragma_set(:synchronous, :off)
+mogc =  MogileFS::MogileFS.new(hosts: #{@hosts.inspect}, domain: "testdom")
+run OMGDAV::App.new(db, mogc)
+EOF
+    uni_sock = TCPServer.new(@test_host, 0)
+    port = uni_sock.addr[1]
+    @url = "http://#@test_host:#{port}/"
+    @uni_pid = fork do
+      ENV["UNICORN_FD"] = uni_sock.fileno.to_s
+      ENV["RACK_ENV"] = "deployment"
+      $stderr.reopen(@uni_err.path, "a")
+      uni = Unicorn::HttpServer.new(@app, listeners: ["#@test_host:#{port}"])
+      uni.start.join
+    end
+    uni_sock.close
+  end
+
+  def test_litmus
+    cmd = %W(#{ENV["MAKE"] || 'make'} -C #{LITMUS_DIR} URL=#@url check)
+    out = Tempfile.new("err")
+    err = Tempfile.new("out")
+    pid = fork do
+      $stdout.reopen(out)
+      $stderr.reopen(err)
+      exec(*cmd)
+    end
+    _, status = Process.waitpid2(pid)
+    if $DEBUG || $VERBOSE
+      out.rewind
+      err.rewind
+      puts out.read
+      puts err.read
+    end
+    assert status.success?, status.inspect
+  end
+
+  def teardown
+    Process.kill(:QUIT, @uni_pid)
+    _, status = Process.waitpid2(@uni_pid)
+    assert status.success?, status.inspect
+    puts @uni_err.read if $DEBUG || $VERBOSE
+    super
+  end
+end if LITMUS_DIR

data/test/test_move.rb

@@ -0,0 +1,66 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+class TestMove < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def test_move_file
+    # create a file
+    resp = @req.put("/hello", input: StringIO.new("HELLO"))
+    assert_equal 201, resp.status.to_i
+    resp = @req.put("/goodbye", input: StringIO.new("GOODBYE"))
+    assert_equal 201, resp.status.to_i
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/bar',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("MOVE", "/hello", opts)
+    assert_equal 201, status.to_i
+
+    status, headers, body = req("GET", "/bar")
+    assert_equal 200, status.to_i
+    assert_equal "5", headers["Content-Length"]
+    assert_equal "HELLO", body_string(body)
+
+    assert_equal 404, req("GET", "/hello")[0].to_i
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/bar',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("MOVE", "/goodbye", opts)
+    assert_equal 204, status.to_i
+  end
+
+  def test_move_collection
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+    assert_equal 201, req("MKCOL", "/a/b")[0].to_i
+    (0..9).each do |i|
+      input = StringIO.new(i.to_s)
+      assert_equal 201, @req.put("/a/#{i}", input: input).status.to_i
+      input.rewind
+      assert_equal 201, @req.put("/a/b/#{i}", input: input).status.to_i
+    end
+
+    opts = {
+      "HTTP_DESTINATION" => 'http://example.com/c',
+      "HTTP_HOST" => 'example.com',
+    }
+    status, _, _ = req("MOVE", "/a", opts)
+    assert_equal 201, status.to_i
+    (0..9).each do |i|
+      assert_equal 404, @req.get("/a/#{i}").status.to_i
+      assert_equal 404, @req.get("/a/b/#{i}").status.to_i
+
+      status, _, body = req("GET", "/c/#{i}")
+      assert_equal 200, status.to_i
+      assert_equal i.to_s, body_string(body)
+
+      status, _, body = req("GET", "/c/b/#{i}")
+      assert_equal 200, status.to_i
+      assert_equal i.to_s, body_string(body)
+    end
+  end
+end

data/test/test_omgdav_app.rb

@@ -0,0 +1,102 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+class Test_OMGDAV_App < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def test_basic
+    # create a file
+    resp = @req.put("/foo", input: StringIO.new("HELLO"))
+    assert_equal 201, resp.status.to_i
+
+    # make sure it got into MogileFS
+    info = @mogc.file_info("foo")
+    assert_equal 5, info["length"]
+    assert_equal "HELLO", @mogc.get_file_data("foo")
+
+    # ensure we can get it back
+    status, headers, body = req("GET", "/foo")
+    assert_equal 200, status.to_i
+    assert_equal "5", headers["Content-Length"]
+    assert_equal "HELLO", body_string(body)
+    mtime = Time.parse(headers["Last-Modified"])
+    assert_nil body.close
+
+    # ensure we can get a range back
+    status, headers, body = req("GET", "/foo", "HTTP_RANGE" => "bytes=1-")
+    assert_equal 206, status.to_i
+    assert_equal "ELLO", body_string(body)
+    assert_nil body.close
+
+    status, headers, body = req("PROPFIND", "/foo", "HTTP_DEPTH" => "0")
+    assert_equal 207, status.to_i
+    assert_match(%r{\Atext/xml\b}, headers["Content-Type"])
+    foo0 = body_string(body)
+
+    status, headers, body = req("PROPFIND", "/foo", "HTTP_DEPTH" => "1")
+    assert_equal 207, status.to_i
+    assert_match(%r{\Atext/xml\b}, headers["Content-Type"])
+    foo1 = body_string(body)
+    assert_equal foo0, foo1
+
+    # ensure we can get a PROPFIND with Depth: 0
+    status, headers, body = req("PROPFIND", "/", "HTTP_DEPTH" => "0")
+    assert_equal 207, status.to_i
+    xml = Nokogiri::XML(body_string(body))
+    responses = xml.search("//D:multistatus/D:response")
+    assert_equal 1, responses.size
+    root = responses[0]
+    assert_equal "/", root.search("//D:href").text
+    assert_equal 1, root.search(".//D:collection").size
+
+    # PROPFIND with Depth:1
+    status, headers, body = req("PROPFIND", "/", "HTTP_DEPTH" => "1")
+    assert_equal 207, status.to_i
+    xml = Nokogiri::XML(body = body_string(body))
+    responses = xml.search("//D:multistatus/D:response")
+    assert_equal 2, responses.size
+    assert_equal root.to_s, responses[0].to_s
+    foo = responses[1]
+    refute_equal foo.to_s, root.to_s
+    assert_equal "/foo", foo.search(".//D:href").text
+    clen = foo.search(".//lp1:getcontentlength", "lp1" => "DAV:")
+    assert_equal '5', clen.text
+    modified = foo.search(".//lp1:getlastmodified", "lp1" => "DAV:")
+    assert_equal mtime, Time.httpdate(modified.text)
+    created= foo.search(".//lp1:creationdate", "lp1" => "DAV:")
+    assert_equal mtime, Time.xmlschema(created.text)
+    assert_empty foo.search(".//D:collection")
+
+    # PROPFIND with Depth:infinity
+    status, headers, body = req("PROPFIND", "/", "HTTP_DEPTH" => "infinity")
+    assert_equal 400, status.to_i
+
+    # PROPFIND with unspecified Depth (assumed infinity)
+    status, headers, body = req("PROPFIND", "/")
+    assert_equal 400, status.to_i
+  end
+
+  def test_deep_delete
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+    assert_equal 201, req("MKCOL", "/a/b")[0].to_i
+    assert_equal 201, req("MKCOL", "/a/b/c")[0].to_i
+    assert_equal 201, req("MKCOL", "/a/b/c/d")[0].to_i
+    range = (0..9)
+    range.each do |i|
+      resp = @req.put("/a/b/c/d/#{i}", input: StringIO.new("#{i}"))
+      assert_equal 201, resp.status.to_i
+      assert @mogc.exist?("a/b/c/d/#{i}")
+
+      resp = @req.put("/a/#{i}", input: StringIO.new("#{i}"))
+      assert_equal 201, resp.status.to_i
+      assert @mogc.exist?("a/#{i}")
+    end
+    assert_equal 204, req("DELETE", "/a")[0].to_i
+    range.each do |i|
+      refute @mogc.exist?("a/b/c/d/#{i}")
+      refute @mogc.exist?("a/#{i}")
+    end
+    assert_equal [ @app.root_node ], @db[:paths].to_a
+  end
+end

data/test/test_propfind.rb

@@ -0,0 +1,30 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require './test/integration'
+class TestPropfind < MiniTest::Unit::TestCase
+  include TestMogileFSIntegration
+
+  def test_invalid_input
+    opts = {
+      "HTTP_DEPTH" => "0",
+      input: StringIO.new("<foo>bar")
+    }
+    assert_equal 400, req("PROPFIND", "/", opts)[0].to_i
+  end
+
+  def test_dir
+    opts = { "HTTP_DEPTH" => "0" }
+    assert_equal 201, req("MKCOL", "/a")[0].to_i
+
+    status, _, body = req("PROPFIND", "/a", opts)
+    assert_equal 207, status.to_i
+    a0 = body_string(body)
+
+    opts["HTTP_DEPTH"] = "1"
+    status, _, body = req("PROPFIND", "/a", opts)
+    assert_equal 207, status.to_i
+    a1 = body_string(body)
+    assert_equal a0, a1
+  end
+end