omgdav 0.0.0

data/lib/omgdav/move.rb

@@ -0,0 +1,74 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Move
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+  include OMGDAV::CopyMove
+
+  def call_move(env)
+    src, dst = {}, {}
+    err = copy_move_prepare!(env, src, dst) and return err
+
+    dst_node = { name: dst[:basename], parent_id: dst[:parent][:id] }
+
+    if src[:node][:collection]
+      case env['HTTP_DEPTH']
+      when nil, "infinity"
+        move_collection(src[:node], dst_node)
+      else
+        return r(400, "invalid Depth: #{env['HTTP_DEPTH'].inspect}")
+      end
+    else
+      dst_key = node_to_key(dst_node, {})
+      @mogc.rename(src[:key], dst_key)
+    end
+    @db[:paths].where(id: src[:node][:id]).update(dst_node)
+    r(dst[:node] ? 204 : 201)
+  end
+
+  def move_collection(src_node, dst_node)
+    cache_old = {}
+    paths = @db[:paths]
+    queue = [ [ 0, src_node ] ]
+    max_id = paths.max(:id)
+    q = { domain_id: @domain_id }
+
+    # poison cache_new with post-MOVE data
+    cache_new = {}
+    cache_new[src_node[:id]] = node_to_parts(dst_node, {}).freeze
+
+    while cur_job = queue.pop
+      min_id, cur_src = cur_job
+      q[:parent_id] = cur_src[:id]
+      next if min_id == max_id
+      begin
+        continue = false
+        q[:id] = ((min_id+1)..max_id)
+        paths.order(:id).where(q).limit(@sql_limit).each do |child_node|
+          min_id = child_node[:id]
+          if child_node[:collection]
+            queue << [ min_id, cur_src ]
+            queue << [ 0, child_node ]
+
+            # poison cache_new with post-MOVE data
+            tmp = { name: child_node[:name], parent_id: cur_src[:id] }
+            cache_new[child_node[:id]] = node_to_parts(tmp, cache_new).freeze
+
+            continue = false
+            break
+          else
+            old = node_to_key(child_node, cache_old)
+            new = node_to_key(child_node, cache_new)
+            @mogc.rename(old, new)
+            continue = true
+          end
+        end
+      end while continue
+    end
+  end
+end

data/lib/omgdav/options.rb

@@ -0,0 +1,21 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Options
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def call_options(env)
+    resp = r(200)
+    if %r{\A/} =~ env["PATH_INFO"]
+      resp[1]["DAV"] = "1"
+    end
+    resp
+  ensure
+    drain_input(env)
+  end
+end

data/lib/omgdav/propfind.rb

@@ -0,0 +1,46 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+require "omgdav/propfind_response"
+require "omgdav/input_wrapper"
+
+module OMGDAV::Propfind
+  def call_propfind(env)
+    input_validate_propfind(env)
+    parts = path_split(env)
+    depth = env["HTTP_DEPTH"]
+    case depth
+    when "0", "1"
+      # resolve the collection
+      node = node_resolve(parts)
+
+      if node
+        OMGDAV::PropfindResponse.new(env, node, @db).response
+      else
+        r(404)
+      end
+    else
+      return r(400, "Depth: #{depth} not supported")
+    end
+  rescue Nokogiri::SyntaxError => e
+    r(400, "syntax error: #{e.message}")
+  end
+
+  def input_validate_propfind(env)
+    input = OMGDAV::InputWrapper.new(env)
+    parser = Nokogiri::XML::SAX::Parser.new(Validator.new)
+    parser.parse_io(input)
+  rescue Nokogiri::SyntaxError
+    # don't choke on empty input
+    raise if input.bytes != 0
+  end
+
+  class Validator < Nokogiri::XML::SAX::Document
+    def error(string)
+      raise Nokogiri::SyntaxError, string
+    end
+  end
+end

data/lib/omgdav/propfind_response.rb

@@ -0,0 +1,150 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/db"
+
+class OMGDAV::PropfindResponse # :nodoc:
+  include OMGDAV::DB
+
+  # This is a Rack response body for HTTP/1.0 folks who can't handle
+  # Transfer-Encoding: chunked
+  class XMLTmp < Tempfile # :nodoc:
+    def close
+      super true # unlink immediately
+    end
+
+    # no line buffering
+    def each
+      buf = ""
+      while read(16384, buf)
+        yield buf
+      end
+    end
+  end
+
+  def initialize(env, parent, db)
+    @env = env
+    @parent = parent
+    @db = db
+    @n2k_cache = {}
+    @domain_id = parent[:domain_id]
+    @script_name = Rack::Request.new(env).script_name
+    @ct_cache = {}
+  end
+
+  def response
+    headers = { "Content-Type" => 'text/xml; charset="utf-8"' }
+    case @env["HTTP_VERSION"]
+    when "HTTP/1.1"
+      headers["Transfer-Encoding"] = "chunked"
+      body = self
+    else
+      body = XMLTmp.new("omgdav_propfind")
+      body.sync = true
+      each_blob { |blob| body.write(blob) }
+      headers["Content-Length"] = body.size.to_s
+      body.rewind
+    end
+
+    [ 207, headers, body ]
+  end
+
+  # chunks a Rack response body for HTTP/1.1
+  def each
+    each_blob do |blob|
+      yield "#{blob.bytesize.to_s(16)}\r\n#{blob}\r\n"
+    end
+    yield "0\r\n\r\n"
+  end
+
+  def row_xml(x, row)
+    is_col = row[:collection]
+    props = dead_props_get(row)
+    if props
+      prop_ns = 'xmlns:ns0="DAV" '
+      i = 0
+      props_str = ""
+      props.keys.sort.each do |ns|
+        if ns.empty?
+          pfx = ""
+        else
+          pfx = "ns#{i += 1}"
+          prop_ns << "xmlns:#{pfx}=\"#{ns}\" "
+          pfx << ":"
+        end
+        props[ns].each do |name,value|
+          # no need to escape value here because we never escaped it on the
+          # way into the DB(!)
+          props_str << "<#{pfx}#{name}>#{value}</#{pfx}#{name}>"
+        end
+      end
+    end
+
+    x << %Q(<D:response #{prop_ns}xmlns:lp1="DAV:" xmlns:lp2="#{OMGDAV::LP2}">)
+    name = node_to_key(row, @n2k_cache).fast_xs
+    name << "/" if is_col && ! row[:name].empty?
+
+    x << "<D:href>#@script_name/#{name}</D:href>"
+
+    x << "<D:propstat>"
+    x << "<D:prop>"
+
+    if is_col
+      x << "<lp1:resourcetype><D:collection/></lp1:resourcetype>"
+    else
+      x << "<lp1:resourcetype/>"
+      x << "<lp2:executable>#{row[:executable] ? 'T' : 'F'}</lp2:executable>"
+      x << "<lp1:getcontentlength>#{row[:length]}</lp1:getcontentlength>"
+      getcontenttype = content_type(row, @ct_cache)
+      x << "<lp1:getcontenttype>#{getcontenttype}</lp1:getcontenttype>"
+    end
+    x << props_str if props_str
+
+    created = Time.at(row[:created]).utc.xmlschema
+    x << "<lp1:creationdate>#{created}</lp1:creationdate>"
+    mtime = Time.at(row[:mtime]).httpdate
+    x << "<lp1:getlastmodified>#{mtime}</lp1:getlastmodified>"
+    x << "</D:prop>"
+    x << "<D:status>HTTP/1.1 200 OK</D:status>"
+    x << "</D:propstat>"
+    x << "</D:response>"
+  end
+
+  def each_blob
+    paths = @db[:paths]
+    q = { parent_id: @parent[:id], domain_id: @parent[:domain_id] }
+    prev = { name: "" }
+    x = '<?xml version="1.0" encoding="utf-8"?>' \
+        '<D:multistatus xmlns:D="DAV:">'
+
+    # FIXME: this might be horribly inefficient
+    case @env["HTTP_DEPTH"]
+    when "0"
+      row_xml(x, @parent)
+    when "1"
+      if @parent[:collection]
+        row_xml(x, @parent)
+        begin
+          seen = 0
+          paths.order(:name).where {
+            self.&(q, self.>(:name, prev[:name]))
+          }.limit(@sql_limit).each do |row|
+            seen += 1
+            prev = row
+            row_xml(x, row)
+          end
+
+          break if seen != @sql_limit
+          yield x
+          x.clear
+        end while true
+      else
+        row_xml(x, @parent)
+      end
+    end
+
+    x << '</D:multistatus>'
+    yield x
+  end
+end

data/lib/omgdav/proppatch.rb

@@ -0,0 +1,116 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+require "omgdav/input_wrapper"
+
+module OMGDAV::Proppatch
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def value(z)
+    z.children.text.strip
+  end
+
+  def set_props(node, x)
+    props = nil
+    x.children.each do |y|
+      next unless Nokogiri::XML::Element === y
+      next unless "prop" == y.name
+      y.children.each do |z|
+        next unless Nokogiri::XML::Element === z
+        ns = z.namespace.href rescue ""
+        case ns
+        when "DAV:"
+          case z.name
+          when "getcontenttype"
+            node[:contenttype] = content_type_id(value(z))
+          when "getlastmodified"
+            mtime = value(z)
+            node[:mtime] = Time.httpdate(mtime).to_i
+          when "creationdate"
+            created = value(z)
+            node[:created] = Time.iso8601(created).to_i
+          end
+        when OMGDAV::LP2
+          case z.name
+          when "executable"
+            node[:executable] = ("T" == z.children.to_a.join.strip)
+          end
+        else
+          props ||= dead_props_get(node) || {}
+          nsprops = props[ns] ||= {}
+          remove_namespace!(x)
+          nsprops[z.name] = z.children.to_xml
+        end
+      end
+    end
+    dead_props_set(node, props) if props
+  end
+
+  def remove_props(node, x)
+    props = nil
+    x.children.each do |y|
+      next unless Nokogiri::XML::Element === y
+      next unless "prop" == y.name
+      y.children.each do |z|
+        next unless Nokogiri::XML::Element === z
+        ns = z.namespace.href rescue ""
+        case ns
+        when "DAV:"
+          case z.name
+          when "getcontenttype"
+            node[:contenttype] = nil
+          end
+        when OMGDAV::LP2
+          case z.name
+          when "executable"
+            node[:executable] = false
+          end
+        else
+          props ||= dead_props_get(node) or next
+          nsprops = props[ns] or next
+          nsprops.delete(z.name) or next
+        end
+      end
+    end
+    dead_props_set(node, props) if props
+  end
+
+  def call_proppatch(env)
+    node = node_resolve(path_split(env)) or return r(404)
+    input = OMGDAV::InputWrapper.new(env)
+    xml = Nokogiri::XML(input)
+    xmlns = xml.root.namespace
+    return r(400) unless "DAV:" == xmlns.href
+    xml.xpath("//D:propertyupdate", "D" => "DAV:").children.each do |x|
+      next unless Nokogiri::XML::Element === x
+
+      case x.name
+      when "set"
+        set_props(node, x)
+      when "remove"
+        remove_props(node, x)
+      else
+        return r(400, "Unknown name=#{x.name}")
+      end
+    end
+    @db[:paths].where(id: node.delete(:id)).update(node)
+    r(200)
+  rescue Nokogiri::SyntaxError => e
+    r(400, "syntax error #{e.message}")
+  rescue OMGDAV::InvalidContentType => e
+    r(400, "bad getcontenttype: #{e.message}")
+  end
+
+  # removes namespace recursively without operating recursively
+  def remove_namespace!(el)
+    queue = [ el ]
+    while el = queue.shift
+      el.namespace = nil if el.respond_to?(:namespace=)
+      queue.concat(el.children.to_a) if el.respond_to?(:children)
+    end
+  end
+end

data/lib/omgdav/put.rb

@@ -0,0 +1,110 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Put
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def new_file_prepare(env)
+    params = Rack::Utils.parse_query(env["QUERY_STRING"])
+
+    # prepare options for create_open/create_close:
+    new_file_opts = @new_file_opts.dup
+    new_file_opts[:class] = params["class"] || "default"
+
+    # try to give a Content-Length to the tracker
+    clen = env["CONTENT_LENGTH"]
+    clen and new_file_opts[:content_length] = clen.to_i
+
+    if /\bContent-MD5\b/i =~ env["HTTP_TRAILER"]
+      # if the client will give the Content-MD5 as the trailer,
+      # we must lazily populate it since we're not guaranteed to
+      # have the trailer, yet (rack.input is lazily read on unicorn)
+      new_file_opts[:content_md5] = lambda { env["HTTP_CONTENT_MD5"] }
+    elsif cmd5 = env["HTTP_CONTENT_MD5"]
+      # maybe the client gave the Content-MD5 in the header
+      new_file_opts[:content_md5] = cmd5
+    end
+
+    new_file_opts
+  end
+
+  def edit_input(tmp, input, off_out)
+    tmp.seek(off_out)
+    IO.copy_stream(input, tmp)
+    tmp.rewind
+    tmp
+  end
+
+  def call_put(env)
+    return r(403) if %r{/\z} =~ env["PATH_INFO"]
+    return r(100) if %r{\b100-continue\b}i =~ env["HTTP_EXPECT"]
+    parts = path_split(env)
+    key = parts.join("/")
+    basename = parts.pop or return r(403)
+
+    if @create_full_put_path
+      parent = col_vivify(parts)
+    else
+      parent = col_resolve(parts) or return r(404)
+    end
+
+    node = node_lookup(parent[:id], basename)
+    return r(409) if node && @worm
+
+    if range = env["HTTP_CONTENT_RANGE"]
+      %r{\A\s*bytes\s+(\d+)-(\d+)/\*\s*\z} =~ range or
+        return r(400, "Bad range", env)
+      clen = env["CONTENT_LENGTH"] or
+        return r(400, "Content-Length required for Content-Range")
+      off_out = $1.to_i
+      len = $2.to_i - off_out + 1
+      len == clen.to_i or
+             return r(400,
+                      "Bad range, Content-Range: #{range} does not match\n" \
+                      "Content-Length: #{clen.inspect}", env)
+      tmp = Tempfile.new('put_cr')
+      tmp.sync = true
+    end
+
+    input = env["rack.input"]
+    if node
+      if tmp
+        @mogc.get_uris(key, @get_path_opts).each do |uri|
+          case res = OMGDAV::HttpGet.run(nil, uri)
+          when Array
+            res[2].stream_to(tmp)
+            input = edit_input(tmp, input, off_out)
+            break
+          else
+            logger(env).error("#{uri}: #{res.message} (#{res.class})")
+          end
+        end
+        input or return r(500, "Could not retrieve key=#{key.inspect}")
+      end
+    else
+      input = edit_input(tmp, input, off_out) if tmp && off_out != 0
+    end
+
+    # finally, upload the file
+    new_file_opts = new_file_prepare(env)
+    length = @mogc.new_file(key, new_file_opts) do |io|
+      IO.copy_stream(input, io)
+    end
+    info = { "length" => length }
+
+    if node
+      node_update(node, info)
+      r(204)
+    else
+      file_ensure(parent[:id], basename, info)
+      r(201)
+    end
+  ensure
+    tmp.close! if tmp
+  end
+end

data/lib/omgdav/rack_util.rb

@@ -0,0 +1,56 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav"
+require "rack"
+require "rack/utils"
+require "rack/mime"
+require "logger"
+module OMGDAV::RackUtil
+
+  def logger(env)
+    env["rack.logger"] || Logger.new($stderr)
+  end
+
+  # returns a plain-text HTTP response
+  def r(code, msg = nil, env = nil) # :nodoc:
+    if env
+      logger(env).warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
+                       "#{code} #{msg.inspect}")
+    end
+
+    if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
+      [ code, {}, [] ]
+    else
+      msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
+
+      if msg.size > 0
+        # using += to not modify original string (owned by Rack)
+        msg += "\n"
+      end
+
+      [ code,
+        { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
+        [ msg ] ]
+    end
+  end
+
+  def path_split(env)
+    parts = env["PATH_INFO"].gsub(%r{/+\z}, "").split(%r{/+})
+    parts.shift # leading slash
+    parts
+  end
+
+  def drain_input(env)
+    input = env["rack.input"]
+    bytes = 0
+    if buf = input.read(23)
+      bytes += buf.size
+      while input.read(666, buf)
+        bytes += buf.size
+      end
+    end
+    bytes
+  end
+end

data/lib/omgdav/setup.rb

@@ -0,0 +1,16 @@
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# This is the code behind omgdav-setup(1)
+require "omgdav"
+require "rubygems" # we use Gem.load_path
+
+module OMGDAV::Setup
+  def self.run(argv = ARGV.dup)
+    migdir = "#{File.dirname(__FILE__)}/migrations"
+    argv << "-m"
+    argv << migdir
+    ARGV.replace(argv)
+    load Gem.bin_path("sequel", "sequel")
+  end
+end

data/lib/omgdav/sync.rb

@@ -0,0 +1,78 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/db"
+
+# each of these is an sync job
+class OMGDAV::Sync # :nodoc:
+  include OMGDAV::DB
+
+  def initialize(db, mogc, prefix = nil)
+    @db = db
+    @mogc = mogc
+    @prefix = prefix
+    @domain_id = ensure_domain(@mogc.domain)
+  end
+
+  def bad_key(key, msg) # :nodoc:
+    warn "key=#{key.inspect} #{msg}"
+  end
+
+  def info_import(info, seen) # :nodoc:
+    key = info["key"]
+    return bad_key(key, "may not have `//'") if %r{//} =~ key
+    return bad_key(key, "may not start with `/'") if %r{\A/} =~ key
+    return bad_key(key, "may not have a NUL byte") if %r{\0} =~ key
+
+    key.force_encoding(Encoding::UTF_8)
+    return bad_key(key, "is not valid UTF-8") unless key.valid_encoding?
+
+    parts = key.split(%r{/})
+
+    dot = parts.grep(/\A(?:\.\.|\.)\z/)[0]
+    return bad_key(key, "may not contain `..' or `.' as a component") if dot
+
+    filename = parts.pop
+    full_col = parts.join("/")
+
+    begin
+      unless parent_id = seen[full_col]
+        parent_id = root_node[:id]
+        parts.each do |colname|
+          col = col_ensure(parent_id, colname)
+          parent_id = col[:id]
+        end
+        seen[full_col] = parent_id
+      end
+
+      file_ensure(parent_id, filename, info)
+    rescue OMGDAV::TypeConflict
+      bad_key(key, "path conflicts with existing collection or file")
+    end
+  end
+
+  def sync # :nodoc:
+    synctmp = @db[:synctmp]
+    seen = {} # optimization for keys with many path elements
+
+    # snapshot all existing path ids into synctmp table
+    synctmp.delete
+    @db["INSERT INTO synctmp(id) SELECT id FROM paths"].insert
+
+    pd = synctmp.where(id: :$i).prepare(:delete, :delete_by_id, id: :$i)
+
+    @mogc.each_file_info(@prefix) do |info|
+      # delete valid nodes from synctmp as we iterate
+      node = info_import(info, seen) and pd.call(i: node[:id])
+
+      # don't let a pathological case OOM us
+      seen.clear if seen.size > 1000
+    end
+
+    # any ids leftover in the synctmp table are stale
+    @db["DELETE FROM paths WHERE id IN (SELECT id FROM synctmp)"].delete
+  ensure
+    synctmp.delete # cleanup
+  end
+end

data/lib/omgdav/version.rb

@@ -0,0 +1,2 @@
+# :enddoc:
+OMGDAV::VERSION = '0.0.0'

data/lib/omgdav.rb

@@ -0,0 +1,27 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "tempfile"
+require 'time'
+require 'uri'
+require 'nokogiri'
+require 'json'
+require 'sequel'
+require 'mogilefs'
+require 'fast_xs'
+require 'rack'
+require 'rack/request'
+# :startdoc:
+
+module OMGDAV
+  # :stopdoc:
+  TypeConflict = Class.new(TypeError)
+  InvalidContentType = Class.new(ArgumentError)
+  BadResponse = Class.new(RuntimeError)
+
+  LP2 = "http://apache.org/dav/props/"
+  # :startdoc:
+end
+# :enddoc:
+require "omgdav/version"