omgdav 0.0.0

data/lib/omgdav/db.rb

@@ -0,0 +1,258 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav"
+module OMGDAV::DB
+
+  # returns a domain ID
+  def ensure_domain(domain)
+    domains = @db[:domains]
+    @db.transaction do
+      q = { domain: domain }
+      dom = domains[q]
+      dom ? dom[:id] : domains.insert(q)
+    end
+  end
+
+  def node_merge_info(node, info)
+    info or return
+    node[:length] = info["length"]
+  end
+
+  def node_update(node, info = nil)
+    node_merge_info(node, info)
+    mtime = node[:mtime] = Time.now.to_i
+    node_id = node.delete(:id)
+    @db[:paths].where(id: node_id).update(node)
+
+    unless node[:collection]
+      @db[:paths].where(id: node[:parent_id]).update(mtime: mtime)
+    end
+
+    node[:id] = node_id
+  end
+
+  def node_update_maybe(node, info)
+    info or return
+    if info && node[:length] != info["length"]
+      node_update(node)
+    end
+  end
+
+  def node_ensure(parent_id, name, info = nil)
+    q = { name: name, domain_id: @domain_id, parent_id: parent_id }
+    collection = info ? false : true
+
+    node = @db[:paths][q]
+    if node
+      # node already exists
+      raise OMGDAV::TypeConflict if node[:collection] != collection
+      node_update_maybe(node, info)
+    else
+      # brand new node
+      node = q.dup
+      node[:created] = node[:mtime] = Time.now.to_i
+      node[:collection] = collection
+      node_merge_info(node, info)
+      begin
+        node[:id] = @db[:paths].insert(node)
+      rescue Sequel::DatabaseError
+        # we may conflict on insert if we didn't use a transaction
+        raise if @db.in_transaction?
+        node = @db[:paths][q] or raise
+        raise OMGDAV::TypeConflict if node[:collection] != collection
+      end
+    end
+
+    node
+  end
+
+  # ensures the given args maps to a directory, returns a hash for the row
+  def col_ensure(parent_id, colname)
+    node_ensure(parent_id, colname)
+  end
+
+  # ensures the given args maps to a file, returns a hash for the row
+  def file_ensure(parent_id, filename, info)
+    node_ensure(parent_id, filename, info)
+  end
+
+  def node_lookup(parent_id, name)
+    @db[:paths][name: name, domain_id: @domain_id, parent_id: parent_id]
+  end
+
+  def node_to_parts(node, cache)
+    root_id = root_node[:id]
+    return [] if root_id == node[:id] # special case
+
+    orig_name = node[:name]
+    cache_key = node[:parent_id]
+    unless parts = cache[cache_key]
+      parts = []
+      until root_id == node[:parent_id]
+        node = @db[:paths][id: node[:parent_id], domain_id: @domain_id] or break
+        parts.unshift(node[:name].dup.freeze)
+      end
+
+      cache[cache_key] = parts.freeze
+    end
+
+    parts += [ orig_name ]
+  end
+
+  def node_to_key(node, cache)
+    node_to_parts(node, cache).join("/")
+  end
+
+  def col_resolve(parts)
+    node = root_node
+    parts.each do |colpart|
+      node = node_lookup(node[:id], colpart)
+      return unless node
+      raise OMGDAV::TypeConflict unless node[:collection]
+    end
+    raise OMGDAV::TypeConflict unless node[:collection]
+    node
+  end
+
+  def node_resolve(parts)
+    return root_node if parts.empty?
+    parts = parts.dup
+    basename = parts.pop or return
+
+    parent = col_resolve(parts) or return
+    node_lookup(parent[:id], basename)
+  end
+
+  def file_resolve(parts)
+    node = node_resolve(parts) or return
+    raise OMGDAV::TypeConflict if node[:collection]
+    node
+  end
+
+  def col_vivify(parts)
+    col = root_node
+    parts.each do |colname|
+      col = col_ensure(col[:id], colname)
+    end
+    col
+  end
+
+  def node_delete(node)
+    rc = @db[:paths].where(id: node[:id]).delete
+    if rc != 1
+      key = node_to_key(node, {}).inspect
+      warn "expected rc=1 for delete(id=#{node[:id]}) (key=#{key}) got rc=#{rc}"
+    end
+    rc
+  end
+
+  def dead_props_get(node)
+    db_props = node[:dead_props] or return
+    db_props = JSON.parse(db_props)
+    mapping = {}
+    @db[:prop_mappings].each { |row| mapping[row[:id]] = row[:value] }
+    raw_props = {}
+    db_props.each do |ns, hash|
+      tmp = raw_props[mapping[ns.to_i] || ns] = {}
+      hash.each { |i, value| tmp[mapping[i.to_i] || i] = value }
+    end
+    raw_props
+  end
+
+  # returns the new property ID for a given value
+  def prop_mapping_ensure(value)
+    tbl = @db[:prop_mappings]
+    q = { value: value }
+    tbl.insert(q)
+  rescue Sequel::Database::Error
+    raise if @db.in_transaction?
+    mapping = tbl[q] or raise
+    mapping[:id]
+  end
+
+  def dead_props_set(node, raw_props)
+    mapping = {}
+    @db[:prop_mappings].each { |row| mapping[row[:value]] = row[:id] }
+    db_props = {}
+    raw_props.each do |ns, hash|
+      ns_id = mapping[ns] ||= prop_mapping_ensure(ns)
+      tmp = db_props[ns_id] = {}
+      hash.each do |key,value|
+        key_id = mapping[key] ||= prop_mapping_ensure(key)
+        tmp[key_id] = value
+      end
+    end
+    node[:dead_props] = db_props.to_json
+  end
+
+  def root_node
+    q = @root_node and return q
+    # root node always has parent_id:0
+    q = {
+      parent_id: 0,
+      collection: true,
+      name: "",
+      domain_id: @domain_id
+    }
+    node = @db[:paths][q] and return (@root_node = node)
+    q[:mtime] = q[:created] = Time.now.to_i
+    begin
+      q[:id] = @db[:paths].insert(q)
+      q
+    rescue Sequel::DatabaseError
+      # we may conflict on insert if we didn't use a transaction
+      raise if @db.in_transaction?
+      @root_node = @db[:paths][q] or raise
+    end
+  end
+
+  # returns the mime type based on key name
+  def key_mime_type(key) # :nodoc:
+    /(\.[^.]+)\z/ =~ key
+    Rack::Mime.mime_type($1) # nil => 'application/octet-stream'
+  end
+
+  def content_type(node, cache = nil)
+    if ctid = node[:contenttype]
+      if row = @db[:prop_mappings][id: ctid]
+        type = row[:value]
+      else
+        warn "DB consistency error: no property mapping for #{ctid.inspect}"
+        type = key_mime_type(node[:name])
+      end
+      cache ? (cache[ctid] = type) : type
+    else
+      key_mime_type(node[:name])
+    end
+  end
+
+  def check_attr!(name, value)
+    # TODO: check compliance, erring on the side of being too strict here
+    /\A[\w\.\+-]+\z/ =~ value or
+      raise OMGDAV::InvalidContentType, "#{name}=#{value.inspect} invalid"
+  end
+
+  def content_type_id(value)
+    # normalize the content type
+    type_subtype, parameter = value.split(/;\s+/, 2)
+    type, subtype = type_subtype.downcase.split(%r{/}, 2)
+    check_attr!(:type, type)
+    check_attr!(:subtype, subtype)
+    value = "#{type}/#{subtype}"
+
+    if parameter
+      attr_key, attr_val = parameter.split(/=/, 2)
+      attr_val or
+        raise OMGDAV::InvalidContentType,
+              "parameter=#{parameter.inspect} invalid"
+      attr_key.downcase!
+      check_attr!(:parameter, attr_key)
+      value << "; #{attr_key}=#{attr_val}"
+    end
+
+    row = @db[:prop_mappings][value: value] and return row[:id]
+    prop_mapping_ensure(value)
+  end
+end

data/lib/omgdav/delete.rb

@@ -0,0 +1,66 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Delete
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def delete_file(node, cache)
+    node_delete(node)
+    key = node_to_key(node, cache)
+    @mogc.delete(key)
+  rescue MogileFS::Backend::UnknownKeyError
+    warn "key=#{key.inspect} missing on delete from domain=#{@mogc.domain}"
+  end
+
+  def delete_collection(node, cache)
+    queue = [ node ]
+    q = { domain_id: @domain_id }
+
+    while cur_node = queue.pop
+      q[:parent_id] = cur_node[:id]
+
+      begin
+        state = :done
+        @db[:paths].where(q).limit(@sql_limit).each do |child_node|
+          if child_node[:collection]
+            queue << cur_node
+            queue << child_node
+            state = :descend
+            break
+          else
+            delete_file(child_node, cache)
+            state = :continue
+          end
+        end
+      end while state == :continue
+
+      node_delete(cur_node) if state == :done
+    end
+  end
+
+  def delete_entry(node, cache)
+    if node[:collection]
+      delete_collection(node, cache)
+    else
+      delete_file(node, cache)
+    end
+  end
+
+  def call_delete(env)
+    # makes litmus happy with the "delete_fragment" test
+    # FRAGMENT may be set by Mongrel and descendents (unicorn/thin)
+    env["FRAGMENT"] and return r(400)
+
+    parts = path_split(env)
+    node = node_resolve(parts) or return r(404)
+    node[:parent_id] == 0 and return r(403)
+    delete_entry(node, {})
+
+    r(204)
+  end
+end

data/lib/omgdav/get.rb

@@ -0,0 +1,35 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Get
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def call_get(env)
+    parts = path_split(env)
+    basename = parts.pop
+    parent = col_resolve(parts) or return r(404)
+    node = node_lookup(parent[:id], basename) or return r(404)
+    length = node[:length]
+    parts << basename
+    @mogc.get_uris(parts.join("/"), @get_path_opts).each do |uri|
+      case res = OMGDAV::HttpGet.run(env, uri)
+      when Array
+        headers = res[1]
+        headers["Last-Modified"] = Time.at(node[:mtime]).httpdate
+        headers["Content-Type"] = content_type(node)
+        return res
+      else
+        logger(env).error("#{uri}: #{res.message} (#{res.class})")
+      end
+    end
+    r(500)
+  rescue MogileFS::Backend::UnknownKeyError,
+         MogileFS::Backend::DomainNotFoundError
+    r(404, "")
+  end
+end

data/lib/omgdav/http_get.rb

@@ -0,0 +1,146 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "thread"
+require "timeout"
+require "kcar"
+
+class OMGDAV::HttpGet < Kcar::Response # :nodoc:
+  # support gzip and partial requests if storage nodes support it
+  PASS = %w(Range Accept-Encoding).map! do |x|
+    [ "HTTP_#{x.tr('a-z-', 'A-Z_')}", x ]
+  end
+
+  DEF_ENV = { "REQUEST_METHOD" => "GET" }
+  ADDR_LOCK = Mutex.new
+  ADDR_CACHE = {}
+  POOL_LOCK = Mutex.new
+  POOL = Hash.new { |pool,ip_port| pool[ip_port] = [] }
+
+  def self.reset
+    ADDR_LOCK.synchronize { ADDR_CACHE.clear }
+    POOL_LOCK.synchronize { POOL.clear }
+  end
+
+  class MySocket < Kgio::Socket # :nodoc:
+    attr_writer :expiry
+
+    def wait_time
+      tout = @expiry ? @expiry - Time.now : @timeout
+      raise Timeout::Error if tout < 0
+      tout
+    end
+
+    def readpartial(bytes, buf = Thread.current[:omgdav_buf] ||= "")
+      case kgio_tryread(bytes, buf)
+      when String
+        return buf
+      when :wait_readable
+        kgio_wait_readable(wait_time)
+      when nil
+        raise EOFError, "end of file reached", []
+      end while true
+    end
+
+    def start(buf, timeout)
+      @timeout = timeout
+      @expiry = Time.now + timeout
+      case rv = kgio_trywrite(buf)
+      when :wait_writable
+        kgio_wait_writable(wait_time)
+      when nil
+        return
+      when String
+        buf = rv
+      end while true
+    end
+  end
+
+  # Called by the Rack server at the end of a successful response
+  def close
+    reusable = @parser.keepalive? && @parser.body_eof?
+    super
+    POOL_LOCK.synchronize { POOL[@ip_port] << self } if reusable
+    nil
+  end
+
+  def dispatch(req, timeout)
+    @sock.start(req, timeout)
+  end
+
+  # returns true if the socket is still alive, nil if dead
+  def sock_alive?
+    @reused = (:wait_readable == (@sock.kgio_tryread(1) rescue nil)) ?
+              true : @sock.close
+  end
+
+  # returns true if the socket was reused and thus retryable
+  def fail_retryable?
+    @sock.close
+    @reused
+  end
+
+  def initialize(sock, ip_port)
+    super(sock)
+    @reused = false
+    @ip_port = ip_port
+  end
+
+  def self.optional_headers(env)
+    PASS.map { |from, to| tmp = env[from] and "#{to}: #{tmp}\r\n" }.join
+  end
+
+  def self.run(env, uri, timeout = 2)
+    obj = obj_get(uri.host, uri.port)
+    env ||= DEF_ENV
+    m = env["REQUEST_METHOD"]
+    req = "#{m} #{uri.path} HTTP/1.0\r\n" \
+          "Connection: keep-alive\r\n" \
+          "#{optional_headers(env)}\r\n"
+    begin
+      obj.dispatch(req, timeout)
+      status, header, body = res = obj.rack
+
+      case m
+      when "GET"
+        body.sock.expiry = nil
+      when "HEAD"
+        # kcar doesn't know if it's a HEAD or GET response, and HEAD
+        # responses have Content-Length in it which fools kcar..
+        body.parser.body_bytes_left = 0
+        res[1] = header.dup
+        body.close # clobbers original header
+        res[2] = body = []
+      end
+
+      case status.to_i
+      when 206, 200
+        res
+      else
+        OMGDAV::BadResponse.new(status.to_s)
+      end
+    rescue => e
+      retry if obj.fail_retryable?
+      return e
+    end
+  end
+
+  def self.obj_get(ip, port)
+    ip_port = "#{ip}:#{port}"
+    while obj = POOL_LOCK.synchronize { POOL[ip_port].pop }
+      return obj if obj.sock_alive?
+    end
+
+    addr = ADDR_LOCK.synchronize do
+      ADDR_CACHE[ip_port] ||= Socket.sockaddr_in(port, ip)
+    end
+
+    new(MySocket.start(addr), ip_port)
+  end
+
+  def stream_to(io)
+    each { |buf| io.write(buf) }
+    close
+  end
+end

data/lib/omgdav/input_wrapper.rb

@@ -0,0 +1,32 @@
+# -*- encoding: binary -*-
+# :enddoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+class OMGDAV::InputWrapper # :nodoc:
+  attr_reader :bytes
+
+  def initialize(env, max = 65536)
+    @input = env["rack.input"]
+    @bytes = 0
+    @max = max
+  end
+
+  def read(*args)
+    rv = @input.read(*args)
+    if String === rv
+      @bytes += rv.size
+      if @bytes > @max
+        buf = ""
+        while @input.read(16384, buf)
+          # drain input
+        end
+        raise "too many bytes for XML: #@bytes > #@max"
+      end
+    end
+    rv
+  end
+
+  # no-op to fool Nokogiri
+  def close
+  end
+end

data/lib/omgdav/migrations/0001_initial.rb

@@ -0,0 +1,45 @@
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# :enddoc:
+Sequel.migration do
+  up do
+    create_table(:paths) do
+      primary_key :id, type: Bignum
+      Integer :domain_id, null: false # MogileFS domain
+      TrueClass :collection, default: false
+      Bignum :parent_id, null: false # parent collection
+      Bignum :mtime, null: false
+      Bignum :created, null: false
+      Bignum :length, null: true
+      TrueClass :executable, null: false, default: false
+      String :name, null: false
+      String :dead_props, null: true, text: true
+      unique [ :domain_id, :parent_id, :name ]
+    end
+
+    create_table(:domains) do
+      primary_key :id
+      String :domain, null: false, unique: true
+    end
+
+    create_table(:prop_mappings) do
+      primary_key :id
+      String :value, null: false, unique: true
+    end
+
+    create_table(:copymove_locks) do
+      Integer :domain_id, null: false
+      Bignum :src_id, null: false
+      Bignum :dst_id, null: false
+      Bignum :lock_expire, null: false
+      unique [ :domain_id, :src_id, :dst_id ]
+    end
+  end
+
+  down do
+    drop_table(:paths)
+    drop_table(:domains)
+    drop_table(:prop_mappings)
+    drop_table(:copymove_locks)
+  end
+end

data/lib/omgdav/migrations/0002_contenttype.rb

@@ -0,0 +1,15 @@
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# :enddoc:
+Sequel.migration do
+  up do
+    alter_table(:paths) do
+      add_column :contenttype, Integer, null: true, default: nil
+    end
+  end
+  down do
+    alter_table(:paths) do
+      drop_column :contenttype
+    end
+  end
+end

data/lib/omgdav/migrations/0003_synctmp.rb

@@ -0,0 +1,14 @@
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+# :enddoc:
+Sequel.migration do
+  up do
+    create_table(:synctmp) do
+      primary_key :id, type: Bignum
+    end
+  end
+
+  down do
+    drop_table(:synctmp)
+  end
+end

data/lib/omgdav/mkcol.rb

@@ -0,0 +1,28 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# Copyright (C) 2012, Eric Wong <normalperson@yhbt.net>
+# License: AGPLv3 or later (https://www.gnu.org/licenses/agpl-3.0.txt)
+require "omgdav/rack_util"
+require "omgdav/db"
+
+module OMGDAV::Mkcol
+  include OMGDAV::DB
+  include OMGDAV::RackUtil
+
+  def call_mkcol(env)
+    bytes = drain_input(env)
+    return r(415) if bytes > 0
+    parts = path_split(env)
+    return r(405) if col_resolve(parts)
+    basename = parts.pop or return r(403)
+    parent = col_resolve(parts) or return r(409)
+    if node = node_lookup(parent[:id], basename)
+      return r(403) unless node[:collection]
+    else
+      node = col_ensure(parent[:id], basename)
+    end
+    r(201)
+  rescue OMGDAV::TypeConflict
+    r(409)
+  end
+end