doze 0.0.11

data/lib/doze/serialization/multipart_form_data.rb

@@ -0,0 +1,162 @@
+require 'doze/media_type'
+require 'doze/serialization/entity'
+require 'doze/serialization/form_data_helpers'
+require 'doze/error'
+require 'doze/utils'
+require 'tempfile'
+
+module Doze::Serialization
+  # Also ripped off largely from Merb::Parse.
+  #
+  # Small differences in the hash it returns for an uploaded file - it will have string keys,
+  # use media_type rather than content_type (for consistency with rest of doze) and adds a temp_path
+  # key.
+  #
+  # These enable it to be used interchangably with nginx upload module if you use config like eg:
+  #
+  # upload_set_form_field $upload_field_name[filename] "$upload_file_name";
+  # upload_set_form_field $upload_field_name[media_type] "$upload_content_type";
+  # upload_set_form_field $upload_field_name[temp_path] "$upload_tmp_path";
+  # upload_aggregate_form_field $upload_field_name[size] "$upload_file_size";
+  #
+  class Entity::MultipartFormData < Entity
+    include FormDataHelpers
+
+    NAME_REGEX         = /Content-Disposition:.* name="?([^\";]*)"?/ni.freeze
+    CONTENT_TYPE_REGEX = /Content-Type: (.*)\r\n/ni.freeze
+    FILENAME_REGEX     = /Content-Disposition:.* filename="?([^\";]*)"?/ni.freeze
+    CRLF               = "\r\n".freeze
+    EOL                = CRLF
+
+    def object_data(try_deserialize=true)
+      @object_data ||= if @lazy_object_data
+        @lazy_object_data.call
+      elsif try_deserialize
+        @binary_data_stream && deserialize_stream
+      end
+    end
+
+    def deserialize_stream
+      boundary = @media_type_params && @media_type_params['boundary'] or raise  "missing boundary parameter for multipart/form-data"
+      boundary = "--#{boundary}"
+      paramhsh = {}
+      buf      = ""
+      input    = @binary_data_stream
+      input.binmode if defined? input.binmode
+      boundary_size = boundary.size + EOL.size
+      bufsize       = 16384
+      length  = @binary_data_length or raise "expected Content-Length for multipart/form-data"
+      length -= boundary_size
+      # status is boundary delimiter line
+      status = input.read(boundary_size)
+      return {} if status == nil || status.empty?
+      raise "bad content body:\n'#{status}' should == '#{boundary + EOL}'"  unless status == boundary + EOL
+      # second argument to Regexp.quote is for KCODE
+      rx = /(?:#{EOL})?#{Regexp.quote(boundary,'n')}(#{EOL}|--)/
+      loop {
+        head      = nil
+        body      = ''
+        filename  = content_type = name = nil
+        read_size = 0
+        until head && buf =~ rx
+          i = buf.index("\r\n\r\n")
+          if( i == nil && read_size == 0 && length == 0 )
+            length = -1
+            break
+          end
+          if !head && i
+            head = buf.slice!(0, i+2) # First \r\n
+            buf.slice!(0, 2)          # Second \r\n
+
+            # String#[] with 2nd arg here is returning
+            # a group from match data
+            filename     = head[FILENAME_REGEX, 1]
+            content_type = head[CONTENT_TYPE_REGEX, 1]
+            name         = head[NAME_REGEX, 1]
+
+            if filename && !filename.empty?
+              body = Tempfile.new(:Doze)
+              body.binmode if defined? body.binmode
+            end
+            next
+          end
+
+          # Save the read body part.
+          if head && (boundary_size+4 < buf.size)
+            body << buf.slice!(0, buf.size - (boundary_size+4))
+          end
+
+          read_size = bufsize < length ? bufsize : length
+          if( read_size > 0 )
+            c = input.read(read_size)
+            raise "bad content body"  if c.nil? || c.empty?
+            buf << c
+            length -= c.size
+          end
+        end
+
+        # Save the rest.
+        if i = buf.index(rx)
+          # correct value of i for some edge cases
+          if (i > 2) && (j = buf.index(rx, i-2)) && (j < i)
+             i = j
+           end
+          body << buf.slice!(0, i)
+          buf.slice!(0, boundary_size+2)
+
+          length = -1  if $1 == "--"
+        end
+
+        if filename && !filename.empty?
+          body.rewind
+          data = {
+            "filename"      => File.basename(filename),
+            "media_type"    => content_type,
+            "tempfile"      => body,
+            "temp_path"     => body.path,
+            "size"          => File.size(body.path)
+          }
+        else
+          data = body
+        end
+        paramhsh = normalize_params(paramhsh,name,data)
+        break  if buf.empty? || length == -1
+      }
+      paramhsh
+    end
+
+    # This is designed to work with either actual file upload fields, or the corresponding
+    # fields generated by nginx upload module as described above.
+    #
+    # yields or returns a Doze::Entity for the uploaded file, with the correct media_type and binary_data_length.
+    # ensures to close and unlinks the underlying tempfile afterwards where used with a block.
+    def param_entity(name)
+      meta = object_data[name]; return unless meta.is_a?(Hash)
+      media_type = meta["media_type"] and media_type = Doze::MediaType[media_type] or return
+      size = meta["size"] && meta["size"].to_i
+      if (tempfile = meta["tempfile"])
+        temp_path = tempfile.path
+      elsif (temp_path = meta["temp_path"])
+        tempfile = File.open(meta["temp_path"], "rb")
+      end
+      return unless tempfile
+      entity = media_type.new_entity(:binary_data_stream => tempfile, :binary_data_length => size)
+
+      return entity unless block_given?
+      begin
+        yield entity
+      ensure
+        tempfile.close
+        begin
+          File.unlink(temp_path)
+        rescue StandardError
+          # we made an effort to clean up - but it is a tempfile, so no biggie
+        end
+      end
+    end
+  end
+
+  # A browser-friendly media type for use with Doze::Serialization::Resource.
+  MULTIPART_FORM_DATA = Doze::MediaType.register('multipart/form-data', :entity_class => Entity::MultipartFormData)
+end
+

data/lib/doze/serialization/resource.rb

@@ -0,0 +1,30 @@
+require 'doze/serialization/json'
+require 'doze/serialization/www_form_encoded'
+require 'doze/serialization/multipart_form_data'
+require 'doze/serialization/html'
+
+# A resource whose representations are all serializations of some ruby data.
+# A good example of how to do media type negotiation
+module Doze::Serialization
+  module Resource
+    # You probably want to override these
+    def serialization_media_types
+      [JSON, HTML]
+    end
+
+    # Analogous to get, but returns data which may be serialized into entities of any one of serialization_media_types
+    def get_data
+    end
+
+    def get
+      serialization_media_types.map do |media_type|
+        media_type.entity_class.new(media_type, :lazy_object_data => proc {get_data})
+      end
+    end
+
+    # You may want to be more particular than this if you can only deal with certain serialization types
+    def accepts_method_with_media_type?(method, entity)
+      entity.is_a?(Doze::Serialization::Entity)
+    end
+  end
+end

data/lib/doze/serialization/resource_proxy.rb

@@ -0,0 +1,14 @@
+require 'doze/serialization/resource'
+require 'doze/resource/proxy'
+
+class Doze::Serialization::ResourceProxy < Doze::Resource::Proxy
+  include Doze::Serialization::Resource
+
+  def serialization_media_types
+    target && target.serialization_media_types
+  end
+
+  def get_data
+    target && target.get_data
+  end
+end

data/lib/doze/serialization/www_form_encoded.rb

@@ -0,0 +1,42 @@
+require 'doze/media_type'
+require 'doze/serialization/entity'
+require 'doze/serialization/form_data_helpers'
+require 'doze/error'
+require 'doze/utils'
+
+module Doze::Serialization
+  # ripped off largely from Merb::Parse
+  # Supports PHP-style nested hashes via foo[bar][baz]=boz
+  class Entity::WWWFormEncoded < Entity
+    include FormDataHelpers
+
+    def serialize(value, prefix=nil)
+      case value
+      when Array
+        value.map {|v| serialize(v, "#{prefix}[]")}.join("&")
+      when Hash
+        value.map {|k,v| serialize(v, prefix ? "#{prefix}[#{escape(k)}]" : escape(k))}.join("&")
+      else
+        "#{prefix}=#{escape(value)}"
+      end
+    end
+
+    def deserialize(data)
+      query = {}
+      for pair in data.split(/[&;] */n)
+        key, value = unescape(pair).split('=',2)
+        next if key.nil?
+        if key.include?('[')
+          normalize_params(query, key, value)
+        else
+          query[key] = value
+        end
+      end
+      query
+    end
+  end
+
+  # A browser-friendly media type for use with Doze::Serialization::Resource.
+  WWW_FORM_ENCODED = Doze::MediaType.register('application/x-www-form-urlencoded', :entity_class => Entity::WWWFormEncoded)
+end
+

data/lib/doze/serialization/yaml.rb

@@ -0,0 +1,25 @@
+require 'yaml'
+require 'doze/media_type'
+require 'doze/serialization/entity'
+require 'doze/error'
+
+# Note that it isn't safe to accept YAML input, unless you trust the sender, as
+# it is possible to craft a YAML message to allow remote code execution (see
+# cve-2013-0156)
+module Doze::Serialization
+  class Entity::YAML < Entity
+    def serialize(ruby_data)
+      ruby_data.to_yaml
+    end
+
+    def deserialize(binary_data)
+      begin
+        ::YAML.load(binary_data)
+      rescue ::YAML::ParseError, ArgumentError
+        raise Doze::ClientEntityError, "Could not parse YAML"
+      end
+    end
+  end
+
+  YAML = Doze::MediaType.register('application/yaml', :plus_suffix => 'yaml', :entity_class => Entity::YAML, :extension => 'yaml')
+end

data/lib/doze/uri_template.rb

@@ -0,0 +1,220 @@
+# Implements a subset of URI template spec.
+# This is somewhat optimised for fast matching and generation of URI strings, although probably
+# a fair bit of mileage still to be gotten out of it.
+class Doze::URITemplate
+  def self.compile(string, var_regexps={})
+    is_varexp = true
+    parts = string.split(/\{(.*?)\}/).map do |bit|
+      if (is_varexp = !is_varexp)
+        case bit
+        when /^\/(.*).quadhexbytes\*$/
+          QuadHexBytesVariable.new($1.to_sym)
+        else
+          var = bit.to_sym
+          Variable.new(var, var_regexps[var] || Variable::DEFAULT_REGEXP)
+        end
+      else
+        String.new(bit)
+      end
+    end
+    template = parts.length > 1 ? Composite.new(parts) : parts.first
+    template.compile_expand!
+    template
+  end
+
+  # Compile a ruby string substitution expression for the 'expand' method to make filling out these templates blazing fast.
+  # This was actually a bottleneck in some simple cache lookups by list of URIs
+  def compile_expand!
+    instance_eval "def expand(vars); \"#{expand_code_fragment}\"; end", __FILE__, __LINE__
+  end
+
+  def anchored_regexp
+    @anchored_regexp ||= Regexp.new("^#{regexp_fragment}$")
+  end
+
+  def start_anchored_regexp
+    @start_anchored_regexp ||= Regexp.new("^#{regexp_fragment}")
+  end
+
+  def parts; @parts ||= [self]; end
+
+  def +(other)
+    other = String.new(other.to_s) unless other.is_a?(Doze::URITemplate)
+    Composite.new(parts + other.parts)
+  end
+
+  def inspect
+    "#<#{self.class} #{to_s}>"
+  end
+
+  def match(uri)
+    match = anchored_regexp.match(uri) or return
+    result = {}; captures = match.captures
+    variables.each_with_index do |var, index|
+      result[var.name] = var.translate_captured_string(captures[index])
+    end
+    result
+  end
+
+  def match_with_trailing(uri)
+    match = start_anchored_regexp.match(uri) or return
+    result = {}; captures = match.captures
+    variables.each_with_index do |var, index|
+      result[var.name] = var.translate_captured_string(captures[index])
+    end
+    trailing = match.post_match
+    trailing = nil if trailing.empty?
+    [result, match.to_s, trailing]
+  end
+
+  class Variable < Doze::URITemplate
+    DEFAULT_REGEXP = "[^\/.,;?]+"
+
+    attr_reader :name
+
+    def initialize(name, regexp=DEFAULT_REGEXP)
+      @name = name; @regexp = regexp
+    end
+
+    def regexp_fragment
+      "(#{@regexp})"
+    end
+
+    def to_s
+      "{#{@name}}"
+    end
+
+    def variables; [self]; end
+
+    def expand(vars)
+      Doze::Utils.escape(vars[@name].to_s)
+    end
+
+    def partially_expand(vars)
+      if vars.has_key?(@name)
+        String.new(expand(vars))
+      else
+        self
+      end
+    end
+
+    def translate_captured_string(string)
+      # inlines Doze::Utils.unescape, but with gsub! rather than gsub since this is faster and the matched string is throwaway
+      string.gsub!(/((?:%[0-9a-fA-F]{2})+)/n) {[$1.delete('%')].pack('H*')}; string
+    end
+
+    # String#size under Ruby 1.8 and String#bytesize under 1.9.
+    BYTESIZE_METHOD = ''.respond_to?(:bytesize) ? 'bytesize' : 'size'
+
+    # inlines Doze::Utils.escape (optimised from Rack::Utils.escape) with further effort to avoid an extra method call for bytesize 1.9 compat.
+    def expand_code_fragment
+      "\#{vars[#{@name.inspect}].to_s.gsub(/([^a-zA-Z0-9_.-]+)/n) {'%'+$1.unpack('H2'*$1.#{BYTESIZE_METHOD}).join('%').upcase}}"
+    end
+  end
+
+  class QuadHexBytesVariable < Variable
+    REGEXP = "(?:/[0-9a-f]{2}){4}"
+
+    def initialize(name)
+      super(name, REGEXP)
+    end
+
+    def to_s
+      "{/#{@name}.quadhexbytes*}"
+    end
+
+    def expand(vars)
+      hex = vars[@name].to_i.to_s(16).rjust(8,'0')
+      "/#{hex[0..1]}/#{hex[2..3]}/#{hex[4..5]}/#{hex[6..7]}"
+    end
+
+    def expand_code_fragment
+      "/\#{hex=vars[#{@name.inspect}].to_i.to_s(16).rjust(8,'0');hex[0..1]}/\#{hex[2..3]}/\#{hex[4..5]}/\#{hex[6..7]}"
+    end
+
+    def translate_captured_string(string)
+      string.tr('/','').to_i(16)
+    end
+  end
+
+  class String < Doze::URITemplate
+    attr_reader :string
+
+    def initialize(string)
+      @string = string
+    end
+
+    def regexp_fragment
+      Regexp.escape(@string)
+    end
+
+    def to_s
+      @string
+    end
+
+    def expand(vars)
+      @string
+    end
+
+    def partially_expand(vars); self; end
+
+    NO_VARS = [].freeze
+    def variables; NO_VARS; end
+
+    def expand_code_fragment
+      @string.inspect[1...-1]
+    end
+  end
+
+  class Composite < Doze::URITemplate
+    def initialize(parts)
+      @parts = parts
+    end
+
+    def regexp_fragment
+      @parts.map {|p| p.regexp_fragment}.join
+    end
+
+    def to_s
+      @parts.join
+    end
+
+    def expand(vars)
+      @parts.map {|p| p.expand(vars)}.join
+    end
+
+    def partially_expand(vars)
+      Composite.new(@parts.map {|p| p.partially_expand(vars)})
+    end
+
+    def variables
+      @variables ||= @parts.map {|p| p.variables}.flatten
+    end
+
+    def expand_code_fragment
+      @parts.map {|p| p.expand_code_fragment}.join
+    end
+
+    attr_reader :parts
+  end
+
+  # A simple case of Composite where a template is prefixed by a string.
+  # This allows the same compiled URI template to be used with many different prefixes
+  # without having to re-compile the expand method for each of them, or use the slower
+  # default implementation
+  class WithPrefix < Composite
+    def initialize(template, prefix)
+      @template = template
+      @prefix = prefix
+      @parts = [String.new(prefix.to_s), *@template.parts]
+    end
+
+    def expand(vars)
+      "#{@prefix}#{@template.expand(vars)}"
+    end
+  end
+
+  def with_prefix(prefix)
+    WithPrefix.new(self, prefix)
+  end
+end

data/lib/doze/utils.rb

@@ -0,0 +1,53 @@
+# Various stateless utility functions which aid the conversion back and forth between HTTP syntax and the more abstracted ruby representations we use.
+module Doze::Utils
+  # Strictly this is a WebDAV extension but very useful in the wider HTTP context
+  # see http://tools.ietf.org/html/rfc4918#section-11.2
+  Rack::Utils::HTTP_STATUS_CODES[422] = 'Unprocessable entity'
+
+  Rack::Utils::HTTP_STATUS_CODES.each do |code,text|
+    const_set('STATUS_' << text.upcase.gsub(/[^A-Z]+/, '_'), code)
+  end
+
+  URI_SCHEMES = Hash.new(URI::Generic).merge!(
+    'http' => URI::HTTP,
+    'https' => URI::HTTPS
+  )
+
+  def request_base_uri(request)
+    URI_SCHEMES[request.scheme].build(
+      :scheme => request.scheme,
+      :port => request.port,
+      :host => request.host
+    )
+  end
+
+  def quote(str)
+    '"' << str.gsub(/[\\\"]/o, "\\\1") << '"'
+  end
+
+  # Note: unescape and escape proved bottlenecks in URI template matching and URI template generation which in turn
+  # were bottlenecks for serving some simple requests and for generating URIs to use for cache lookups.
+  # So perhaps a bit micro-optimised here, but there was a reason :)
+
+  # Rack::Utils.unescape, but without turning '+' into ' '
+  # Also must be passed a string.
+  def unescape(s)
+    s.gsub(/((?:%[0-9a-fA-F]{2})+)/n) {[$1.delete('%')].pack('H*')}
+  end
+
+  # Rack::Utils.escape, but turning ' ' into '%20' rather than '+' (which is not a necessary part of the URI spec) to save an extra call to tr.
+  # Also must be passed a string.
+  # Also avoids an extra call to 1.8/1.9 compatibility wrapper for bytesize/size.
+  if ''.respond_to?(:bytesize)
+    def escape(s)
+      s.gsub(/([^a-zA-Z0-9_.-]+)/n) {'%'+$1.unpack('H2'*$1.bytesize).join('%').upcase}
+    end
+  else
+    def escape(s)
+      s.gsub(/([^a-zA-Z0-9_.-]+)/n) {'%'+$1.unpack('H2'*$1.size).join('%').upcase}
+    end
+  end
+
+  # So utility functions are accessible as Doze::Utils.foo as well as via including the module
+  extend self
+end

data/lib/doze/version.rb

@@ -0,0 +1,3 @@
+module Doze
+  VERSION = '0.0.11'
+end

data/lib/doze.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'rack'
+require 'uri'
+module Doze; end
+require 'doze/application'

data/test/functional/auth_test.rb

@@ -0,0 +1,69 @@
+require 'functional/base'
+
+class AuthTest < Test::Unit::TestCase
+  include Doze::Utils
+  include Doze::TestCase
+
+  def test_deny_unauthenticated_user
+    root.expects(:authorize).with(nil, :get).returns(false).once
+    assert_equal STATUS_UNAUTHORIZED, get.status
+  end
+
+  def test_deny_authenticated_user
+    root.expects(:authorize).with('username', :get).returns(false).once
+    get('REMOTE_USER' => 'username')
+    assert_equal STATUS_FORBIDDEN, last_response.status
+  end
+
+  def test_allow_authenticated_user
+    root.expects(:authorize).with('username', :get).returns(true).once
+    get('REMOTE_USER' => 'username')
+    assert_equal STATUS_OK, last_response.status
+  end
+
+  def test_post_auth
+    root.expects(:supports_post?).returns(true)
+    root.expects(:authorize).with('username', :post).returns(false).once
+    root.expects(:post).never
+    post('REMOTE_USER' => 'username')
+    assert_equal STATUS_FORBIDDEN, last_response.status
+  end
+
+  def test_put_auth
+    root.expects(:supports_put?).returns(true)
+    root.expects(:authorize).with('username', :put).returns(false).once
+    root.expects(:put).never
+    put('REMOTE_USER' => 'username')
+    assert_equal STATUS_FORBIDDEN, last_response.status
+  end
+
+  def test_delete_auth
+    root.expects(:supports_delete?).returns(true)
+    root.expects(:authorize).with('username', :delete).returns(false).once
+    root.expects(:delete).never
+    delete('REMOTE_USER' => 'username')
+    assert_equal STATUS_FORBIDDEN, last_response.status
+  end
+
+  def test_other_method_auth
+    app(:recognized_methods => [:get, :patch])
+    root.expects(:supports_patch?).returns(true)
+    root.expects(:authorize).with('username', :patch).returns(false).once
+    root.expects(:patch).never
+    other_request_method('PATCH', {'REMOTE_USER' => 'username'})
+    assert_equal STATUS_FORBIDDEN, last_response.status
+  end
+
+  def test_can_raise_unauthd_errors
+    root.expects(:get).raises(Doze::UnauthorizedError.new('no homers allowed'))
+    assert_equal STATUS_UNAUTHORIZED, get.status
+    assert_match /Unauthorized\: no homers allowed/, last_response.body
+  end
+
+  def test_can_raise_forbidden_errors
+    root.expects(:get).raises(Doze::ForbiddenError.new('do not go there, girlfriend'))
+    assert_equal STATUS_FORBIDDEN, get.status
+    assert_match /Forbidden\: do not go there, girlfriend/, last_response.body
+  end
+
+end