rest-core 2.0.4 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 13edab18f56e62ce573dc6cff597226d8d5ade6c
-  data.tar.gz: e41513b602d15644f9f4000cfc85a14bd93b71f2
+  metadata.gz: b126115b38499438877d2c55fe68b14c3d9989bd
+  data.tar.gz: 6ba8f157a2025d94fe326767dc36823d11585854
 SHA512:
-  metadata.gz: 70583190f10ad4f2a90075eb894a9f12da622c585353b20a77152de34ea573f5cc1d5c9512ca9d6b19722c13baba87bc829a10a9eb77162bf95c94bdc88b1cc7
-  data.tar.gz: c583d03d2448e7d202a506661aea8a67be6f127f1b2ec3bf97176939585437d78fc28c5bb708f4f9fdcf2501d9e2806f2eb6a75f102949f32f5d238f34665826
+  metadata.gz: 14b1c0022f58a2af1a26bb0126159937b0540ee462bb6fe96db3fa777e0ba7cd14183e6ff8e051e45acc65c02708a69f51b31ad3371d0792e4124c2811029720
+  data.tar.gz: 4a3e037e2b30b77c57e7b030c05322389c1a585766ba414d137ce790c8c3e0b1c141c281a1dd664d07e47150a1f9f2a963059e42ffdad91d63549afd07ce095f

data/CHANGES.md

@@ -1,5 +1,38 @@
 # CHANGES
 
+## rest-core 2.1.0 -- 2013-05-08
+
+### Incompatible changes
+
+* We no longer support Rails-like POST payload, like translating
+  `{:foo => [1, 2]}` to `'foo[]=1&foo[]=2'`. It would now be translated to
+  `'foo=1&foo=2'`. If you like `'foo[]'` as the key, simply pass it as
+  `{'foo[]' => [1, 2]}`.
+
+* This also applies to nested hashes like `{:foo => {:bar => 1}`. If you
+  want that behaviour, just pass `{'foo[bar]' => 1}` which would then be
+  translated to `'foo[bar]=1'`.
+
+### Bugs fixes
+
+* [`Payload`] Now we could correctly support payload with "foo=1&foo=2".
+* [`Client`] Fix inspect spacing.
+
+### Enhancement
+
+* [`Payload`] With this class introduced, replacing rest-client's own
+  payload implementation, we could pass StringIO or other sockets as the
+  payload body. This would also fix the issue that using the same key for
+  different values as allowed in the spec.
+* [`EmHttpRequest`] Send payload as a file directly if it's a file. Buffer
+  the payload into a tempfile if it's from a socket or a large StringIO.
+  This should greatly reduce the memory usage as we don't build large
+  Ruby strings in the memory. Streaming is not yet supported though.
+* [`Client`] Make inspect shorter.
+* [`Client`] Introduce Client#default_env
+* [`Middleware`] Introduce Middleware.percent_encode.
+* [`Middleware`] Introduce Middleware.contain_binary?.
+
 ## rest-core 2.0.4 -- 2013-04-30
 
 * [`EmHttpRequest`] Use `EM.schedule` to fix thread-safety issue.

data/Gemfile

@@ -1,5 +1,5 @@
 
-source 'http://rubygems.org'
+source 'https://rubygems.org/'
 
 gemspec
 

data/README.md

@@ -18,7 +18,7 @@ talk is in Mandarin.
 
 ## DESCRIPTION:
 
-Modular Ruby clients interface for REST APIs
+Modular Ruby clients interface for REST APIs.
 
 There has been an explosion in the number of REST APIs available today.
 To address the need for a way to access these APIs easily and elegantly,

data/Rakefile

@@ -22,6 +22,19 @@ task 'gem:spec' do
 
     s.authors  = ['Cardinal Blue', 'Lin Jen-Shin (godfat)']
     s.email    = ['dev (XD) cardinalblue.com']
+
+    s.post_install_message = <<-MARKDOWN
+# [rest-core] Incompatible changes for POST requests:
+
+* We no longer support Rails-like POST payload, like translating
+  `{:foo => [1, 2]}` to `'foo[]=1&foo[]=2'`. It would now be translated to
+  `'foo=1&foo=2'`. If you like `'foo[]'` as the key, simply pass it as
+  `{'foo[]' => [1, 2]}`.
+
+* This also applies to nested hashes like `{:foo => {:bar => 1}`. If you
+  want that behaviour, just pass `{'foo[bar]' => 1}` which would then be
+  translated to `'foo[bar]=1'`.
+MARKDOWN
   end
 
   Gemgem.write

data/lib/rest-core.rb

@@ -33,7 +33,9 @@ module RestCore
 
   # misc utilities
   autoload :Hmac          , 'rest-core/util/hmac'
+  autoload :Json          , 'rest-core/util/json'
   autoload :ParseQuery    , 'rest-core/util/parse_query'
+  autoload :Payload       , 'rest-core/util/payload'
 
   # middlewares
   autoload :AuthBasic     , 'rest-core/middleware/auth_basic'

data/lib/rest-core/client.rb

@@ -64,8 +64,14 @@ module RestCore::Client
   end
 
   def inspect
-    fields = attributes.map{ |k, v| "#{k}=#{v.inspect}" }.join(', ')
-    "#<struct #{self.class.name}#{if fields.empty? then '' else fields end}>"
+    fields = if size > 0
+               ' ' + attributes.map{ |k, v|
+                 "#{k}=#{v.inspect.sub(/(?<=.{12}).{4,}/, '...')}"
+               }.join(', ')
+             else
+               ''
+             end
+    "#<struct #{self.class.name}#{fields}>"
   end
 
   def lighten! o={}
@@ -171,16 +177,8 @@ module RestCore::Client
   end
 
   def request_full env, app=app, &k
-    response = app.call(build_env(
-      {REQUEST_METHOD  => :get,
-       REQUEST_PATH    => '/' ,
-       REQUEST_QUERY   => {}  ,
-       REQUEST_PAYLOAD => {}  ,
-       REQUEST_HEADERS => {}  ,
-       FAIL            => []  ,
-       LOG             => []  ,
-       ASYNC           => !!k }.merge(env)),
-       &(k || Middleware.id))
+    response = app.call(build_env({ASYNC => !!k}.merge(env)),
+                        &(k || Middleware.id))
 
     # under ASYNC callback, response might not be a response hash
     # in that case (maybe in a user created engine), Client#wait
@@ -198,7 +196,18 @@ module RestCore::Client
   end
 
   def build_env env={}
-    Middleware.string_keys(attributes).merge(Middleware.string_keys(env))
+    default_env.merge(
+      Middleware.string_keys(attributes).merge(Middleware.string_keys(env)))
+  end
+
+  def default_env
+    {REQUEST_METHOD  => :get,
+     REQUEST_PATH    => '/' ,
+     REQUEST_QUERY   => {}  ,
+     REQUEST_PAYLOAD => {}  ,
+     REQUEST_HEADERS => {}  ,
+     FAIL            => []  ,
+     LOG             => []  }
   end
   # ------------------------ instance ---------------------
 

data/lib/rest-core/engine/em-http-request.rb

@@ -20,32 +20,61 @@ class RestCore::EmHttpRequest
               FUTURE           => future)
   end
 
-  def close client
+  def close client, tmpfile
     (client.instance_variable_get(:@callbacks)||[]).clear
     (client.instance_variable_get(:@errbacks )||[]).clear
     client.close
+    if tmpfile.respond_to?(:close!)   # tempfile
+      tmpfile.close!
+    elsif tmpfile.respond_to?(:close) # regular IO
+      tmpfile.close
+    end
   end
 
   def request future, env
-    payload = ::RestClient::Payload.generate(env[REQUEST_PAYLOAD])
+    payload = Payload.generate(env[REQUEST_PAYLOAD])
+    tmpfile = payload2file(payload)
+    args    = if tmpfile.respond_to?(:path)
+                {:file => tmpfile.path}
+              else
+                {:body => tmpfile}
+              end
     client  = ::EventMachine::HttpRequest.new(request_uri(env)).send(
-                 env[REQUEST_METHOD],
-                 :body => payload && payload.read,
-                 :head => payload && payload.headers.
-                                               merge(env[REQUEST_HEADERS]))
+                 env[REQUEST_METHOD], args.merge(
+                   :head => payload.headers.merge(env[REQUEST_HEADERS])))
 
     client.callback{
+      close(client, tmpfile)
       future.on_load(client.response,
                      client.response_header.status,
                      client.response_header)}
 
     client.errback{
-      close(client)
+      close(client, tmpfile)
       future.on_error(client.error)}
 
     env[TIMER].on_timeout{
-      close(client)
+      close(client, tmpfile)
       future.on_error(env[TIMER].error)
     } if env[TIMER]
   end
+
+  def payload2file payload
+    if payload.io.respond_to?(:path) # already a file
+      payload.io
+
+    elsif payload.size == 0 ||       # probably a socket, buffer to disc
+          payload.size >= 81920      # probably too large, buffer to disc
+      create_tmpfile(payload.io)
+
+    else                             # probably not worth buffering to disc
+      payload.read
+    end
+  end
+
+  def create_tmpfile io
+    tempfile = Tempfile.new("rest-core.em-http-request.#{rand(1_000_000)}")
+    IO.copy_stream(io, tempfile)
+    tempfile
+  end
 end

data/lib/rest-core/engine/rest-client.rb

@@ -12,10 +12,12 @@ class RestCore::RestClient
 
     t = future.wrap{ # we can implement thread pool in the future
       begin
-        res = ::RestClient::Request.execute(:method  => env[REQUEST_METHOD ],
-                                            :url     => request_uri(env)    ,
-                                            :payload => env[REQUEST_PAYLOAD],
-                                            :headers => env[REQUEST_HEADERS],
+        payload = Payload.generate(env[REQUEST_PAYLOAD])
+        headers = env[REQUEST_HEADERS].merge(payload.headers)
+        res = ::RestClient::Request.execute(:method  => env[REQUEST_METHOD],
+                                            :url     => request_uri(env)   ,
+                                            :payload => payload            ,
+                                            :headers => headers            ,
                                             :max_redirects => 0)
         future.on_load(res.body, res.code, normalize_headers(res.raw_headers))
 

data/lib/rest-core/middleware.rb

@@ -58,19 +58,40 @@ module RestCore::Middleware
       env[REQUEST_PATH].to_s
     else
       q = if env[REQUEST_PATH] =~ /\?/ then '&' else '?' end
-      "#{env[REQUEST_PATH]}#{q}" \
-      "#{query.sort.map{ |(k, v)|
-        "#{escape(k.to_s)}=#{escape(v.to_s)}" }.join('&')}"
+      "#{env[REQUEST_PATH]}#{q}#{percent_encode(query)}"
     end
   end
   public :request_uri
 
+  def percent_encode query
+    query.sort.map{ |(k, v)|
+      if v.kind_of?(Array)
+        v.map{ |vv| "#{escape(k.to_s)}=#{escape(vv.to_s)}" }.join('&')
+      else
+        "#{escape(k.to_s)}=#{escape(v.to_s)}"
+      end
+    }.join('&')
+  end
+  public :percent_encode
+
   UNRESERVED = /[^a-zA-Z0-9\-\.\_\~]/
   def escape string
     URI.escape(string, UNRESERVED)
   end
   public :escape
 
+  def contain_binary? payload
+    return false unless payload
+    return true  if     payload.respond_to?(:read)
+    return true  if     payload.find{ |k, v|
+      # if payload is an array, then v would be nil
+      (v || k).respond_to?(:read) ||
+      # if v is an array, it could contain binary data
+      (v.kind_of?(Array) && v.any?{ |vv| vv.respond_to?(:read) }) }
+    return false
+  end
+  public :contain_binary?
+
   def string_keys hash
     hash.inject({}){ |r, (k, v)|
       if v.kind_of?(Hash)

data/lib/rest-core/middleware/auth_basic.rb

@@ -9,7 +9,7 @@ class RestCore::AuthBasic
     if username(env)
       if password(env)
         app.call(env.merge(REQUEST_HEADERS =>
-          auth_basic_header(env).merge(env[REQUEST_HEADERS] || {})), &k)
+          auth_basic_header(env).merge(env[REQUEST_HEADERS])), &k)
       else
         app.call(log(env, "AuthBasic: username provided: #{username(env)}," \
                           " but password is missing."), &k)

data/lib/rest-core/middleware/cache.rb

@@ -106,10 +106,10 @@ class RestCore::Cache
   end
 
   def cache_for? env
-    [:get, :head, :otpions].include?(env[REQUEST_METHOD])
+    [:get, :head, :otpions].include?(env[REQUEST_METHOD]) && !env[DRY]
   end
 
   def header_cache_key env
-    (env[REQUEST_HEADERS]||{}).sort.map{|(k,v)|"#{k}=#{v}"}.join('&')
+    env[REQUEST_HEADERS].sort.map{|(k,v)|"#{k}=#{v}"}.join('&')
   end
 end

data/lib/rest-core/middleware/default_headers.rb

@@ -6,6 +6,6 @@ class RestCore::DefaultHeaders
   include RestCore::Middleware
   def call env, &k
     app.call(env.merge(REQUEST_HEADERS =>
-      @headers.merge(headers(env)).merge(env[REQUEST_HEADERS] || {})), &k)
+      @headers.merge(headers(env)).merge(env[REQUEST_HEADERS])), &k)
   end
 end

data/lib/rest-core/middleware/default_payload.rb

@@ -14,7 +14,7 @@ class RestCore::DefaultPayload
     defaults = merge(@payload, payload(env))
 
     app.call(env.merge(REQUEST_PAYLOAD =>
-      merge(defaults, env[REQUEST_PAYLOAD] || {})), &k)
+      merge(defaults, env[REQUEST_PAYLOAD])), &k)
   end
 
   # this method is intended to merge payloads if they are non-empty hashes,

data/lib/rest-core/middleware/default_query.rb

@@ -14,6 +14,6 @@ class RestCore::DefaultQuery
     defaults = string_keys(@query).merge(string_keys(query(env)))
 
     app.call(env.merge(REQUEST_QUERY =>
-      defaults.merge(env[REQUEST_QUERY] || {})), &k)
+      defaults.merge(env[REQUEST_QUERY])), &k)
   end
 end

data/lib/rest-core/middleware/oauth1_header.rb

@@ -16,7 +16,7 @@ class RestCore::Oauth1Header
   def call env, &k
     start_time = Time.now
     headers = {'Authorization' => oauth_header(env)}.
-                merge(env[REQUEST_HEADERS] || {})
+                merge(env[REQUEST_HEADERS])
 
     event = Event::WithHeader.new(Time.now - start_time,
               "Authorization: #{headers['Authorization']}")
@@ -52,7 +52,7 @@ class RestCore::Oauth1Header
     method   = env[REQUEST_METHOD].to_s.upcase
     base_uri = env[REQUEST_PATH]
     payload  = payload_params(env)
-    query    = reject_blank(env[REQUEST_QUERY] || {})
+    query    = reject_blank(env[REQUEST_QUERY])
     params   = reject_blank(oauth_params.merge(query.merge(payload))).
       to_a.sort.map{ |(k, v)|
         "#{escape(k.to_s)}=#{escape(v.to_s)}"}.join('&')
@@ -85,20 +85,10 @@ class RestCore::Oauth1Header
 
     # so the Content-Type header must be application/x-www-form-urlencoded
     else
-      reject_blank(env[REQUEST_PAYLOAD] || {})
+      reject_blank(env[REQUEST_PAYLOAD])
     end
   end
 
-  def contain_binary? payload
-    return false unless payload
-    return true  if     payload.kind_of?(IO)    ||
-                        payload.respond_to?(:read)
-    return true  if     payload.find{ |k, v|
-      # if payload is an array, then v would be nil
-      (v || k).kind_of?(IO) || (v || k).respond_to?(:read) }
-    return false
-  end
-
   def reject_blank params
     params.reject{ |k, v| v.nil? || v == false ||
                                     (v.respond_to?(:strip) &&

data/lib/rest-core/middleware/oauth2_header.rb

@@ -9,7 +9,7 @@ class RestCore::Oauth2Header
     start_time = Time.now
     headers = {'Authorization' =>
                  "#{access_token_type(env)} #{access_token(env)}"}.
-                merge(env[REQUEST_HEADERS] || {}) if access_token(env)
+                merge(env[REQUEST_HEADERS]) if access_token(env)
 
     event = Event::WithHeader.new(Time.now - start_time,
               "Authorization: #{headers['Authorization']}") if headers

data/lib/rest-core/middleware/oauth2_query.rb

@@ -9,7 +9,7 @@ class RestCore::Oauth2Query
     local = if access_token(env)
               env.merge(REQUEST_QUERY =>
                           {'access_token' => access_token(env)}.
-                           merge(env[REQUEST_QUERY] || {}))
+                           merge(env[REQUEST_QUERY]))
             else
               env
             end

data/lib/rest-core/test.rb

@@ -23,4 +23,52 @@ module Kernel
   def lt? rhs
     self < rhs
   end
+
+  def with_img
+    f = Tempfile.new(['img', '.jpg'])
+    n = File.basename(f.path)
+    f.write('a'*10)
+    f.rewind
+    yield(f, n)
+  ensure
+    f.close!
+  end
+end
+
+# https://github.com/bblimke/webmock/pull/280
+class ::EventMachine::WebMockHttpClient
+  def build_request_signature
+    headers, body = @req.headers, @req.body
+
+    @conn.middleware.select {|m| m.respond_to?(:request) }.each do |m|
+      headers, body = m.request(self, headers, body)
+    end
+
+    method = @req.method
+    uri = @req.uri.clone
+    auth = @req.headers[:'proxy-authorization']
+    query = @req.query
+
+    if auth
+      userinfo = auth.join(':')
+      userinfo = WebMock::Util::URI.encode_unsafe_chars_in_userinfo(userinfo)
+      if @req
+        @req.proxy.reject! {|k,v| t.to_s == 'authorization' }
+      else
+        options.reject! {|k,v| k.to_s == 'authorization' } #we added it to url userinfo
+      end
+      uri.userinfo = userinfo
+    end
+
+    uri.query = encode_query(@req.uri, query).slice(/\?(.*)/, 1)
+
+    body = form_encode_body(body) if body.is_a?(Hash)
+
+    WebMock::RequestSignature.new(
+      method.downcase.to_sym,
+      uri.to_s,
+      :body => body || (@req.file && File.read(@req.file)),
+      :headers => headers
+    )
+  end
 end