rest-core 2.1.2 → 3.0.0

data/Rakefile

@@ -1,41 +1,15 @@
-# encoding: utf-8
 
 begin
   require "#{dir = File.dirname(__FILE__)}/task/gemgem"
 rescue LoadError
-  sh "git submodule update --init"
-  exec Gem.ruby, "-S", "rake", *ARGV
+  sh 'git submodule update --init'
+  exec Gem.ruby, '-S', $PROGRAM_NAME, *ARGV
 end
 
-Gemgem.dir = dir
-($LOAD_PATH << File.expand_path("#{Gemgem.dir}/lib" )).uniq!
-
-desc 'Generate gemspec'
-task 'gem:spec' do
-  Gemgem.spec = Gemgem.create do |s|
-    require 'rest-core/version'
-    s.name     = 'rest-core'
-    s.version  = RestCore::VERSION
-    s.homepage = 'https://github.com/cardinalblue/rest-core'
-
-    %w[rest-client].each{ |g| s.add_runtime_dependency(g) }
-
-    s.authors  = ['Cardinal Blue', 'Lin Jen-Shin (godfat)']
-    s.email    = ['dev (XD) cardinalblue.com']
-
-    s.post_install_message = <<-MARKDOWN
-# [rest-core] Since 2.1.0, Incompatible changes for POST requests:
-
-* We no longer support Rails-like POST payload, like translating
-  `{:foo => [1, 2]}` to `'foo[]=1&foo[]=2'`. It would now be translated to
-  `'foo=1&foo=2'`. If you like `'foo[]'` as the key, simply pass it as
-  `{'foo[]' => [1, 2]}`.
-
-* This also applies to nested hashes like `{:foo => {:bar => 1}`. If you
-  want that behaviour, just pass `{'foo[bar]' => 1}` which would then be
-  translated to `'foo[bar]=1'`.
-MARKDOWN
-  end
-
-  Gemgem.write
+Gemgem.init(dir) do |s|
+  require 'rest-core/version'
+  s.name     = 'rest-core'
+  s.version  = RestCore::VERSION
+  s.homepage = 'https://github.com/godfat/rest-core'
+  %w[httpclient mime-types timers].each{ |g| s.add_runtime_dependency(g) }
 end

data/TODO.md

@@ -1,8 +1,6 @@
 # TODO
 
 * middleware revisit (how to initialize?)
-* streaming?
-* connection pool?
 * X-Method-Override
 
 # BUG
@@ -15,3 +13,6 @@
 * headers and payload logs for CommonLogger
 
 # rest-request
+
+* fix DRY by defining `prepare :: env -> env`
+* FAIL and LOG need to be reimplemented as well.

data/example/simple.rb

@@ -8,21 +8,23 @@ YourClient = RC::Builder.client do
   use RC::Cache       , nil, 3600
 end
 
+YourClient.pool_size = 5
+
 client = YourClient.new(:cache => {})
-p client.get('cardinalblue') # cache miss
+p client.get('godfat') # cache miss
 puts
-p client.get('cardinalblue') # cache hit
+p client.get('godfat') # cache hit
 
 client.cache = false
 
 puts "concurrent requests"
-a = [client.get('cardinalblue'), client.get('godfat')]
+a = [client.get('godfat'), client.get('cardinalblue')]
 puts "It's not blocking... but doing concurrent requests underneath"
 p a.map{ |r| r['name'] } # here we want the values, so it blocks here
 puts "DONE"
 
 puts "callback"
-client.get('cardinalblue'){ |v| p v }
+client.get('godfat'){ |v| p v }
 puts "It's not blocking... but doing concurrent requests underneath"
 client.wait # we block here to wait for the request done
 puts "DONE"

data/example/use-cases.rb

@@ -1,11 +1,17 @@
 
-require 'fiber'
-require 'em-http-request'
 require 'rest-core'
 RC.eagerload
 
+RC::Universal.pool_size = 0 # default to no thread pool
+
 def def_use_case name, &block
-  singleton_class.send(:define_method, "#{name}_", &block)
+  singleton_class.send(:define_method, "#{name}_") do
+    begin
+      yield
+    rescue => e
+      q "Encountering: #{e}"
+    end
+  end
   singleton_class.send(:define_method, name) do
     @count ||= 0
     printf "Use case #%02d: %s\n", @count+=1, name
@@ -30,29 +36,37 @@ end
 
 def_use_case 'pure_ruby_single_request' do
   q RC::Universal.new(:json_response => true).
-    get('https://api.github.com/users/godfat')['name']
+    get('https://graph.facebook.com/4')['name']
 end
 
 def_use_case 'pure_ruby_concurrent_requests' do
   client = RC::Universal.new(:json_response => true,
-                             :site => 'https://api.github.com/users/')
-  q [client.get('godfat'), client.get('cardinalblue')].map{ |u| u['name'] }
+                             :site => 'https://graph.facebook.com/')
+  q [client.get('4'), client.get('5')].map{ |u| u['name'] }
 end
 
 def_use_case 'pure_ruby_cache_requests' do
   client = RC::Universal.new(:json_response => true, :cache => {})
-  3.times{ q client.get('https://api.github.com/users/godfat')['name'] }
+  3.times{ q client.get('https://graph.facebook.com/4')['name'] }
 end
 
 def_use_case 'pure_ruby_callback_requests' do
   m = Mutex.new
   RC::Universal.new(:json_response => true                                  ,
-                    :site          => 'https://api.github.com/users/'       ,
+                    :site          => 'https://graph.facebook.com/'         ,
                     :log_method    => lambda{|str| m.synchronize{puts(str)}}).
-    get('godfat'){ |res|
+    get('4'){ |res|
+      if res.kind_of?(Exception)
+        p "Encountering: #{res}"
+        next
+      end
       q res['name'], m
     }.
-    get('cardinalblue'){ |res|
+    get('5'){ |res|
+      if res.kind_of?(Exception)
+        p "Encountering: #{res}"
+        next
+      end
       q res['name'], m
     }.wait
 end
@@ -60,30 +74,26 @@ end
 def_use_case 'pure_ruby_nested_concurrent_requests' do
   m = Mutex.new
   c = RC::Universal.new(:json_response => true                              ,
-                        :site          => 'https://api.github.com'          ,
+                        :site          => 'https://graph.facebook.com/'     ,
                         :log_method => lambda{|str| m.synchronize{puts(str)}})
 
-  %w[rubytaiwan godfat].each{ |user|
-    c.get("/users/#{user}/repos", :per_page => 100){ |repos|
-      rs = repos.reject{ |r| r['fork'] }
-      rs = [{}] if rs.size == 1 # out of API limit :(
-      most_watched = rs.max_by{ |r| r['watchers'] }['name']
-      most_size    = rs.max_by{ |r| r['size']     }['name']
-
-      watch_contri = c.get("/repos/#{user}/#{most_watched}/contributors")
-       size_contri = c.get("/repos/#{user}/#{most_size}/contributors")
+  %w[4 5].each{ |user|
+    c.get(user, :fields => 'cover'){ |data|
+      if data.kind_of?(Exception)
+        q "Encountering: #{data}", m
+        next
+      end
 
-      watch_contri = [{}] if watch_contri.size == 1 # out of API limit :(
-       size_contri = [{}] if  size_contri.size == 1 # out of API limit :(
+      cover    = data['cover']
+      comments = c.get("#{cover['id']}/comments")
+      likes    = c.get("#{cover['id']}/likes")
+      most_liked_comment = comments['data'].max_by{|d|d['like_count']}
 
-      most_watched_most_contri = watch_contri.max_by{ |c| c['contributions'] }
-      most_size_most_contri    =  size_contri.max_by{ |c| c['contributions'] }
+      q "Author of most liked comment from #{user}'s cover photo:", m
+      q most_liked_comment['from']['name'], m
 
-      q "Most contributed user for most watched: #{user}/#{most_watched}:", m
-      q most_watched_most_contri['login'], m
-
-      q "Most contributed user for most size   : #{user}/#{most_size}:", m
-      q most_size_most_contri['login'], m
+      y = !!likes['data'].find{|d|d['id'] == most_liked_comment['from']['id']}
+      q "Did the user also like the cover?: #{y}", m
     }
   }
 
@@ -92,72 +102,6 @@ end
 
 # ----------------------------------------------------------------------
 
-def_use_case 'eventmachine_fiber_single_request'             do
-  EM.run{  Fiber.new{ pure_ruby_single_request_            ; EM.stop }.resume}
-end
-
-def_use_case 'eventmachine_fiber_concurrent_requests'        do
-  EM.run{  Fiber.new{ pure_ruby_concurrent_requests_       ; EM.stop }.resume}
-end
-
-def_use_case 'eventmachine_fiber_cache_requests'             do
-  EM.run{  Fiber.new{ pure_ruby_cache_requests_            ; EM.stop }.resume}
-end
-
-def_use_case 'eventmachine_fiber_callback_requests'          do
-  EM.run{  Fiber.new{ pure_ruby_callback_requests_         ; EM.stop }.resume}
-end
-
-def_use_case 'eventmachine_fiber_nested_concurrent_requests' do
-  EM.run{  Fiber.new{ pure_ruby_nested_concurrent_requests_; EM.stop }.resume}
-end
-
-# ----------------------------------------------------------------------
-
-def_use_case 'eventmachine_thread_single_request'             do
-  EM.run{ Thread.new{ pure_ruby_single_request_            ; EM.stop } }
-end
-
-def_use_case 'eventmachine_thread_concurrent_requests'        do
-  EM.run{ Thread.new{ pure_ruby_concurrent_requests_       ; EM.stop } }
-end
-
-def_use_case 'eventmachine_thread_cache_requests'             do
-  EM.run{ Thread.new{ pure_ruby_cache_requests_            ; EM.stop } }
-end
-
-def_use_case 'eventmachine_thread_callback_requests'          do
-  EM.run{ Thread.new{ pure_ruby_callback_requests_         ; EM.stop } }
-end
-
-def_use_case 'eventmachine_thread_nested_concurrent_requests' do
-  EM.run{ Thread.new{ pure_ruby_nested_concurrent_requests_; EM.stop } }
-end
-
-# ----------------------------------------------------------------------
-
-def_use_case 'eventmachine_rest_client_single_request'        do
-  EM.run{             pure_ruby_single_request_            ; EM.stop   }
-end
-
-def_use_case 'eventmachine_rest_client_concurrent_requests'   do
-  EM.run{             pure_ruby_concurrent_requests_       ; EM.stop   }
-end
-
-def_use_case 'eventmachine_rest_client_cache_requests'        do
-  EM.run{             pure_ruby_cache_requests_            ; EM.stop   }
-end
-
-def_use_case 'eventmachine_rest_client_callback_requests'     do
-  EM.run{             pure_ruby_callback_requests_         ; EM.stop   }
-end
-
-def_use_case 'eventmachine_rest_client_nested_concurrent_requests' do
-  EM.run{             pure_ruby_nested_concurrent_requests_; EM.stop   }
-end
-
-# ----------------------------------------------------------------------
-
 def_use_case 'pure_ruby' do
   pure_ruby_single_request
   pure_ruby_concurrent_requests
@@ -166,33 +110,6 @@ def_use_case 'pure_ruby' do
   pure_ruby_nested_concurrent_requests
 end
 
-def_use_case 'eventmachine_fiber' do
-  eventmachine_fiber_single_request
-  eventmachine_fiber_concurrent_requests
-  eventmachine_fiber_cache_requests
-  eventmachine_fiber_callback_requests
-  eventmachine_fiber_nested_concurrent_requests
-end
-
-def_use_case 'eventmachine_thread' do
-  eventmachine_thread_single_request
-  eventmachine_thread_concurrent_requests
-  eventmachine_thread_cache_requests
-  eventmachine_thread_callback_requests
-  eventmachine_thread_nested_concurrent_requests
-end
-
-def_use_case 'eventmachine_rest_client' do
-  eventmachine_rest_client_single_request
-  eventmachine_rest_client_concurrent_requests
-  eventmachine_rest_client_cache_requests
-  eventmachine_rest_client_callback_requests
-  eventmachine_rest_client_nested_concurrent_requests
-end
-
 # ----------------------------------------------------------------------
 
 pure_ruby
-eventmachine_fiber
-eventmachine_thread
-eventmachine_rest_client

data/lib/rest-core.rb

@@ -9,16 +9,17 @@ module RestCore
   RESPONSE_BODY    = 'RESPONSE_BODY'
   RESPONSE_STATUS  = 'RESPONSE_STATUS'
   RESPONSE_HEADERS = 'RESPONSE_HEADERS'
+  RESPONSE_SOCKET  = 'RESPONSE_SOCKET'
 
   DRY              = 'core.dry'
   FAIL             = 'core.fail'
   LOG              = 'core.log'
+  CLIENT           = 'core.client'
 
   ASYNC            = 'async.callback'
   TIMER            = 'async.timer'
-  FUTURE           = 'async.future'
-
-  RootFiber        = Fiber.respond_to?(:current) && Fiber.current
+  PROMISE          = 'async.promise'
+  HIJACK           = 'async.hijack'
 
   # core utilities
   autoload :Builder       , 'rest-core/builder'
@@ -27,6 +28,9 @@ module RestCore
   autoload :Event         , 'rest-core/event'
   autoload :Middleware    , 'rest-core/middleware'
   autoload :Wrapper       , 'rest-core/wrapper'
+  autoload :Promise       , 'rest-core/promise'
+  autoload :ThreadPool    , 'rest-core/thread_pool'
+  autoload :EventSource   , 'rest-core/event_source'
 
   # oauth1 utilities
   autoload :ClientOauth1  , 'rest-core/client_oauth1'
@@ -59,10 +63,10 @@ module RestCore
   autoload :Timeout       , 'rest-core/middleware/timeout'
 
   # engines
-  autoload :Auto          , 'rest-core/engine/auto'
   autoload :Dry           , 'rest-core/engine/dry'
+  autoload :HttpClient    , 'rest-core/engine/http-client'
   autoload :RestClient    , 'rest-core/engine/rest-client'
-  autoload :EmHttpRequest , 'rest-core/engine/em-http-request'
+  autoload :NetHttpPersistent, 'rest-core/engine/net-http-persistent'
 
   # clients
   autoload :Simple        , 'rest-core/client/simple'
@@ -83,6 +87,11 @@ module RestCore
       eagerload(c, loaded) if c.respond_to?(:constants) && !loaded[n]
     }
   end
+
+  # identity function
+  def self.id
+    @id ||= lambda{ |a| a }
+  end
 end
 
 RC = RestCore unless Object.const_defined?(:RC)

data/lib/rest-core/builder.rb

@@ -7,7 +7,7 @@ class RestCore::Builder
   include Wrapper
 
   def self.default_engine
-    @default_engine ||= RestCore::Auto
+    @default_engine ||= RestCore::HttpClient
   end
 
   def self.client *attrs, &block
@@ -24,12 +24,22 @@ class RestCore::Builder
     client = Class.new(struct)
     client.const_set('Struct', struct)
     client.send(:include, Client)
-    class << client; attr_reader :builder; end
+    class << client
+      attr_reader   :builder
+      attr_accessor :pool_size, :pool_idle_time
+
+      def thread_pool
+        RestCore::ThreadPool[self]
+      end
+    end
     client.instance_variable_set(:@builder, self)
+    client.instance_variable_set(:@pool_size, 0) # default to no pool
+    client.instance_variable_set(:@pool_idle_time, 60) # default to 60 seconds
     client
   end
 
   def initialize &block
+    @engine    = nil
     @middles ||= []
     instance_eval(&block) if block_given?
   end

data/lib/rest-core/client.rb

@@ -1,7 +1,6 @@
 
-require 'rest-core'
-
 require 'weakref'
+require 'rest-core'
 
 module RestCore::Client
   include RestCore
@@ -43,12 +42,13 @@ module RestCore::Client
     mod.send(:include, accessor)
   end
 
-  attr_reader :app, :dry, :futures
+  attr_reader :app, :dry, :promises
   def initialize o={}
     @app ||= self.class.builder.to_app # lighten! would reinitialize anyway
     @dry ||= self.class.builder.to_app(Dry)
-    @futures = []  # don't record any futures in lighten!
-    @mutex   = nil # for locking futures, lazily initialized for serialization
+    @promises = []  # don't record any promises in lighten!
+    @mutex    = nil # for locking promises, lazily initialized
+                    # for serialization
     o.each{ |key, value| send("#{key}=", value) if respond_to?("#{key}=") }
   end
 
@@ -85,13 +85,13 @@ module RestCore::Client
   end
 
   def wait
-    return self if futures.empty?
-    current_futures = nil
+    return self if promises.empty?
+    current_promises = nil
     mutex.synchronize{
-      current_futures = futures.dup
-      futures.clear
+      current_promises = promises.dup
+      promises.clear
     }
-    current_futures.each{ |f|
+    current_promises.each{ |f|
       begin
         f.wait
       rescue WeakRef::RefError # it's gc'ed after we think it's alive
@@ -111,14 +111,14 @@ module RestCore::Client
     request(
       {REQUEST_METHOD  => :get   ,
        REQUEST_PATH    => path   ,
-       REQUEST_QUERY   => query  }.merge(opts), &cb)
+       REQUEST_QUERY   => query  }.merge(opts), response_key(opts), &cb)
   end
 
   def delete path, query={}, opts={}, &cb
     request(
       {REQUEST_METHOD  => :delete,
        REQUEST_PATH    => path   ,
-       REQUEST_QUERY   => query  }.merge(opts), &cb)
+       REQUEST_QUERY   => query  }.merge(opts), response_key(opts), &cb)
   end
 
   def head path, query={}, opts={}, &cb
@@ -132,7 +132,7 @@ module RestCore::Client
     request(
       {REQUEST_METHOD  => :options,
        REQUEST_PATH    => path   ,
-       REQUEST_QUERY   => query  }.merge(opts), &cb)
+       REQUEST_QUERY   => query  }.merge(opts), RESPONSE_HEADERS, &cb)
   end
 
   def post   path, payload={}, query={}, opts={}, &cb
@@ -140,7 +140,7 @@ module RestCore::Client
       {REQUEST_METHOD  => :post  ,
        REQUEST_PATH    => path   ,
        REQUEST_QUERY   => query  ,
-       REQUEST_PAYLOAD => payload}.merge(opts), &cb)
+       REQUEST_PAYLOAD => payload}.merge(opts), response_key(opts), &cb)
   end
 
   def put    path, payload={}, query={}, opts={}, &cb
@@ -148,7 +148,7 @@ module RestCore::Client
       {REQUEST_METHOD  => :put   ,
        REQUEST_PATH    => path   ,
        REQUEST_QUERY   => query  ,
-       REQUEST_PAYLOAD => payload}.merge(opts), &cb)
+       REQUEST_PAYLOAD => payload}.merge(opts), response_key(opts), &cb)
   end
 
   def patch  path, payload={}, query={}, opts={}, &cb
@@ -156,14 +156,16 @@ module RestCore::Client
       {REQUEST_METHOD  => :patch ,
        REQUEST_PATH    => path   ,
        REQUEST_QUERY   => query  ,
-       REQUEST_PAYLOAD => payload}.merge(opts), &cb)
+       REQUEST_PAYLOAD => payload}.merge(opts), response_key(opts), &cb)
+  end
+
+  def event_source path, query={}, opts={}
+    EventSource.new(self, path, query, opts)
   end
 
   def request env, key=RESPONSE_BODY, app=app
     if block_given?
-      request_full(env, app){ |response|
-        yield(response[key])
-      }
+      request_full(env, app){ |response| yield(response[key]) }
     else
       request_full(env, app)[key]
     end
@@ -171,14 +173,13 @@ module RestCore::Client
 
   def request_full env, app=app, &k
     response = app.call(build_env({ASYNC => !!k}.merge(env)),
-                        &(k || Middleware.id))
+                        &(k || RC.id))
 
     # under ASYNC callback, response might not be a response hash
     # in that case (maybe in a user created engine), Client#wait
-    # won't work because we have no way to track the future.
-    if response.kind_of?(Hash) && RestCore.const_defined?(:Future) &&
-       response[FUTURE].kind_of?(Future)
-      mutex.synchronize{ futures << WeakRef.new(response[FUTURE]) }
+    # won't work because we have no way to track the promise.
+    if response.kind_of?(Hash) && response[PROMISE].kind_of?(Promise)
+      mutex.synchronize{ promises << WeakRef.new(response[PROMISE]) }
     end
 
     if block_given?
@@ -200,7 +201,8 @@ module RestCore::Client
      REQUEST_PAYLOAD => {}  ,
      REQUEST_HEADERS => {}  ,
      FAIL            => []  ,
-     LOG             => []  }
+     LOG             => []  ,
+     CLIENT          => self}
   end
   # ------------------------ instance ---------------------
 
@@ -211,6 +213,10 @@ module RestCore::Client
     @mutex ||= Mutex.new
   end
 
+  def response_key opts
+    if opts[HIJACK] then RESPONSE_SOCKET else RESPONSE_BODY end
+  end
+
   def lighten_hash hash
     Hash[hash.map{ |(key, value)|
       case value