rest-core 1.0.3 → 2.0.0

data/TODO.md

@@ -1,8 +1,9 @@
 # TODO
 
-* HTTP method in Requested log
 * middleware revisit (how to initialize?)
-* test utility
+* streaming?
+* connection pool?
+* X-Method-Override
 
 # BUG
 
@@ -11,10 +12,6 @@
 # FEATURE
 
 * middleware composer
-* RC::Payload which can handle Content-Type: application/json
-* how to pass a arbitrary payload generator?
-* RC::JsonRequest
-* RC::FakeGetWithPayload
+* headers and payload logs for CommonLogger
 
-* ResponseThunk and EmHttpRequestThunk
-* options for apps
+# rest-request

data/example/multi.rb

@@ -1,35 +1,42 @@
 
+require 'fiber'
+require 'em-http-request'
 require 'rest-core'
-require 'eventmachine'
-RestCore::EmHttpRequest # there might be a autoload bug?
-                        # omitting this line would cause
-                        # stack level too deep (SystemStackError)
 
-YourClient = RestCore::Builder.client do
-  s = RestCore
-  use s::DefaultSite , 'https://api.github.com/users/'
-  use s::JsonDecode  , true
-  use s::CommonLogger, method(:puts)
-  use s::Cache       , nil, 3600
-  run s::Auto
+YourClient = RC::Builder.client do
+  use RC::DefaultSite , 'https://api.github.com/users/'
+  use RC::JsonResponse, true
+  use RC::CommonLogger, method(:puts)
+  use RC::Cache       , nil, 3600
 end
 
 client = YourClient.new
+puts "rest-client with threads doing concurrent requests"
+a = [client.get('cardinalblue')['name'], client.get('godfat')['name']]
+puts "It's not blocking... but doing concurrent requests underneath"
+p a # here we want the values, so it blocks here
+puts "DONE"
+
+puts; puts
+
+puts "eventmachine with threads doing concurrent requests"
+EM.run{
+  Thread.new{
+    p [client.get('cardinalblue')['name'], client.get('godfat')['name']]
+    puts "DONE"
+    EM.stop
+  }
+  puts "It's not blocking... but doing concurrent requests underneath"
+}
+
+puts; puts
+
+puts "eventmachine with fibers doing concurrent requests"
 EM.run{
   Fiber.new{
-    fiber = Fiber.current
-    result = {}
-    client.get('cardinalblue'){ |response|
-      result[0] = response
-      fiber.resume(result) if result.size == 2
-    }
-    puts "It's not blocking..."
-    client.get('cardinalblue'){ |response|
-      result[1] = response
-      fiber.resume(result) if result.size == 2
-    }
-    p Fiber.yield
+    p [client.get('cardinalblue')['name'], client.get('godfat')['name']]
+    puts "DONE"
     EM.stop
   }.resume
-  puts "It's not blocking..."
+  puts "It's not blocking... but doing concurrent requests underneath"
 }

data/example/simple.rb

@@ -0,0 +1,28 @@
+
+require 'rest-core'
+
+YourClient = RC::Builder.client do
+  use RC::DefaultSite , 'https://api.github.com/users/'
+  use RC::JsonResponse, true
+  use RC::CommonLogger, method(:puts)
+  use RC::Cache       , nil, 3600
+end
+
+client = YourClient.new(:cache => {})
+p client.get('cardinalblue') # cache miss
+puts
+p client.get('cardinalblue') # cache hit
+
+client.cache = false
+
+puts "concurrent requests"
+a = [client.get('cardinalblue')['name'], client.get('godfat')['name']]
+puts "It's not blocking... but doing concurrent requests underneath"
+p a # here we want the values, so it blocks here
+puts "DONE"
+
+puts "callback"
+client.get('cardinalblue'){ |v| p v }
+puts "It's not blocking... but doing concurrent requests underneath"
+client.wait # we block here to wait for the request done
+puts "DONE"

data/example/use-cases.rb

@@ -0,0 +1,194 @@
+
+require 'fiber'
+require 'em-http-request'
+require 'rest-core'
+RC.eagerload
+
+def def_use_case name, &block
+  singleton_class.send(:define_method, "#{name}_", &block)
+  singleton_class.send(:define_method, name) do
+    @count ||= 0
+    printf "Use case #%02d: %s\n", @count+=1, name
+    puts '-' * 70
+    start = Time.now
+    send("#{name}_")
+    puts "Spent #{Time.now - start} seconds for this use case."
+    puts
+  end
+end
+
+def q str, m=nil
+  p = lambda{ puts "\e[33m=> #{str.inspect}\e[0m" }
+  if m
+    m.synchronize(&p)
+  else
+    p.call
+  end
+end
+
+# ----------------------------------------------------------------------
+
+def_use_case 'pure_ruby_single_request' do
+  q RC::Universal.new(:json_response => true).
+    get('https://api.github.com/users/godfat')['name']
+end
+
+def_use_case 'pure_ruby_concurrent_requests' do
+  client = RC::Universal.new(:json_response => true,
+                             :site => 'https://api.github.com/users/')
+  q [client.get('godfat'), client.get('cardinalblue')].map{ |u| u['name'] }
+end
+
+def_use_case 'pure_ruby_cache_requests' do
+  client = RC::Universal.new(:json_response => true, :cache => {})
+  3.times{ q client.get('https://api.github.com/users/godfat')['name'] }
+end
+
+def_use_case 'pure_ruby_callback_requests' do
+  m = Mutex.new
+  RC::Universal.new(:json_response => true                                  ,
+                    :site          => 'https://api.github.com/users/'       ,
+                    :log_method    => lambda{|str| m.synchronize{puts(str)}}).
+    get('godfat'){ |res|
+      q res['name'], m
+    }.
+    get('cardinalblue'){ |res|
+      q res['name'], m
+    }.wait
+end
+
+def_use_case 'pure_ruby_nested_concurrent_requests' do
+  m = Mutex.new
+  c = RC::Universal.new(:json_response => true                              ,
+                        :site          => 'https://api.github.com'          ,
+                        :log_method => lambda{|str| m.synchronize{puts(str)}})
+
+  %w[rubytaiwan godfat].each{ |user|
+    c.get("/users/#{user}/repos", :per_page => 100){ |repos|
+      rs = repos.reject{ |r| r['fork'] }
+      most_watched = rs.max_by{ |r| r['watchers'] }['name']
+      most_size    = rs.max_by{ |r| r['size']     }['name']
+
+      watch_contri = c.get("/repos/#{user}/#{most_watched}/contributors")
+       size_contri = c.get("/repos/#{user}/#{most_size}/contributors")
+
+      most_watched_most_contri = watch_contri.max_by{ |c| c['contributions'] }
+      most_size_most_contri    =  size_contri.max_by{ |c| c['contributions'] }
+
+      q "Most contributed user for most watched: #{user}/#{most_watched}:", m
+      q most_watched_most_contri['login'], m
+
+      q "Most contributed user for most size   : #{user}/#{most_size}:", m
+      q most_size_most_contri['login'], m
+    }
+  }
+
+  c.wait
+end
+
+# ----------------------------------------------------------------------
+
+def_use_case 'eventmachine_fiber_single_request'             do
+  EM.run{  Fiber.new{ pure_ruby_single_request_            ; EM.stop }.resume}
+end
+
+def_use_case 'eventmachine_fiber_concurrent_requests'        do
+  EM.run{  Fiber.new{ pure_ruby_concurrent_requests_       ; EM.stop }.resume}
+end
+
+def_use_case 'eventmachine_fiber_cache_requests'             do
+  EM.run{  Fiber.new{ pure_ruby_cache_requests_            ; EM.stop }.resume}
+end
+
+def_use_case 'eventmachine_fiber_callback_requests'          do
+  EM.run{  Fiber.new{ pure_ruby_callback_requests_         ; EM.stop }.resume}
+end
+
+def_use_case 'eventmachine_fiber_nested_concurrent_requests' do
+  EM.run{  Fiber.new{ pure_ruby_nested_concurrent_requests_; EM.stop }.resume}
+end
+
+# ----------------------------------------------------------------------
+
+def_use_case 'eventmachine_thread_single_request'             do
+  EM.run{ Thread.new{ pure_ruby_single_request_            ; EM.stop } }
+end
+
+def_use_case 'eventmachine_thread_concurrent_requests'        do
+  EM.run{ Thread.new{ pure_ruby_concurrent_requests_       ; EM.stop } }
+end
+
+def_use_case 'eventmachine_thread_cache_requests'             do
+  EM.run{ Thread.new{ pure_ruby_cache_requests_            ; EM.stop } }
+end
+
+def_use_case 'eventmachine_thread_callback_requests'          do
+  EM.run{ Thread.new{ pure_ruby_callback_requests_         ; EM.stop } }
+end
+
+def_use_case 'eventmachine_thread_nested_concurrent_requests' do
+  EM.run{ Thread.new{ pure_ruby_nested_concurrent_requests_; EM.stop } }
+end
+
+# ----------------------------------------------------------------------
+
+def_use_case 'eventmachine_rest_client_single_request'        do
+  EM.run{             pure_ruby_single_request_            ; EM.stop   }
+end
+
+def_use_case 'eventmachine_rest_client_concurrent_requests'   do
+  EM.run{             pure_ruby_concurrent_requests_       ; EM.stop   }
+end
+
+def_use_case 'eventmachine_rest_client_cache_requests'        do
+  EM.run{             pure_ruby_cache_requests_            ; EM.stop   }
+end
+
+def_use_case 'eventmachine_rest_client_callback_requests'     do
+  EM.run{             pure_ruby_callback_requests_         ; EM.stop   }
+end
+
+def_use_case 'eventmachine_rest_client_nested_concurrent_requests' do
+  EM.run{             pure_ruby_nested_concurrent_requests_; EM.stop   }
+end
+
+# ----------------------------------------------------------------------
+
+def_use_case 'pure_ruby' do
+  pure_ruby_single_request
+  pure_ruby_concurrent_requests
+  pure_ruby_cache_requests
+  pure_ruby_callback_requests
+  pure_ruby_nested_concurrent_requests
+end
+
+def_use_case 'eventmachine_fiber' do
+  eventmachine_fiber_single_request
+  eventmachine_fiber_concurrent_requests
+  eventmachine_fiber_cache_requests
+  eventmachine_fiber_callback_requests
+  eventmachine_fiber_nested_concurrent_requests
+end
+
+def_use_case 'eventmachine_thread' do
+  eventmachine_thread_single_request
+  eventmachine_thread_concurrent_requests
+  eventmachine_thread_cache_requests
+  eventmachine_thread_callback_requests
+  eventmachine_thread_nested_concurrent_requests
+end
+
+def_use_case 'eventmachine_rest_client' do
+  eventmachine_rest_client_single_request
+  eventmachine_rest_client_concurrent_requests
+  eventmachine_rest_client_cache_requests
+  eventmachine_rest_client_callback_requests
+  eventmachine_rest_client_nested_concurrent_requests
+end
+
+# ----------------------------------------------------------------------
+
+pure_ruby
+eventmachine_fiber
+eventmachine_thread
+eventmachine_rest_client

data/lib/rest-core.rb

@@ -16,6 +16,9 @@ module RestCore
 
   ASYNC            = 'async.callback'
   TIMER            = 'async.timer'
+  FUTURE           = 'async.future'
+
+  RootFiber        = Fiber.respond_to?(:current) && Fiber.current
 
   # core utilities
   autoload :Builder       , 'rest-core/builder'
@@ -46,34 +49,38 @@ module RestCore
   autoload :ErrorDetectorHttp, 'rest-core/middleware/error_detector_http'
   autoload :ErrorHandler  , 'rest-core/middleware/error_handler'
   autoload :FollowRedirect, 'rest-core/middleware/follow_redirect'
-  autoload :JsonDecode    , 'rest-core/middleware/json_decode'
+  autoload :JsonRequest   , 'rest-core/middleware/json_request'
+  autoload :JsonResponse  , 'rest-core/middleware/json_response'
   autoload :Oauth1Header  , 'rest-core/middleware/oauth1_header'
   autoload :Oauth2Header  , 'rest-core/middleware/oauth2_header'
   autoload :Oauth2Query   , 'rest-core/middleware/oauth2_query'
   autoload :Timeout       , 'rest-core/middleware/timeout'
 
-  # apps
-  autoload :Auto          , 'rest-core/app/auto'
-  autoload :Dry           , 'rest-core/app/dry'
-  autoload :RestClient    , 'rest-core/app/rest-client'
-  autoload :Coolio        , 'rest-core/app/coolio'
-  autoload :CoolioAsync   , 'rest-core/app/coolio-async'
-  autoload :CoolioFiber   , 'rest-core/app/coolio-fiber'
-  autoload :EmHttpRequest     , 'rest-core/app/em-http-request'
-  autoload :EmHttpRequestAsync, 'rest-core/app/em-http-request-async'
-  autoload :EmHttpRequestFiber, 'rest-core/app/em-http-request-fiber'
+  # engines
+  autoload :Auto          , 'rest-core/engine/auto'
+  autoload :Dry           , 'rest-core/engine/dry'
+  autoload :RestClient    , 'rest-core/engine/rest-client'
+  autoload :EmHttpRequest , 'rest-core/engine/em-http-request'
 
   # clients
   autoload :Simple        , 'rest-core/client/simple'
   autoload :Universal     , 'rest-core/client/universal'
+
+  # You might want to call this before launching your application in a
+  # threaded environment to avoid thread-safety issue in autoload.
+  def self.eagerload const=self, loaded={}
+    return if loaded[const.name]
+    loaded[const.name] = true
+    const.constants.each{ |n|
+      begin
+        c = const.const_get(n)
+      rescue LoadError => e
+        warn "RestCore: WARN: #{e} for #{const}\n" \
+             "  from #{e.backtrace.grep(/top.+required/).first}"
+      end
+      eagerload(c, loaded) if c.respond_to?(:constants) && !loaded[n]
+    }
+  end
 end
 
 RC = RestCore unless Object.const_defined?(:RC)
-
-begin
-  require 'fiber'
-rescue LoadError
-end
-# assume we would always require 'rest-core' in root fiber
-RestCore::RootFiber = Fiber.current if Object.const_defined?(:Fiber) &&
-                                       Fiber.respond_to?(:current)

data/lib/rest-core/builder.rb

@@ -6,8 +6,8 @@ class RestCore::Builder
   include RestCore
   include Wrapper
 
-  def self.default_app
-    @default_app ||= RestClient
+  def self.default_engine
+    @default_engine ||= RestCore::Auto
   end
 
   def self.client *attrs, &block

data/lib/rest-core/client.rb

@@ -1,6 +1,8 @@
 
 require 'rest-core'
 
+require 'weakref'
+
 module RestCore::Client
   include RestCore
 
@@ -48,10 +50,12 @@ module RestCore::Client
     mod.send(:include, accessor)
   end
 
-  attr_reader :app, :dry
+  attr_reader :app, :dry, :futures
   def initialize o={}
-    @app ||= self.class.builder.to_app
+    @app ||= self.class.builder.to_app # lighten! would reinitialize anyway
     @dry ||= self.class.builder.to_app(Dry)
+    @futures = []  # don't record any futures in lighten!
+    @mutex   = nil # for locking futures, lazily initialized for serialization
     o.each{ |key, value| send("#{key}=", value) if respond_to?("#{key}=") }
   end
 
@@ -81,12 +85,27 @@ module RestCore::Client
     dup.lighten!(o)
   end
 
+  def wait
+    return self if futures.empty?
+    current_futures = nil
+    mutex.synchronize{
+      current_futures = futures.dup
+      futures.clear
+    }
+    current_futures.each{ |f|
+      begin
+        f.wait
+      rescue WeakRef::RefError # it's gc'ed after we think it's alive
+      end if f.weakref_alive?
+    }
+    wait
+  end
+
   def url path, query={}, opts={}
-    Middleware.request_uri(
-      dry.call(build_env({
-        REQUEST_PATH  => path,
-        REQUEST_QUERY => query,
-        DRY           => true}.merge(opts))))
+    dry.call(build_env({
+      REQUEST_PATH  => path,
+      REQUEST_QUERY => query,
+      DRY           => true}.merge(opts)), &Middleware.method(:request_uri))
   end
 
   def get    path, query={}, opts={}, &cb
@@ -151,7 +170,7 @@ module RestCore::Client
     end
   end
 
-  def request_full env, app=app
+  def request_full env, app=app, &k
     response = app.call(build_env(
       {REQUEST_METHOD  => :get,
        REQUEST_PATH    => '/' ,
@@ -160,11 +179,16 @@ module RestCore::Client
        REQUEST_HEADERS => {}  ,
        FAIL            => []  ,
        LOG             => []  ,
-       ASYNC           => if block_given?
-                            lambda{ |res| yield(res) }
-                          else
-                            nil
-                          end}.merge(env)))
+       ASYNC           => !!k }.merge(env)),
+       &(k || Middleware.id))
+
+    # under ASYNC callback, response might not be a response hash
+    # in that case (maybe in a user created engine), Client#wait
+    # won't work because we have no way to track the future.
+    if response.kind_of?(Hash) && RestCore.const_defined?(:Future) &&
+       response[FUTURE].kind_of?(Future)
+      mutex.synchronize{ futures << WeakRef.new(response[FUTURE]) }
+    end
 
     if block_given?
       self
@@ -174,26 +198,15 @@ module RestCore::Client
   end
 
   def build_env env={}
-    string_keys(attributes).merge(string_keys(env))
+    Middleware.string_keys(attributes).merge(Middleware.string_keys(env))
   end
   # ------------------------ instance ---------------------
 
 
 
   private
-  def string_keys hash
-    hash.inject({}){ |r, (k, v)|
-      if v.kind_of?(Hash)
-        r[k.to_s] = case k.to_s
-                      when REQUEST_QUERY, REQUEST_PAYLOAD, REQUEST_HEADERS
-                        string_keys(v)
-                      else;         v
-                    end
-      else
-        r[k.to_s] = v
-      end
-      r
-    }
+  def mutex
+    @mutex ||= Mutex.new
   end
 
   def lighten_hash hash