rest-core 2.1.2 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -2
  3. data/.travis.yml +3 -5
  4. data/CHANGES.md +65 -5
  5. data/Gemfile +10 -5
  6. data/NOTE.md +1 -1
  7. data/README.md +194 -128
  8. data/Rakefile +8 -34
  9. data/TODO.md +3 -2
  10. data/example/simple.rb +6 -4
  11. data/example/use-cases.rb +39 -122
  12. data/lib/rest-core.rb +14 -5
  13. data/lib/rest-core/builder.rb +12 -2
  14. data/lib/rest-core/client.rb +31 -25
  15. data/lib/rest-core/engine.rb +39 -0
  16. data/lib/rest-core/engine/http-client.rb +41 -0
  17. data/lib/rest-core/engine/net-http-persistent.rb +21 -0
  18. data/lib/rest-core/engine/rest-client.rb +13 -42
  19. data/lib/rest-core/event_source.rb +91 -0
  20. data/lib/rest-core/middleware.rb +17 -11
  21. data/lib/rest-core/middleware/error_detector.rb +1 -6
  22. data/lib/rest-core/middleware/oauth1_header.rb +1 -0
  23. data/lib/rest-core/middleware/oauth2_header.rb +20 -8
  24. data/lib/rest-core/middleware/oauth2_query.rb +1 -0
  25. data/lib/rest-core/middleware/timeout.rb +5 -19
  26. data/lib/rest-core/promise.rb +137 -0
  27. data/lib/rest-core/test.rb +2 -43
  28. data/lib/rest-core/thread_pool.rb +122 -0
  29. data/lib/rest-core/timer.rb +30 -0
  30. data/lib/rest-core/util/hmac.rb +0 -8
  31. data/lib/rest-core/version.rb +1 -1
  32. data/lib/rest-core/wrapper.rb +1 -1
  33. data/rest-core.gemspec +36 -25
  34. data/task/README.md +54 -0
  35. data/task/gemgem.rb +150 -156
  36. data/test/test_builder.rb +2 -2
  37. data/test/test_cache.rb +8 -8
  38. data/test/test_client.rb +16 -6
  39. data/test/test_client_oauth1.rb +1 -1
  40. data/test/test_event_source.rb +77 -0
  41. data/test/test_follow_redirect.rb +1 -1
  42. data/test/test_future.rb +16 -0
  43. data/test/test_oauth2_header.rb +28 -0
  44. data/test/test_promise.rb +89 -0
  45. data/test/test_rest-client.rb +21 -0
  46. data/test/test_thread_pool.rb +10 -0
  47. data/test/test_timeout.rb +13 -8
  48. metadata +61 -37
  49. data/example/multi.rb +0 -44
  50. data/lib/rest-core/engine/auto.rb +0 -25
  51. data/lib/rest-core/engine/em-http-request.rb +0 -90
  52. data/lib/rest-core/engine/future/future.rb +0 -107
  53. data/lib/rest-core/engine/future/future_fiber.rb +0 -32
  54. data/lib/rest-core/engine/future/future_thread.rb +0 -29
  55. data/lib/rest-core/middleware/timeout/timer_em.rb +0 -26
  56. data/lib/rest-core/middleware/timeout/timer_thread.rb +0 -36
  57. data/task/.gitignore +0 -1
  58. data/test/test_em-http-request.rb +0 -186
@@ -0,0 +1,39 @@
1
+
2
+ require 'rest-core/promise'
3
+ require 'rest-core/middleware'
4
+
5
+ class RestCore::Engine
6
+ include RestCore::Middleware
7
+
8
+ def call env, &k
9
+ promise = Promise.new(env, k, env[ASYNC])
10
+ promise.defer{ request(promise, env) }
11
+ env.merge(RESPONSE_BODY => promise.future_body,
12
+ RESPONSE_STATUS => promise.future_status,
13
+ RESPONSE_HEADERS => promise.future_headers,
14
+ RESPONSE_SOCKET => promise.future_socket,
15
+ FAIL => promise.future_failures,
16
+ PROMISE => promise)
17
+ end
18
+
19
+ private
20
+ def payload_and_headers env
21
+ Payload.generate_with_headers(env[REQUEST_PAYLOAD], env[REQUEST_HEADERS])
22
+ end
23
+
24
+ def calculate_timeout timer
25
+ return [] unless timer
26
+ [timer.timeout, timer.timeout]
27
+ end
28
+
29
+ def normalize_headers headers
30
+ headers.inject({}){ |r, (k, v)|
31
+ r[k.to_s.upcase.tr('-', '_')] = if v.kind_of?(Array) && v.size == 1
32
+ v.first
33
+ else
34
+ v
35
+ end
36
+ r
37
+ }
38
+ end
39
+ end
@@ -0,0 +1,41 @@
1
+
2
+ require 'httpclient'
3
+ require 'rest-core/engine'
4
+
5
+ class RestCore::HttpClient < RestCore::Engine
6
+ private
7
+ def request promise, env
8
+ client = ::HTTPClient.new
9
+ client.cookie_manager = nil
10
+ client.follow_redirect_count = 0
11
+ client.transparent_gzip_decompression = true
12
+ payload, headers = payload_and_headers(env)
13
+
14
+ if env[HIJACK]
15
+ request_async(client, payload, headers, promise, env)
16
+ else
17
+ request_sync(client, payload, headers, promise, env)
18
+ end
19
+ rescue Exception => e
20
+ promise.reject(e)
21
+ end
22
+
23
+ def request_sync client, payload, headers, promise, env
24
+ client.connect_timeout, client.receive_timeout =
25
+ calculate_timeout(env[TIMER])
26
+
27
+ res = client.request(env[REQUEST_METHOD], request_uri(env), nil,
28
+ payload, {'User-Agent' => 'Ruby'}.merge(headers))
29
+
30
+ promise.fulfill(res.content, res.status,
31
+ normalize_headers(res.header.all))
32
+ end
33
+
34
+ def request_async client, payload, headers, promise, env
35
+ res = client.request_async(env[REQUEST_METHOD], request_uri(env), nil,
36
+ payload, {'User-Agent' => 'Ruby'}.merge(headers)).pop
37
+
38
+ promise.fulfill('', res.status,
39
+ normalize_headers(res.header.all), res.content)
40
+ end
41
+ end
@@ -0,0 +1,21 @@
1
+
2
+ require 'net/http/persistent'
3
+ require 'rest-core/engine'
4
+
5
+ class RestCore::NetHttpPersistent < RestCore::Engine
6
+ def request promise, env
7
+ http = ::Net::HTTP::Persistent.new
8
+ http.open_timeout, http.read_timeout = calculate_timeout(env[TIMER])
9
+ payload, headers = payload_and_headers(env)
10
+
11
+ uri = ::URI.parse(request_uri(env))
12
+ req = ::Net::HTTP.const_get(env[REQUEST_METHOD].to_s.capitalize).
13
+ new(uri, headers)
14
+ req.body_stream = payload
15
+ res = http.request(uri, req)
16
+
17
+ promise.fulfill(res.body, res.code.to_i, normalize_headers(res.to_hash))
18
+ rescue Exception => e
19
+ promise.reject(e)
20
+ end
21
+ end
@@ -1,58 +1,29 @@
1
1
 
2
2
  require 'restclient'
3
3
  require 'rest-core/patch/rest-client'
4
+ require 'rest-core/engine'
4
5
 
5
- require 'rest-core/engine/future/future'
6
- require 'rest-core/middleware'
7
-
8
- class RestCore::RestClient
9
- include RestCore::Middleware
10
- def call env, &k
11
- future = Future::FutureThread.new(env, k, env[ASYNC])
12
-
13
- # we can implement thread pool in the future
14
- t = future.wrap{ request(future, env) }
15
-
16
- env[TIMER].on_timeout{
17
- t.kill
18
- future.on_error(env[TIMER].error)
19
- } if env[TIMER]
20
-
21
- env.merge(RESPONSE_BODY => future.proxy_body,
22
- RESPONSE_STATUS => future.proxy_status,
23
- RESPONSE_HEADERS => future.proxy_headers,
24
- FUTURE => future)
25
- end
26
-
27
- def request future, env
28
- payload, headers = Payload.generate_with_headers(env[REQUEST_PAYLOAD],
29
- env[REQUEST_HEADERS])
6
+ class RestCore::RestClient < RestCore::Engine
7
+ def request promise, env
8
+ open_timeout, read_timeout = calculate_timeout(env[TIMER])
9
+ payload, headers = payload_and_headers(env)
30
10
  res = ::RestClient::Request.execute(:method => env[REQUEST_METHOD],
31
11
  :url => request_uri(env) ,
32
12
  :payload => payload ,
33
13
  :headers => headers ,
34
- :max_redirects => 0)
35
- future.on_load(res.body, res.code, normalize_headers(res.raw_headers))
36
-
14
+ :max_redirects => 0 ,
15
+ :open_timeout => open_timeout ,
16
+ :timeout => read_timeout )
17
+ promise.fulfill(res.body, res.code, normalize_headers(res.raw_headers))
37
18
  rescue ::RestClient::Exception => e
38
19
  if res = e.response
39
20
  # we don't want to raise an exception for 404 requests
40
- future.on_load(res.body, res.code, normalize_headers(res.raw_headers))
21
+ promise.fulfill(res.body, res.code, normalize_headers(res.raw_headers))
41
22
  else
42
- future.on_error(e)
23
+ promise.reject(e)
43
24
  end
44
- rescue Exception => e
45
- future.on_error(e)
46
- end
47
25
 
48
- def normalize_headers raw_headers
49
- raw_headers.inject({}){ |r, (k, v)|
50
- r[k.to_s.upcase.tr('-', '_')] = if v.kind_of?(Array) && v.size == 1
51
- v.first
52
- else
53
- v
54
- end
55
- r
56
- }
26
+ rescue Exception => e
27
+ promise.reject(e)
57
28
  end
58
29
  end
@@ -0,0 +1,91 @@
1
+
2
+ require 'thread'
3
+ require 'rest-core'
4
+
5
+ class RestCore::EventSource < Struct.new(:client, :path, :query, :opts,
6
+ :socket)
7
+ include RestCore
8
+ def start
9
+ self.mutex = Mutex.new
10
+ self.condv = ConditionVariable.new
11
+ @onopen ||= nil
12
+ @onmessage_for ||= nil
13
+ @onerror ||= nil
14
+
15
+ o = {REQUEST_HEADERS => {'Accept' => 'text/event-stream'},
16
+ HIJACK => true}.merge(opts)
17
+ client.get(path, query, o){ |sock| onopen(sock) }
18
+ end
19
+
20
+ def closed?
21
+ !!(socket && socket.closed?)
22
+ end
23
+
24
+ def close
25
+ socket && socket.close
26
+ rescue IOError
27
+ end
28
+
29
+ def wait
30
+ raise RC::Error.new("Not yet started for: #{self}") unless mutex
31
+ mutex.synchronize{ condv.wait(mutex) until closed? } unless closed?
32
+ end
33
+
34
+ def onopen sock=nil, &cb
35
+ if block_given?
36
+ @onopen = cb
37
+ else
38
+ @onopen.call(sock) if @onopen
39
+ onmessage_for(sock)
40
+ end
41
+ rescue Exception => e
42
+ begin # close the socket since we're going to stop anyway
43
+ sock.close # if we don't close it, client might wait forever
44
+ rescue IOError
45
+ end
46
+ # let the client has a chance to handle this, and make signal
47
+ onerror(e, sock)
48
+ end
49
+
50
+ def onmessage event=nil, sock=nil, &cb
51
+ if block_given?
52
+ @onmessage = cb
53
+ elsif @onmessage
54
+ @onmessage.call(event, sock)
55
+ end
56
+ end
57
+
58
+ # would also be called upon closing, would always be called at least once
59
+ def onerror error=nil, sock=nil, &cb
60
+ if block_given?
61
+ @onerror = cb
62
+ else
63
+ begin
64
+ @onerror.call(error, sock) if @onerror
65
+ ensure
66
+ condv.signal # should never deadlock someone
67
+ end
68
+ end
69
+ end
70
+
71
+ protected
72
+ attr_accessor :mutex, :condv
73
+
74
+ private
75
+ # called in requesting thread after the request is done
76
+ def onmessage_for sock
77
+ self.socket = sock # for you to track the socket
78
+ until sock.eof?
79
+ event = sock.readline("\n\n").split("\n").inject({}) do |r, i|
80
+ k, v = i.split(': ', 2)
81
+ r[k] = v
82
+ r
83
+ end
84
+ onmessage(event, sock)
85
+ end
86
+ sock.close
87
+ onerror(EOFError.new, sock)
88
+ rescue IOError => e
89
+ onerror(e, sock)
90
+ end
91
+ end
@@ -1,16 +1,10 @@
1
1
 
2
- require 'rest-core'
3
-
4
2
  require 'uri'
3
+ require 'rest-core'
5
4
 
6
5
  module RestCore::Middleware
7
6
  include RestCore
8
7
 
9
- # identity function
10
- def self.id
11
- @id ||= lambda{ |a| a }
12
- end
13
-
14
8
  def self.included mod
15
9
  mod.send(:include, RestCore)
16
10
  mod.send(:attr_reader, :app)
@@ -39,10 +33,22 @@ module RestCore::Middleware
39
33
  mod.send(:include, accessor)
40
34
  end
41
35
 
42
- def call env, &k; app.call(env, &(k || id)) ; end
43
- def fail env, obj; env.merge(FAIL => (env[FAIL] || []) + [obj]); end
44
- def log env, obj; env.merge(LOG => (env[LOG] || []) + [obj]); end
45
- def id ; Middleware.id ; end
36
+ def call env, &k; app.call(env, &(k || id)); end
37
+ def id ; RC.id ; end
38
+ def fail env, obj
39
+ if obj
40
+ env.merge(FAIL => (env[FAIL] || []) + [obj])
41
+ else
42
+ env
43
+ end
44
+ end
45
+ def log env, obj
46
+ if obj
47
+ env.merge(LOG => (env[LOG] || []) + [obj])
48
+ else
49
+ env
50
+ end
51
+ end
46
52
  def run app=app
47
53
  if app.respond_to?(:app) && app.app
48
54
  run(app.app)
@@ -8,12 +8,7 @@ class RestCore::ErrorDetector
8
8
  def call env
9
9
  app.call(env){ |response|
10
10
  detector = error_detector(env)
11
- yield(
12
- if error = (detector && detector.call(response))
13
- fail(response, error)
14
- else
15
- response
16
- end)
11
+ yield(fail(response, detector && detector.call(response)))
17
12
  }
18
13
  end
19
14
  end
@@ -5,6 +5,7 @@ require 'rest-core/util/hmac'
5
5
  require 'uri'
6
6
  require 'openssl'
7
7
 
8
+ # http://tools.ietf.org/html/rfc5849
8
9
  class RestCore::Oauth1Header
9
10
  def self.members
10
11
  [:request_token_path, :access_token_path, :authorize_path,
@@ -1,21 +1,33 @@
1
1
 
2
2
  require 'rest-core/middleware'
3
3
 
4
+ # http://tools.ietf.org/html/rfc6749
4
5
  class RestCore::Oauth2Header
5
6
  def self.members; [:access_token_type, :access_token]; end
6
7
  include RestCore::Middleware
7
8
 
8
9
  def call env, &k
9
10
  start_time = Time.now
10
- headers = {'Authorization' =>
11
- "#{access_token_type(env)} #{access_token(env)}"}.
12
- merge(env[REQUEST_HEADERS]) if access_token(env)
11
+ headers = build_headers(env)
12
+ auth = headers['Authorization']
13
+ event = Event::WithHeader.new(Time.now - start_time,
14
+ "Authorization: #{auth}") if auth
13
15
 
14
- event = Event::WithHeader.new(Time.now - start_time,
15
- "Authorization: #{headers['Authorization']}") if headers
16
+ app.call(log(env.merge(REQUEST_HEADERS => headers), event), &k)
17
+ end
18
+
19
+ def build_headers env
20
+ auth = case token = access_token(env)
21
+ when String
22
+ token
23
+ when Hash
24
+ token.map{ |(k, v)| "#{k}=\"#{v}\"" }.join(', ')
25
+ end
16
26
 
17
- app.call(log(env.merge(REQUEST_HEADERS => headers ||
18
- env[REQUEST_HEADERS]), event),
19
- &k)
27
+ if auth
28
+ {'Authorization' => "#{access_token_type(env)} #{auth}"}
29
+ else
30
+ {}
31
+ end.merge(env[REQUEST_HEADERS])
20
32
  end
21
33
  end
@@ -1,6 +1,7 @@
1
1
 
2
2
  require 'rest-core/middleware'
3
3
 
4
+ # http://tools.ietf.org/html/rfc6749
4
5
  class RestCore::Oauth2Query
5
6
  def self.members; [:access_token]; end
6
7
  include RestCore::Middleware
@@ -1,7 +1,6 @@
1
1
 
2
2
  require 'rest-core/middleware'
3
-
4
- require 'timeout'
3
+ require 'rest-core/timer'
5
4
 
6
5
  class RestCore::Timeout
7
6
  def self.members; [:timeout]; end
@@ -22,27 +21,14 @@ class RestCore::Timeout
22
21
  end
23
22
 
24
23
  def monitor env
25
- class_name = case name = run.class.to_s
26
- when /Auto/
27
- run.http_client.class.to_s
28
- else
29
- name
30
- end
31
-
32
- timer = case class_name
33
- when /EmHttpRequest/
34
- TimerEm
35
- else
36
- TimerThread
37
- end.new(timeout(env), timeout_error)
38
-
24
+ timer = Timer.new(timeout(env), timeout_error)
39
25
  yield(env.merge(TIMER => timer))
26
+ rescue Exception
27
+ timer.cancel
28
+ raise
40
29
  end
41
30
 
42
31
  def timeout_error
43
32
  ::Timeout::Error.new('execution expired')
44
33
  end
45
-
46
- autoload :TimerEm , 'rest-core/middleware/timeout/timer_em'
47
- autoload :TimerThread, 'rest-core/middleware/timeout/timer_thread'
48
34
  end
@@ -0,0 +1,137 @@
1
+
2
+ require 'thread'
3
+ require 'rest-core'
4
+
5
+ class RestCore::Promise
6
+ include RestCore
7
+
8
+ class Future < BasicObject
9
+ def initialize promise, target
10
+ @promise, @target = promise, target
11
+ end
12
+
13
+ def method_missing msg, *args, &block
14
+ @promise.yield[@target].__send__(msg, *args, &block)
15
+ end
16
+ end
17
+
18
+ def initialize env, k=RC.id, immediate=false, &job
19
+ self.env = env
20
+ self.k = k
21
+ self.immediate = immediate
22
+
23
+ self.body, self.status, self.headers,
24
+ self.socket, self.response, self.error, = nil
25
+
26
+ self.condv = ConditionVariable.new
27
+ self.mutex = Mutex.new
28
+
29
+ defer(&job) if job
30
+ end
31
+
32
+ def inspect
33
+ "<#{self.class.name} for #{env[REQUEST_PATH]}>"
34
+ end
35
+
36
+ def future_body ; Future.new(self, RESPONSE_BODY ); end
37
+ def future_status ; Future.new(self, RESPONSE_STATUS ); end
38
+ def future_headers ; Future.new(self, RESPONSE_HEADERS); end
39
+ def future_socket ; Future.new(self, RESPONSE_SOCKET ); end
40
+ def future_failures; Future.new(self, FAIL) ; end
41
+
42
+ # called in client thread
43
+ def defer &job
44
+ if pool_size < 0 # negative number for blocking call
45
+ job.call
46
+ elsif pool_size > 0
47
+ self.task = client_class.thread_pool.defer do
48
+ synchronized_yield{ job.call }
49
+ end
50
+ else
51
+ Thread.new{ synchronized_yield{ job.call } }
52
+ end
53
+ env[TIMER].on_timeout{ reject(env[TIMER].error) } if env[TIMER]
54
+ end
55
+
56
+ # called in client thread (client.wait)
57
+ def wait
58
+ # it might be awaken by some other futures!
59
+ mutex.synchronize{ condv.wait(mutex) until !!status } unless !!status
60
+ end
61
+
62
+ # called in client thread (from the future (e.g. body))
63
+ def yield
64
+ wait
65
+ callback
66
+ end
67
+
68
+ # called in requesting thread after the request is done
69
+ def fulfill body, status, headers, socket=nil
70
+ env[TIMER].cancel if env[TIMER]
71
+ self.body, self.status, self.headers, self.socket =
72
+ body, status, headers, socket
73
+ # under ASYNC callback, should call immediately
74
+ callback_in_async if immediate
75
+ condv.broadcast # client or response might be waiting
76
+ end
77
+
78
+ # called in requesting thread if something goes wrong or timed out
79
+ def reject error
80
+ task.cancel if task
81
+
82
+ self.error = if error.kind_of?(Exception)
83
+ error
84
+ else
85
+ Error.new(error || 'unknown')
86
+ end
87
+ fulfill('', 0, {})
88
+ end
89
+
90
+ protected
91
+ attr_accessor :env, :k, :immediate,
92
+ :response, :body, :status, :headers, :socket, :error,
93
+ :condv, :mutex, :task
94
+
95
+ private
96
+ # called in a new thread if pool_size == 0, otherwise from the pool
97
+ # i.e. requesting thread
98
+ def synchronized_yield
99
+ mutex.synchronize{ yield }
100
+ rescue Exception => e
101
+ # nothing we can do here for an asynchronous exception,
102
+ # so we just log the error
103
+ # TODO: add error_log_method
104
+ warn "RestCore: ERROR: #{e}\n from #{e.backtrace.inspect}"
105
+ reject(e) # should never deadlock someone
106
+ end
107
+
108
+ # called in client thread, when yield is called
109
+ def callback
110
+ self.response ||= k.call(
111
+ env.merge(RESPONSE_BODY => body ,
112
+ RESPONSE_STATUS => status,
113
+ RESPONSE_HEADERS => headers,
114
+ RESPONSE_SOCKET => socket,
115
+ FAIL => ((env[FAIL]||[]) + [error]).compact,
116
+ LOG => env[LOG] ||[]))
117
+ end
118
+
119
+ # called in requesting thread, whenever the request is done
120
+ def callback_in_async
121
+ callback
122
+ rescue Exception => e
123
+ # nothing we can do here for an asynchronous exception,
124
+ # so we just log the error
125
+ # TODO: add error_log_method
126
+ warn "RestCore: ERROR: #{e}\n from #{e.backtrace.inspect}"
127
+ end
128
+
129
+ def client_class; env[CLIENT].class; end
130
+ def pool_size
131
+ @pool_size ||= if client_class.respond_to?(:pool_size)
132
+ client_class.pool_size
133
+ else
134
+ 0
135
+ end
136
+ end
137
+ end