carbon 2.1.0 → 2.2.1
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +17 -0
- data/carbon.gemspec +0 -1
- data/lib/carbon.rb +8 -5
- data/lib/carbon/future.rb +10 -15
- data/lib/carbon/version.rb +1 -1
- data/test/carbon_test.rb +49 -25
- metadata +16 -28
- data/developer/REDUCE_HTTP_CONNECTIONS.markdown +0 -46
data/CHANGELOG
CHANGED
@@ -1,9 +1,26 @@
|
|
1
|
+
2.2.1 / 2012-03-21
|
2
|
+
|
3
|
+
* Enhancements
|
4
|
+
|
5
|
+
* Switched from EventMachine to pure Ruby threads, a small speed sacrifice that gains us compatibilty with MRI 1.8, MRI 1.9, and JRuby. Also works better in multithreaded environments - no accidentally turning off somebody's EventMachine reactor :)
|
6
|
+
|
7
|
+
2.2.0 / 2012-03-20
|
8
|
+
|
9
|
+
* Enhancements
|
10
|
+
|
11
|
+
* Carbon.query(os) now returns a Hash keyed by the "os", instead of just a carefully ordered Array.
|
12
|
+
|
13
|
+
* Breaking changes
|
14
|
+
|
15
|
+
* Carbon.query(os) has a different return value!
|
16
|
+
|
1
17
|
2.1.0 / 2012-03-14
|
2
18
|
|
3
19
|
* Enhancements
|
4
20
|
|
5
21
|
* Simplified API down to just Carbon.query. It has three method signatures.
|
6
22
|
* Simplified EventMachine reactor
|
23
|
+
* Limited concurrency to 16 connections at a time
|
7
24
|
|
8
25
|
* Breaking changes
|
9
26
|
|
data/carbon.gemspec
CHANGED
@@ -15,7 +15,6 @@ Gem::Specification.new do |s|
|
|
15
15
|
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
16
|
s.require_paths = ["lib"]
|
17
17
|
|
18
|
-
s.add_runtime_dependency 'em-http-request'
|
19
18
|
s.add_runtime_dependency 'activesupport'
|
20
19
|
s.add_runtime_dependency 'multi_json'
|
21
20
|
s.add_runtime_dependency 'hashie'
|
data/lib/carbon.rb
CHANGED
@@ -66,11 +66,11 @@ module Carbon
|
|
66
66
|
# @overload query(os)
|
67
67
|
# Get multiple impact estimates for arrays and/or query-able objects concurrently.
|
68
68
|
# @param [Array<Array, #as_impact_query>] os An array of arrays in +[emitter, params]+ format and/or objects that respond to +#as_impact_query+.
|
69
|
-
# @return [
|
69
|
+
# @return [Hash{Object => Hashie::Mash}] A +Hash+ of +Hashie::Mash+ objects, keyed on the original query object.
|
70
70
|
#
|
71
71
|
# @note We make up to 16 requests concurrently (hardcoded, per the Brighter Planet Terms of Service) and it can be more than 90% faster than running queries serially!
|
72
72
|
#
|
73
|
-
# @
|
73
|
+
# @raise [ArgumentError] If your arguments don't match any of the method signatures.
|
74
74
|
#
|
75
75
|
# @example A flight taken in 2009
|
76
76
|
# Carbon.query('Flight', :origin_airport => 'MSN', :destination_airport => 'ORD', :date => '2009-01-01', :timeframe => Timeframe.new(:year => 2009), :comply => [:tcr])
|
@@ -107,7 +107,7 @@ module Carbon
|
|
107
107
|
# @example Flights and cars (concurrently, as query-able objects)
|
108
108
|
# Carbon.query(MyFlight.all+MyCar.all)
|
109
109
|
#
|
110
|
-
# @example Cars month-by-month
|
110
|
+
# @example Cars month-by-month (note that you won't get MyCar objects back, you'll get Arrays back. This will be fixed soon.)
|
111
111
|
# cars_by_month = MyCar.all.inject([]) do |memo, my_car|
|
112
112
|
# months.each do |first_day_of_the_month|
|
113
113
|
# my_car.as_impact_query(:date => first_day_of_the_month)
|
@@ -131,9 +131,12 @@ module Carbon
|
|
131
131
|
future.multi!
|
132
132
|
future
|
133
133
|
end
|
134
|
-
Future.multi(futures).
|
135
|
-
future.result
|
134
|
+
Future.multi(futures).inject({}) do |memo, future|
|
135
|
+
memo[future.object] = future.result
|
136
|
+
memo
|
136
137
|
end
|
138
|
+
else
|
139
|
+
raise ::ArgumentError, "Didn't match any of the method signatures. If you want multiple queries, make sure to pass an unsplatted Array."
|
137
140
|
end
|
138
141
|
end
|
139
142
|
|
data/lib/carbon/future.rb
CHANGED
@@ -3,18 +3,19 @@ require 'net/http'
|
|
3
3
|
require 'cache_method'
|
4
4
|
require 'hashie/mash'
|
5
5
|
require 'multi_json'
|
6
|
-
require 'em-http-request'
|
7
6
|
|
8
7
|
module Carbon
|
9
8
|
# @private
|
10
9
|
class Future
|
11
10
|
class << self
|
12
11
|
def wrap(query_array_or_o)
|
13
|
-
if query_array_or_o.is_a?(::Array)
|
12
|
+
future = if query_array_or_o.is_a?(::Array)
|
14
13
|
new(*query_array_or_o)
|
15
14
|
else
|
16
15
|
new(*query_array_or_o.as_impact_query)
|
17
16
|
end
|
17
|
+
future.object = query_array_or_o
|
18
|
+
future
|
18
19
|
end
|
19
20
|
|
20
21
|
def single(future)
|
@@ -26,20 +27,12 @@ module Carbon
|
|
26
27
|
|
27
28
|
def multi(futures)
|
28
29
|
uniq_pending_futures = futures.uniq.select { |future| future.pending? }
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
pool = (0..(pool_size-1)).map { ::EventMachine::HttpRequest.new(Carbon::DOMAIN) }
|
33
|
-
pool_idx = 0
|
34
|
-
::EventMachine.run do
|
35
|
-
uniq_pending_futures.each do |future|
|
36
|
-
multi.add future, pool[pool_idx].post(:path => "/#{future.emitter.underscore.pluralize}.json", :body => future.params)
|
37
|
-
pool_idx = (pool_idx + 1) % pool_size
|
30
|
+
while (set = uniq_pending_futures.pop(Carbon::CONCURRENCY)).any?
|
31
|
+
ts = set.map do |future|
|
32
|
+
::Thread.new { single future }
|
38
33
|
end
|
39
|
-
|
40
|
-
|
41
|
-
multi.responses[:errback].each { |future, http| future.finalize http.response_header.status }
|
42
|
-
::EventMachine.stop
|
34
|
+
ts.each do |t|
|
35
|
+
t.join
|
43
36
|
end
|
44
37
|
end
|
45
38
|
futures
|
@@ -49,6 +42,8 @@ module Carbon
|
|
49
42
|
attr_reader :emitter
|
50
43
|
attr_reader :params
|
51
44
|
|
45
|
+
attr_accessor :object
|
46
|
+
|
52
47
|
def initialize(emitter, params = {})
|
53
48
|
@result = nil
|
54
49
|
@emitter = emitter
|
data/lib/carbon/version.rb
CHANGED
data/test/carbon_test.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require File.expand_path("../helper", __FILE__)
|
2
2
|
|
3
|
+
Thread.abort_on_exception = true
|
3
4
|
Carbon.key = 'carbon_test'
|
4
5
|
|
5
6
|
class MyNissan
|
@@ -77,9 +78,15 @@ describe Carbon do
|
|
77
78
|
result.errors.first.must_equal 'Good job'
|
78
79
|
end
|
79
80
|
end
|
81
|
+
it "raises ArgumentError if args are bad" do
|
82
|
+
lambda {
|
83
|
+
Carbon.query(['Flight'])
|
84
|
+
}.must_raise ArgumentError
|
85
|
+
end
|
80
86
|
end
|
81
87
|
describe '(in parallel)' do
|
82
88
|
before do
|
89
|
+
flunk if ENV['SKIP_MULTI'] == 'true'
|
83
90
|
@queries = []
|
84
91
|
@queries << ['Flight', {:origin_airport => 'LAX', :destination_airport => 'SFO', :segments_per_trip => 1, :trips => 1}]
|
85
92
|
@queries << ['Flight', {:origin_airport => 'MSN', :destination_airport => 'ORD', :segments_per_trip => 1, :trips => 1}]
|
@@ -98,34 +105,50 @@ describe Carbon do
|
|
98
105
|
@queries << ['Monkey', {:bananas => '3'}]
|
99
106
|
@queries = @queries.sort_by { rand }
|
100
107
|
end
|
108
|
+
it "is easy to use" do
|
109
|
+
flight = ['Flight']
|
110
|
+
rail_trip = ['RailTrip']
|
111
|
+
results = Carbon.query([flight, rail_trip])
|
112
|
+
results[flight].decisions.must_equal Carbon.query('Flight').decisions
|
113
|
+
results[rail_trip].decisions.must_equal Carbon.query('RailTrip').decisions
|
114
|
+
end
|
101
115
|
it "doesn't hang up on 0 queries" do
|
102
|
-
Timeout.timeout(0.5) { Carbon.query([]) }.must_equal
|
116
|
+
Timeout.timeout(0.5) { Carbon.query([]) }.must_equal(Hash.new)
|
103
117
|
end
|
104
118
|
it "can be used on objects that respond to #as_impact_query" do
|
105
|
-
Carbon.query([MyNissanAltima.new(2001), MyNissanAltima.new(2006)]).map(&:decisions).must_equal Carbon.query([MyNissanAltima.new(2001).as_impact_query, MyNissanAltima.new(2006).as_impact_query]).map(&:decisions)
|
119
|
+
Carbon.query([MyNissanAltima.new(2001), MyNissanAltima.new(2006)]).values.map(&:decisions).map(&:carbon).map(&:object).map(&:value).must_equal Carbon.query([MyNissanAltima.new(2001).as_impact_query, MyNissanAltima.new(2006).as_impact_query]).values.map(&:decisions).map(&:carbon).map(&:object).map(&:value)
|
106
120
|
end
|
107
121
|
it "runs multiple queries at once" do
|
108
|
-
reference_results = @queries.
|
109
|
-
Carbon.query(*query)
|
122
|
+
reference_results = @queries.inject({}) do |memo, query|
|
123
|
+
memo[query] = Carbon.query(*query)
|
124
|
+
memo
|
110
125
|
end
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
126
|
+
ts = []
|
127
|
+
3.times do
|
128
|
+
ts << Thread.new do
|
129
|
+
flush_cache! # important!
|
130
|
+
multi_results = Carbon.query(@queries)
|
131
|
+
error_count = 0
|
132
|
+
multi_results.each do |query, result|
|
133
|
+
if result.success
|
134
|
+
result.decisions.carbon.object.value.must_be :>, 0
|
135
|
+
result.decisions.carbon.object.value.must_be :<, 10_000
|
136
|
+
else
|
137
|
+
error_count += 1
|
138
|
+
end
|
139
|
+
end
|
140
|
+
error_count.must_equal 3
|
141
|
+
reference_results.each do |query, reference_result|
|
142
|
+
if reference_result.success
|
143
|
+
multi_results[query].decisions.must_equal reference_result.decisions
|
144
|
+
else
|
145
|
+
multi_results[query].must_equal reference_result
|
146
|
+
end
|
147
|
+
end
|
120
148
|
end
|
121
149
|
end
|
122
|
-
|
123
|
-
|
124
|
-
if reference_result.success
|
125
|
-
multi_results[idx].decisions.must_equal reference_result.decisions
|
126
|
-
else
|
127
|
-
multi_results[idx].must_equal reference_result
|
128
|
-
end
|
150
|
+
ts.each do |t|
|
151
|
+
t.join
|
129
152
|
end
|
130
153
|
end
|
131
154
|
it "is faster than single threaded" do
|
@@ -153,17 +176,18 @@ describe Carbon do
|
|
153
176
|
$stderr.puts " Cached multi-threaded was #{((multi_threaded_time - cached_multi_threaded_time) / multi_threaded_time * 100).round}% faster than uncached multi-threaded"
|
154
177
|
end
|
155
178
|
it "safely uniq's and caches queries" do
|
156
|
-
reference_results = @queries.
|
157
|
-
Carbon.query(*query)
|
179
|
+
reference_results = @queries.inject({}) do |memo, query|
|
180
|
+
memo[query] = Carbon.query(*query)
|
181
|
+
memo
|
158
182
|
end
|
159
183
|
flush_cache! # important!
|
160
184
|
3.times do
|
161
185
|
multi_results = Carbon.query(@queries)
|
162
|
-
reference_results.
|
186
|
+
reference_results.each do |query, reference_result|
|
163
187
|
if reference_result.success
|
164
|
-
multi_results[
|
188
|
+
multi_results[query].decisions.must_equal reference_result.decisions
|
165
189
|
else
|
166
|
-
multi_results[
|
190
|
+
multi_results[query].must_equal reference_result
|
167
191
|
end
|
168
192
|
end
|
169
193
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: carbon
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1
|
4
|
+
version: 2.2.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,22 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-03-
|
12
|
+
date: 2012-03-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
-
- !ruby/object:Gem::Dependency
|
15
|
-
name: em-http-request
|
16
|
-
requirement: &2164986440 !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
|
-
requirements:
|
19
|
-
- - ! '>='
|
20
|
-
- !ruby/object:Gem::Version
|
21
|
-
version: '0'
|
22
|
-
type: :runtime
|
23
|
-
prerelease: false
|
24
|
-
version_requirements: *2164986440
|
25
14
|
- !ruby/object:Gem::Dependency
|
26
15
|
name: activesupport
|
27
|
-
requirement: &
|
16
|
+
requirement: &2169272660 !ruby/object:Gem::Requirement
|
28
17
|
none: false
|
29
18
|
requirements:
|
30
19
|
- - ! '>='
|
@@ -32,10 +21,10 @@ dependencies:
|
|
32
21
|
version: '0'
|
33
22
|
type: :runtime
|
34
23
|
prerelease: false
|
35
|
-
version_requirements: *
|
24
|
+
version_requirements: *2169272660
|
36
25
|
- !ruby/object:Gem::Dependency
|
37
26
|
name: multi_json
|
38
|
-
requirement: &
|
27
|
+
requirement: &2169272200 !ruby/object:Gem::Requirement
|
39
28
|
none: false
|
40
29
|
requirements:
|
41
30
|
- - ! '>='
|
@@ -43,10 +32,10 @@ dependencies:
|
|
43
32
|
version: '0'
|
44
33
|
type: :runtime
|
45
34
|
prerelease: false
|
46
|
-
version_requirements: *
|
35
|
+
version_requirements: *2169272200
|
47
36
|
- !ruby/object:Gem::Dependency
|
48
37
|
name: hashie
|
49
|
-
requirement: &
|
38
|
+
requirement: &2169271780 !ruby/object:Gem::Requirement
|
50
39
|
none: false
|
51
40
|
requirements:
|
52
41
|
- - ! '>='
|
@@ -54,10 +43,10 @@ dependencies:
|
|
54
43
|
version: '0'
|
55
44
|
type: :runtime
|
56
45
|
prerelease: false
|
57
|
-
version_requirements: *
|
46
|
+
version_requirements: *2169271780
|
58
47
|
- !ruby/object:Gem::Dependency
|
59
48
|
name: cache_method
|
60
|
-
requirement: &
|
49
|
+
requirement: &2169271220 !ruby/object:Gem::Requirement
|
61
50
|
none: false
|
62
51
|
requirements:
|
63
52
|
- - ! '>='
|
@@ -65,10 +54,10 @@ dependencies:
|
|
65
54
|
version: '0'
|
66
55
|
type: :runtime
|
67
56
|
prerelease: false
|
68
|
-
version_requirements: *
|
57
|
+
version_requirements: *2169271220
|
69
58
|
- !ruby/object:Gem::Dependency
|
70
59
|
name: bombshell
|
71
|
-
requirement: &
|
60
|
+
requirement: &2169270600 !ruby/object:Gem::Requirement
|
72
61
|
none: false
|
73
62
|
requirements:
|
74
63
|
- - ! '>='
|
@@ -76,10 +65,10 @@ dependencies:
|
|
76
65
|
version: '0'
|
77
66
|
type: :runtime
|
78
67
|
prerelease: false
|
79
|
-
version_requirements: *
|
68
|
+
version_requirements: *2169270600
|
80
69
|
- !ruby/object:Gem::Dependency
|
81
70
|
name: conversions
|
82
|
-
requirement: &
|
71
|
+
requirement: &2169269920 !ruby/object:Gem::Requirement
|
83
72
|
none: false
|
84
73
|
requirements:
|
85
74
|
- - ! '>='
|
@@ -87,10 +76,10 @@ dependencies:
|
|
87
76
|
version: '0'
|
88
77
|
type: :runtime
|
89
78
|
prerelease: false
|
90
|
-
version_requirements: *
|
79
|
+
version_requirements: *2169269920
|
91
80
|
- !ruby/object:Gem::Dependency
|
92
81
|
name: brighter_planet_metadata
|
93
|
-
requirement: &
|
82
|
+
requirement: &2169269500 !ruby/object:Gem::Requirement
|
94
83
|
none: false
|
95
84
|
requirements:
|
96
85
|
- - ! '>='
|
@@ -98,7 +87,7 @@ dependencies:
|
|
98
87
|
version: '0'
|
99
88
|
type: :runtime
|
100
89
|
prerelease: false
|
101
|
-
version_requirements: *
|
90
|
+
version_requirements: *2169269500
|
102
91
|
description: Brighter Planet API client for Ruby
|
103
92
|
email:
|
104
93
|
- seamus@abshere.net
|
@@ -118,7 +107,6 @@ files:
|
|
118
107
|
- bin/carbon
|
119
108
|
- carbon.gemspec
|
120
109
|
- developer/MULTI.markdown
|
121
|
-
- developer/REDUCE_HTTP_CONNECTIONS.markdown
|
122
110
|
- developer/avro_helper.rb
|
123
111
|
- developer/cm1_avro.rb
|
124
112
|
- features/shell.feature
|
@@ -1,46 +0,0 @@
|
|
1
|
-
One way to reduce the number of connections by a constant... but it makes it slower (because requests are serialized) and less reliable (because there is a 30s heroku limit)
|
2
|
-
|
3
|
-
def self.multi(queries)
|
4
|
-
unsorted = {}
|
5
|
-
pool_size = (queries.length.to_f / 3).ceil
|
6
|
-
$stderr.puts "Starting #{pool_size} workers"
|
7
|
-
::EventMachine.run do
|
8
|
-
multi = ::EventMachine::MultiRequest.new
|
9
|
-
pool = 0.upto(pool_size).map do
|
10
|
-
::EventMachine::HttpRequest.new("http://#{domain}")
|
11
|
-
end
|
12
|
-
pool_idx = 0
|
13
|
-
queries.each_with_index do |(emitter, params), query_idx|
|
14
|
-
params ||= {}
|
15
|
-
multi.add query_idx, pool[pool_idx].post(:path => "/#{emitter.underscore.pluralize}.json", :body => params, :keepalive => true)
|
16
|
-
pool_idx = (pool_idx + 1) % pool_size
|
17
|
-
end
|
18
|
-
multi.callback do
|
19
|
-
multi.responses[:callback].each do |query_idx, http|
|
20
|
-
result = ::Hashie::Mash.new
|
21
|
-
result.status = http.response_header.status
|
22
|
-
if (200..299).include?(result.status)
|
23
|
-
result.success = true
|
24
|
-
result.merge! ::MultiJson.decode(http.response)
|
25
|
-
else
|
26
|
-
result.success = false
|
27
|
-
result.errors = [http.response]
|
28
|
-
end
|
29
|
-
unsorted[query_idx] = result
|
30
|
-
end
|
31
|
-
multi.responses[:errback].each do |query_idx, http|
|
32
|
-
result = ::Hashie::Mash.new
|
33
|
-
result.status = http.response_header.status
|
34
|
-
result.success = false
|
35
|
-
result.errors = ['Timeout or other network error.']
|
36
|
-
unsorted[query_idx] = result
|
37
|
-
end
|
38
|
-
::EventMachine.stop
|
39
|
-
end
|
40
|
-
end
|
41
|
-
unsorted.sort_by do |query_idx, _|
|
42
|
-
query_idx
|
43
|
-
end.map do |_, result|
|
44
|
-
result
|
45
|
-
end
|
46
|
-
end
|