carbon 2.1.0 → 2.2.1

data/CHANGELOG

@@ -1,9 +1,26 @@
+2.2.1 / 2012-03-21
+
+* Enhancements
+
+  * Switched from EventMachine to pure Ruby threads, a small speed sacrifice that gains us compatibilty with MRI 1.8, MRI 1.9, and JRuby. Also works better in multithreaded environments - no accidentally turning off somebody's EventMachine reactor :)
+
+2.2.0 / 2012-03-20
+
+* Enhancements
+
+  * Carbon.query(os) now returns a Hash keyed by the "os", instead of just a carefully ordered Array.
+
+* Breaking changes
+
+  * Carbon.query(os) has a different return value!
+
 2.1.0 / 2012-03-14
 
 * Enhancements
 
   * Simplified API down to just Carbon.query. It has three method signatures.
   * Simplified EventMachine reactor
+  * Limited concurrency to 16 connections at a time
 
 * Breaking changes
 

data/carbon.gemspec

@@ -15,7 +15,6 @@ Gem::Specification.new do |s|
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.require_paths = ["lib"]
 
-  s.add_runtime_dependency 'em-http-request'
   s.add_runtime_dependency 'activesupport'
   s.add_runtime_dependency 'multi_json'
   s.add_runtime_dependency 'hashie'

data/lib/carbon.rb

@@ -66,11 +66,11 @@ module Carbon
   # @overload query(os)
   #   Get multiple impact estimates for arrays and/or query-able objects concurrently.
   #   @param [Array<Array, #as_impact_query>] os An array of arrays in +[emitter, params]+ format and/or objects that respond to +#as_impact_query+.
-  #   @return [Array<Hashie::Mash>] An array of +Hashie::Mash+ objects in the same order.
+  #   @return [Hash{Object => Hashie::Mash}] A +Hash+ of +Hashie::Mash+ objects, keyed on the original query object.
   #
   # @note We make up to 16 requests concurrently (hardcoded, per the Brighter Planet Terms of Service) and it can be more than 90% faster than running queries serially!
   #
-  # @note Using concurrency on JRuby, you may get errors like SOCKET: SET COMM INACTIVITY UNIMPLEMENTED 10 because under the hood we're using {https://github.com/igrigorik/em-http-request em-http-request}, which suffers from {https://github.com/eventmachine/eventmachine/issues/155 an issue with +pending_connect_timeout+}.
+  # @raise [ArgumentError] If your arguments don't match any of the method signatures.
   #
   # @example A flight taken in 2009
   #   Carbon.query('Flight', :origin_airport => 'MSN', :destination_airport => 'ORD', :date => '2009-01-01', :timeframe => Timeframe.new(:year => 2009), :comply => [:tcr])
@@ -107,7 +107,7 @@ module Carbon
   # @example Flights and cars (concurrently, as query-able objects)
   #   Carbon.query(MyFlight.all+MyCar.all)
   #
-  # @example Cars month-by-month
+  # @example Cars month-by-month (note that you won't get MyCar objects back, you'll get Arrays back. This will be fixed soon.)
   #   cars_by_month = MyCar.all.inject([]) do |memo, my_car|
   #     months.each do |first_day_of_the_month|
   #       my_car.as_impact_query(:date => first_day_of_the_month)
@@ -131,9 +131,12 @@ module Carbon
         future.multi!
         future
       end
-      Future.multi(futures).map do |future|
-        future.result
+      Future.multi(futures).inject({}) do |memo, future|
+        memo[future.object] = future.result
+        memo
       end
+    else
+      raise ::ArgumentError, "Didn't match any of the method signatures. If you want multiple queries, make sure to pass an unsplatted Array."
     end
   end
 

data/lib/carbon/future.rb

@@ -3,18 +3,19 @@ require 'net/http'
 require 'cache_method'
 require 'hashie/mash'
 require 'multi_json'
-require 'em-http-request'
 
 module Carbon
   # @private
   class Future
     class << self
       def wrap(query_array_or_o)
-        if query_array_or_o.is_a?(::Array)
+        future = if query_array_or_o.is_a?(::Array)
           new(*query_array_or_o)
         else
           new(*query_array_or_o.as_impact_query)
         end
+        future.object = query_array_or_o
+        future
       end
 
       def single(future)
@@ -26,20 +27,12 @@ module Carbon
 
       def multi(futures)
         uniq_pending_futures = futures.uniq.select { |future| future.pending? }
-        return futures if uniq_pending_futures.empty?
-        pool_size = [Carbon::CONCURRENCY, uniq_pending_futures.length].min
-        multi = ::EventMachine::MultiRequest.new
-        pool = (0..(pool_size-1)).map { ::EventMachine::HttpRequest.new(Carbon::DOMAIN) }
-        pool_idx = 0
-        ::EventMachine.run do
-          uniq_pending_futures.each do |future|
-            multi.add future, pool[pool_idx].post(:path => "/#{future.emitter.underscore.pluralize}.json", :body => future.params)
-            pool_idx = (pool_idx + 1) % pool_size
+        while (set = uniq_pending_futures.pop(Carbon::CONCURRENCY)).any?
+          ts = set.map do |future|
+            ::Thread.new { single future }
           end
-          multi.callback do
-            multi.responses[:callback].each { |future, http| future.finalize http.response_header.status, http.response }
-            multi.responses[:errback].each  { |future, http| future.finalize http.response_header.status }
-            ::EventMachine.stop
+          ts.each do |t|
+            t.join
           end
         end
         futures
@@ -49,6 +42,8 @@ module Carbon
     attr_reader :emitter
     attr_reader :params
 
+    attr_accessor :object
+
     def initialize(emitter, params = {})
       @result = nil
       @emitter = emitter

data/lib/carbon/version.rb

@@ -1,3 +1,3 @@
 module Carbon
-  VERSION = "2.1.0"
+  VERSION = "2.2.1"
 end

data/test/carbon_test.rb

@@ -1,5 +1,6 @@
 require File.expand_path("../helper", __FILE__)
 
+Thread.abort_on_exception = true
 Carbon.key = 'carbon_test'
 
 class MyNissan
@@ -77,9 +78,15 @@ describe Carbon do
           result.errors.first.must_equal 'Good job'
         end
       end
+      it "raises ArgumentError if args are bad" do
+        lambda {
+          Carbon.query(['Flight'])
+        }.must_raise ArgumentError
+      end
     end
     describe '(in parallel)' do
       before do
+        flunk if ENV['SKIP_MULTI'] == 'true'
         @queries = []
         @queries << ['Flight', {:origin_airport => 'LAX', :destination_airport => 'SFO', :segments_per_trip => 1, :trips => 1}]
         @queries << ['Flight', {:origin_airport => 'MSN', :destination_airport => 'ORD', :segments_per_trip => 1, :trips => 1}]
@@ -98,34 +105,50 @@ describe Carbon do
         @queries << ['Monkey', {:bananas => '3'}]
         @queries = @queries.sort_by { rand }
       end
+      it "is easy to use" do
+        flight = ['Flight']
+        rail_trip = ['RailTrip']
+        results = Carbon.query([flight, rail_trip])
+        results[flight].decisions.must_equal Carbon.query('Flight').decisions
+        results[rail_trip].decisions.must_equal Carbon.query('RailTrip').decisions
+      end
       it "doesn't hang up on 0 queries" do
-        Timeout.timeout(0.5) { Carbon.query([]) }.must_equal []
+        Timeout.timeout(0.5) { Carbon.query([]) }.must_equal(Hash.new)
       end
       it "can be used on objects that respond to #as_impact_query" do
-        Carbon.query([MyNissanAltima.new(2001), MyNissanAltima.new(2006)]).map(&:decisions).must_equal Carbon.query([MyNissanAltima.new(2001).as_impact_query, MyNissanAltima.new(2006).as_impact_query]).map(&:decisions)
+        Carbon.query([MyNissanAltima.new(2001), MyNissanAltima.new(2006)]).values.map(&:decisions).map(&:carbon).map(&:object).map(&:value).must_equal Carbon.query([MyNissanAltima.new(2001).as_impact_query, MyNissanAltima.new(2006).as_impact_query]).values.map(&:decisions).map(&:carbon).map(&:object).map(&:value)
       end
       it "runs multiple queries at once" do
-        reference_results = @queries.map do |query|
-          Carbon.query(*query)
+        reference_results = @queries.inject({}) do |memo, query|
+          memo[query] = Carbon.query(*query)
+          memo
         end
-        flush_cache! # important!
-        multi_results = Carbon.query(@queries)
-        error_count = 0
-        multi_results.each do |result|
-          if result.success
-            result.decisions.carbon.object.value.must_be :>, 0
-            result.decisions.carbon.object.value.must_be :<, 10_000
-          else
-            error_count += 1
+        ts = []
+        3.times do
+          ts << Thread.new do
+            flush_cache! # important!
+            multi_results = Carbon.query(@queries)
+            error_count = 0
+            multi_results.each do |query, result|
+              if result.success
+                result.decisions.carbon.object.value.must_be :>, 0
+                result.decisions.carbon.object.value.must_be :<, 10_000
+              else
+                error_count += 1
+              end
+            end
+            error_count.must_equal 3
+            reference_results.each do |query, reference_result|
+              if reference_result.success
+                multi_results[query].decisions.must_equal reference_result.decisions
+              else
+                multi_results[query].must_equal reference_result
+              end
+            end
           end
         end
-        error_count.must_equal 3
-        reference_results.each_with_index do |reference_result, idx|
-          if reference_result.success
-            multi_results[idx].decisions.must_equal reference_result.decisions
-          else
-            multi_results[idx].must_equal reference_result
-          end
+        ts.each do |t|
+          t.join
         end
       end
       it "is faster than single threaded" do
@@ -153,17 +176,18 @@ describe Carbon do
         $stderr.puts "   Cached multi-threaded was #{((multi_threaded_time - cached_multi_threaded_time) / multi_threaded_time * 100).round}% faster than uncached multi-threaded"
       end
       it "safely uniq's and caches queries" do
-        reference_results = @queries.map do |query|
-          Carbon.query(*query)
+        reference_results = @queries.inject({}) do |memo, query|
+          memo[query] = Carbon.query(*query)
+          memo
         end
         flush_cache! # important!
         3.times do
           multi_results = Carbon.query(@queries)
-          reference_results.each_with_index do |reference_result, idx|
+          reference_results.each do |query, reference_result|
             if reference_result.success
-              multi_results[idx].decisions.must_equal reference_result.decisions
+              multi_results[query].decisions.must_equal reference_result.decisions
             else
-              multi_results[idx].must_equal reference_result
+              multi_results[query].must_equal reference_result
             end
           end
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: carbon
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-15 00:00:00.000000000 Z
+date: 2012-03-21 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: em-http-request
-  requirement: &2164986440 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *2164986440
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2165006220 !ruby/object:Gem::Requirement
+  requirement: &2169272660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165006220
+  version_requirements: *2169272660
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &2165005800 !ruby/object:Gem::Requirement
+  requirement: &2169272200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165005800
+  version_requirements: *2169272200
 - !ruby/object:Gem::Dependency
   name: hashie
-  requirement: &2165005380 !ruby/object:Gem::Requirement
+  requirement: &2169271780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165005380
+  version_requirements: *2169271780
 - !ruby/object:Gem::Dependency
   name: cache_method
-  requirement: &2165004960 !ruby/object:Gem::Requirement
+  requirement: &2169271220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165004960
+  version_requirements: *2169271220
 - !ruby/object:Gem::Dependency
   name: bombshell
-  requirement: &2165004500 !ruby/object:Gem::Requirement
+  requirement: &2169270600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165004500
+  version_requirements: *2169270600
 - !ruby/object:Gem::Dependency
   name: conversions
-  requirement: &2165004060 !ruby/object:Gem::Requirement
+  requirement: &2169269920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165004060
+  version_requirements: *2169269920
 - !ruby/object:Gem::Dependency
   name: brighter_planet_metadata
-  requirement: &2165003560 !ruby/object:Gem::Requirement
+  requirement: &2169269500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,7 +87,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2165003560
+  version_requirements: *2169269500
 description: Brighter Planet API client for Ruby
 email:
 - seamus@abshere.net
@@ -118,7 +107,6 @@ files:
 - bin/carbon
 - carbon.gemspec
 - developer/MULTI.markdown
-- developer/REDUCE_HTTP_CONNECTIONS.markdown
 - developer/avro_helper.rb
 - developer/cm1_avro.rb
 - features/shell.feature

data/developer/REDUCE_HTTP_CONNECTIONS.markdown

@@ -1,46 +0,0 @@
-One way to reduce the number of connections by a constant... but it makes it slower (because requests are serialized) and less reliable (because there is a 30s heroku limit)
-
-    def self.multi(queries)
-      unsorted = {}
-      pool_size = (queries.length.to_f / 3).ceil
-      $stderr.puts "Starting #{pool_size} workers"
-      ::EventMachine.run do
-        multi = ::EventMachine::MultiRequest.new
-        pool = 0.upto(pool_size).map do
-          ::EventMachine::HttpRequest.new("http://#{domain}")
-        end
-        pool_idx = 0
-        queries.each_with_index do |(emitter, params), query_idx|
-          params ||= {}
-          multi.add query_idx, pool[pool_idx].post(:path => "/#{emitter.underscore.pluralize}.json", :body => params, :keepalive => true)
-          pool_idx = (pool_idx + 1) % pool_size
-        end
-        multi.callback do
-          multi.responses[:callback].each do |query_idx, http|
-            result = ::Hashie::Mash.new
-            result.status = http.response_header.status
-            if (200..299).include?(result.status)
-              result.success = true
-              result.merge! ::MultiJson.decode(http.response)
-            else
-              result.success = false
-              result.errors = [http.response]
-            end
-            unsorted[query_idx] = result
-          end
-          multi.responses[:errback].each do |query_idx, http|
-            result = ::Hashie::Mash.new
-            result.status = http.response_header.status
-            result.success = false
-            result.errors = ['Timeout or other network error.']
-            unsorted[query_idx] = result
-          end
-          ::EventMachine.stop
-        end
-      end
-      unsorted.sort_by do |query_idx, _|
-        query_idx
-      end.map do |_, result|
-        result
-      end
-    end