shopify_unlimited 0.0.13.threadsafe → 0.0.14.threadsafe

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MWZmMDAyMjRlNjFkMDAxZTg1MDE1OGYzNWVhYmI1NmRlYjQ5OTBiYw==
+    MGJhYTI2MWQ4NjQxNjNlODZmN2I3ZTY3MjEzZGQ0MjIxZGZhZDI2ZQ==
   data.tar.gz: !binary |-
-    MDQyNTNhMTc0ZGI0NDNhZWUyYWM1ZDQ4ZTFlYTdlNDE0ZTY3MGI0YQ==
+    ZTQzMDliNDE3MGZmY2RiY2E5OTRjYjdkZDRmZjYwNjZhNDQ2ZmE4Yw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    N2M0NGQ4ODk5YTU5ZDcxYTU4ZGVlNTFjNmE2ZTdhMmU0YzEyMGE4OTU3ZDUy
-    ZjEwZmJkMzQ4ZjgxOTMyMjJlNjFlMmI3MTIxYzM3ZTA0ZWQzNzkzODhhN2Rh
-    NWY1OWE1NjZiNTFjNDU4NjMyNmNlYjJkMmQzYjM5MWRjYTBjMzg=
+    NjgyNzMxMTE5NWIyOGQxODAyNWI4YTdkZGUwZjBiNTQwOWQzMWJhZDRiYjNk
+    MzA3MDI3YTJiOGI4YjNiODRjN2FlNDAxMjI5NTRhNzE5ODVlYTU3NjhmNWRj
+    MDRjN2RkNmFiNzc1YjRiMWJlN2UwY2QyZTQyMzdlM2U2OWZiNDI=
   data.tar.gz: !binary |-
-    NWE3NTc1YzFlMGQxYjM5ZWZiNDAyZDU2Y2EzMDhiNzdjYzYzNDA1MjQ2NWE0
-    ZmU0YTg2YTI4NWZhNzE4ZDJkYTVjN2JkZDhkYzVlMzdmYjNlYmY1NWY5Zjgz
-    N2ZkYjYwMTE1YjI3MmU2OWRjNmFkY2UxMjVlMzEwMjlmMTA5N2I=
+    ZWE3MjM3ODk3NmY3ZGYzNDZjYzI4ZjU4OGZhNGNmNzU1ZGFkNDk1ZDIwYWFj
+    MjE5OGE2ZjBiNjM1YzFkMjk4NmJhMjQ4YTRhNjVlNWQ3YzU0NTg1MzQzMDY4
+    ZWQxZDEwNzlkMDRjMjc0ZDhiMGMzMjdlYzZlZGM0MzhhNDM5MWU=

data/lib/shopify_unlimited/active_resource/base_extensions.rb

@@ -38,15 +38,18 @@ module ActiveResource
           page = 0
           # as long as the number of results we got back is not less than the limit we (probably) have more to fetch
           while( (results.count - last_count) >= limit) do
-            raise ShopifyAPI::Limits::Error.new if ShopifyAPI.credit_maxed?
             page +=1
             last_count = results.count
             options[:params][:page] = page
-            results.concat find_every.bind(self).call(options)
+            ShopifyAPI::Shop.current.throttle.run do
+              results.concat find_every.bind(self).call(options)
+            end
             results.requests_made += 1
           end
         else
-          results.concat find_every.bind(self).call(options)
+          ShopifyAPI::Shop.current.throttle.run do
+            results.concat find_every.bind(self).call(options)
+          end
           results.requests_made += 1
         end
 

data/lib/shopify_unlimited/shopify_api/throttle.rb

@@ -0,0 +1,75 @@
+# this allows multiple concurrent workers to behave well, almost
+# never triggering a 429, and distributing requests fairly evenly,
+# rather than 1 worker hogging the bulk
+# of requests until finished, which will typically happen with
+# any naive, non-stochastic implementation.
+
+
+module ShopifyAPI
+  class Shop
+    def throttle
+      @throttle ||= Throttle.new
+    end
+  end
+
+  class Throttle
+    attr_accessor :throttle, :throttle_increment, :requests_threshold
+    def initialize
+      @throttle = 0.6
+      @throttle_increment = @throttle
+      @requests_threshold = 10
+    end
+    
+    def run(&block)
+      value = nil
+      retries ||= 0
+      begin
+        left = ShopifyAPI.credit_left
+        over = @requests_threshold - left
+        if over > 0
+          @throttle += (over * rand/20) + rand/10
+          sleep @throttle + rand/10
+        else
+          @throttle = (0.94 + rand/20) * @throttle
+        end
+        t = Time.now
+        value = yield
+      rescue ActiveResource::ClientError => e
+        case e.response.code
+        when '404'
+          logger.fatal "Shopify returned not found"
+          sleep 5 + retries + (rand * rand * 5)
+          retries += 1
+          if retries < 4
+            ActiveResource::Base.logger = Logger.new(STDOUT) 
+            retry
+          else
+            ActiveResource::Base.logger = nil
+            raise
+          end
+        when '429'
+          logger.warn "Shopify hit api limit"
+          @throttle += rand/5
+          retries += 1
+          sleep (@throttle * 4 * retries) + rand/10
+          if retries < 10
+            retry
+          else
+            raise
+          end
+        else
+          raise
+        end
+      end
+      requests_made = left - ShopifyAPI.credit_left
+      if requests_made > 1
+        @throttle += (rand/20) * requests_made
+        sleep [0, (t + (requests_made * @throttle) - Time.now)].max + rand/10
+      else
+        @throttle = (0.94 + rand/20) * @throttle
+        sleep rand/20
+      end
+      value
+    end
+  end
+end

data/lib/shopify_unlimited/version.rb

@@ -1,3 +1,3 @@
 module ShopifyUnlimited
-  VERSION = "0.0.13.threadsafe"
+  VERSION = "0.0.14.threadsafe"
 end

data/lib/shopify_unlimited.rb

@@ -1,46 +1,3 @@
 require "shopify_unlimited/version"
 require 'shopify_unlimited/active_resource/base_extensions'
-require 'shopify_unlimited/active_resource/connection_extensions'
-
-module ShopifyUnlimited
-  SHOPIFY_CREDIT_LIMIT_PERIOD = 5.minutes
-
-  class << self
-    attr_accessor :use_memcached
-    def memcached
-      return nil unless use_memcached
-      @memcached ||= Dalli::Client.new()
-    end
-
-    def cache_key
-      URI.parse(ShopifyAPI::Base.site.to_s).host + "_api_reset_time"
-    end
-
-    def cached_time(max)
-      now = Time.now
-      result = now
-      return result unless memcached
-      unless memcached.add(cache_key, now, 500)
-        memcached.cas(cache_key, 500) do |cached_time|
-          result = ((now - cached_time) > max) ? now : cached_time
-          result
-        end 
-      end
-      result
-    end
-
-    def set_cached_time(time)
-      return unless memcached
-      memcached.set(cache_key, time, 500)
-    end  
-
-
-    def estimated_time_until_reset
-      credit_record = ShopifyAPI::Base.connection.shopify_credit
-      time = credit_record.time unless credit_record.nil?
-      time ||= cached_time(ShopifyUnlimited::SHOPIFY_CREDIT_LIMIT_PERIOD)
-      est = ShopifyUnlimited::SHOPIFY_CREDIT_LIMIT_PERIOD - (Time.now - time)
-      [est, 0].max
-    end
-  end
-end
+require 'shopify_unlimited/shopify_api/throttle'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shopify_unlimited
 version: !ruby/object:Gem::Version
-  version: 0.0.13.threadsafe
+  version: 0.0.14.threadsafe
 platform: ruby
 authors:
 - Michael Johnston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-01 00:00:00.000000000 Z
+date: 2014-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeresource
@@ -109,7 +109,7 @@ files:
 - Rakefile
 - lib/shopify_unlimited.rb
 - lib/shopify_unlimited/active_resource/base_extensions.rb
-- lib/shopify_unlimited/active_resource/connection_extensions.rb
+- lib/shopify_unlimited/shopify_api/throttle.rb
 - lib/shopify_unlimited/version.rb
 - shopify_unlimited.gemspec
 - spec/spec_helper.rb

data/lib/shopify_unlimited/active_resource/connection_extensions.rb

@@ -1,49 +0,0 @@
-module ::ShopifyUnlimited
-
-  class CreditUsed    
-
-    attr_accessor :time, :used
-    def initialize(used)
-      @time = Time.now
-      @used = used
-    end
-
-    def stale?(used)
-      (@used > used) || ((Time.now - @time) > ShopifyUnlimited::SHOPIFY_CREDIT_LIMIT_PERIOD)
-    end
-
-    def estimated_time_until_reset
-      est = ShopifyUnlimited::SHOPIFY_CREDIT_LIMIT_PERIOD - (Time.now - @time)
-      [est, 0].max
-    end
-
-  end
-end
-
-module ::ActiveResource
-  class Connection
-    SHOPIFY_CREDIT_LIMIT_HEADER_PARAM = 'http_x_shopify_shop_api_call_limit'
-
-    attr_reader :shopify_credit
-    def handle_response_with_response_time_capture(response)
-      handle_response_without_response_time_capture(response)
-      
-      if(shopify_credit_header = response[SHOPIFY_CREDIT_LIMIT_HEADER_PARAM])
-        used = shopify_credit_header.split('/').shift.to_i
-
-        if @shopify_credit.nil?
-          cached_time = ::ShopifyUnlimited.cached_time(ShopifyUnlimited::SHOPIFY_CREDIT_LIMIT_PERIOD)
-          @shopify_credit = ::ShopifyUnlimited::CreditUsed.new(used)
-          @shopify_credit.time = cached_time
-        elsif @shopify_credit.stale?(used)
-          @shopify_credit = ::ShopifyUnlimited::CreditUsed.new(used)
-          ::ShopifyUnlimited.set_cached_time(Time.now)
-        else
-          @shopify_credit.used = used
-        end
-      end
-      response
-    end
-    alias_method_chain :handle_response, :response_time_capture
-  end
-end