shopify_unlimited 0.0.16.threadsafe → 0.0.17.threadsafe

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZjE1N2RjYWE2NTk0OTZmNzA1NmZmMzE3NTFjYTM4MzE3YjFkN2RiMg==
-  data.tar.gz: !binary |-
-    MGE0N2VhMTcxYjIwZTRkMDZlMzA0Nzk0NWI1Yzk2MzFlMTgzY2UxMQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    OWFkYzVhMmNhODAxYjA4OGFiNDExM2NjOTIyMjQxNTk2MDdiMjg0NWQwNThh
-    MjUwZjIyZmYzOTMwZTRmNGYxZGNjN2I5MWE5MGZlOGI5Yzk0MGRhM2NkYzMx
-    NWJhYTY0ZjU0NTQ3OGZhMmY1YTkyNTY3NWYwMmEwNThmNTNjMGE=
-  data.tar.gz: !binary |-
-    OTJlMTNmN2YyYWM0OGIyMjg1MDNjYzVhMThhMWNhZGE3YjZiNGExNDUzMjc4
-    OTc1NWUwZmRkOTY2NTI3ODU1NjVmNTQzNzlmZTc4YTY2MjM1MDY2YTgwMjhi
-    ZGY5YWYwZjQ0NDM2MGNlZjliZjJkZWRiOTYxZWY5ZWE5YTEzZTI=
+SHA1:
+  metadata.gz: aa8f62c2ce2a1f1dcbf25df03bd1210234b7d76b
+  data.tar.gz: 326c1b9728cee6f230768c50b0528a325b04f44a
+SHA512:
+  metadata.gz: 0ae8d3d2dbed282c326976ab386d7ce31eb28db82ecb94e1a0560b37c470d35187a20b587ffdd73e5f0331d0a9487e2ee3f04ba34bfa6bf4143b3cb79ac30909
+  data.tar.gz: b5495e4f699511dce761a08fe72a6d244d2f195504cc508c11bb16ce25c1cb642294dd1219cabe96017c0f5a9bca85c87087fa1990d2f35cd32c4cf6e10eaa4d

data/lib/shopify_unlimited/active_resource/base_extensions.rb

@@ -41,17 +41,13 @@ module ActiveResource
             page +=1
             last_count = results.count
             options[:params][:page] = page
-            ShopifyAPI::Shop.current.throttle.run do
-              next_result = find_every.bind(self).call(options)
-              results.concat next_result unless next_result.nil?
-            end
-            results.requests_made += 1
-          end
-        else
-          ShopifyAPI::Shop.current.throttle.run do
             next_result = find_every.bind(self).call(options)
             results.concat next_result unless next_result.nil?
+            results.requests_made += 1
           end
+        else
+          next_result = find_every.bind(self).call(options)
+          results.concat next_result unless next_result.nil?
           results.requests_made += 1
         end
 

data/lib/shopify_unlimited/active_resource/connection_extensions.rb

@@ -0,0 +1,19 @@
+module ShopifyUnlimited
+  module ThrottledConnection
+    def throttle
+      @throttle ||= ShopifyUnlimited::Throttle.new
+    end
+    def with_auth
+      throttle.run do
+        super
+      end
+    end
+  end
+end
+
+module ActiveResource
+  class Connection
+    prepend ShopifyUnlimited::ThrottledConnection
+  end
+end
+

data/lib/shopify_unlimited/{shopify_api/throttle.rb → throttle.rb}

@@ -4,50 +4,50 @@
 # of requests until finished, which will typically happen with
 # any naive, non-stochastic implementation.
 
-# currently this is only true when only one request at a time
-# is passed to throttle.run
-# When it becomes useful, we could add an argument to throttle.run
-# to specify the number of expected requests. However, that would have
-# has the downside that the thread might end up waiting a long time 
-# for a large enough block of requests.
-
-module ShopifyAPI
-  
-  class Shop
-    def throttle
-      @throttle ||= Throttle.new
-    end
-  end
-
+module ShopifyUnlimited
   class Throttle
-    attr_accessor :throttle, :throttle_increment, :requests_threshold
+    attr_accessor :throttle, :throttle_increment, :requests_threshold, :running
     def initialize
       @throttle = 0.6
       @throttle_increment = @throttle
       @requests_threshold = 10
+      @debug = ! ENV['SHOPIFY_UNLIMITED_DEBUG'].blank?
     end
     
-    def run(&block)
+    def dbg(msg)
+      puts msg if @debug
+    end
+    
+    def run
+      if @running
+        return yield
+      end
+      @running = true
       value = nil
       retries ||= 0
       orig_logger = ActiveResource::Base.logger
       begin
-        left = ShopifyAPI.credit_left
+        dbg "--- throttle.run ---"
+        left = ShopifyAPI::Base.connection.response.nil? ? @requests_threshold : ShopifyAPI.credit_left
         over = @requests_threshold - left
+        dbg "left: #{left} over: #{over}"
         if over > 0
           @throttle += (over * rand/20) + rand/10
+          dbg "  sleep #{@throttle + rand/10}"
           sleep @throttle + rand/10
         else
           @throttle = (0.94 + rand/20) * @throttle
         end
         t = Time.now
+        dbg "  yield"
         value = yield
       rescue ActiveResource::ClientError => e
         case e.response.code
         when '404'
+          dbg "  404"
           sleep 5 + retries + (rand * rand * 5)
           retries += 1
-          if retries < 4
+          if retries < 2
             ActiveResource::Base.logger ||= Logger.new(STDOUT)
             ActiveResource::Base.logger.info "Shopify returned not found: #{e.message}. Retrying"
             retry
@@ -56,6 +56,7 @@ module ShopifyAPI
             raise
           end
         when '429'
+          dbg "  429"
           ActiveResource::Base.logger.info "Shopify hit api limit" if ActiveResource::Base.logger
           @throttle += rand/5
           retries += 1
@@ -71,13 +72,20 @@ module ShopifyAPI
       end
       requests_made = left - ShopifyAPI.credit_left
       if requests_made > 1
+        dbg "  #{requests_made} requests_made"
         @throttle += (rand/20) * requests_made
-        sleep [0, (t + (requests_made * @throttle) - Time.now)].max + rand/10
+        fuzzy_sleep_time = [0, (t + (requests_made * @throttle) - Time.now)].max + rand/10
+        dbg "  sleep #{fuzzy_sleep_time}"
+        sleep fuzzy_sleep_time
       else
         @throttle = (0.94 + rand/20) * @throttle
+        dbg "  sleep #{rand/20}"
         sleep rand/20
       end
+      dbg "=== throttle.done ==="
       value
+    ensure
+      @running = false
     end
   end
 end

data/lib/shopify_unlimited/version.rb

@@ -1,3 +1,3 @@
 module ShopifyUnlimited
-  VERSION = "0.0.16.threadsafe"
+  VERSION = "0.0.17.threadsafe"
 end

data/lib/shopify_unlimited.rb

@@ -1,3 +1,4 @@
 require "shopify_unlimited/version"
+require 'shopify_unlimited/active_resource/connection_extensions'
 require 'shopify_unlimited/active_resource/base_extensions'
-require 'shopify_unlimited/shopify_api/throttle'
+require 'shopify_unlimited/throttle'

metadata

@@ -1,107 +1,107 @@
 --- !ruby/object:Gem::Specification
 name: shopify_unlimited
 version: !ruby/object:Gem::Version
-  version: 0.0.16.threadsafe
+  version: 0.0.17.threadsafe
 platform: ruby
 authors:
 - Michael Johnston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-02 00:00:00.000000000 Z
+date: 2014-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeresource
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: shopify_api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 3.2.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: ! 'allow use of limit: false in api requests'
+description: 'allow use of limit: false in api requests'
 email:
 - lastobelus@mac.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - Guardfile
 - LICENSE.txt
@@ -109,7 +109,8 @@ files:
 - Rakefile
 - lib/shopify_unlimited.rb
 - lib/shopify_unlimited/active_resource/base_extensions.rb
-- lib/shopify_unlimited/shopify_api/throttle.rb
+- lib/shopify_unlimited/active_resource/connection_extensions.rb
+- lib/shopify_unlimited/throttle.rb
 - lib/shopify_unlimited/version.rb
 - shopify_unlimited.gemspec
 - spec/spec_helper.rb
@@ -123,19 +124,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>'
+  - - ">"
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: ! 'allow use of limit: false in api requests'
+summary: 'allow use of limit: false in api requests'
 test_files:
 - spec/spec_helper.rb