shopify_unlimited 0.0.17.threadsafe → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa8f62c2ce2a1f1dcbf25df03bd1210234b7d76b
-  data.tar.gz: 326c1b9728cee6f230768c50b0528a325b04f44a
+  metadata.gz: 6330aa14a7c439f7200721b4af410b9f7a099be8
+  data.tar.gz: 5d3379107c2df7a6d24af4c10e8d315a89f70222
 SHA512:
-  metadata.gz: 0ae8d3d2dbed282c326976ab386d7ce31eb28db82ecb94e1a0560b37c470d35187a20b587ffdd73e5f0331d0a9487e2ee3f04ba34bfa6bf4143b3cb79ac30909
-  data.tar.gz: b5495e4f699511dce761a08fe72a6d244d2f195504cc508c11bb16ce25c1cb642294dd1219cabe96017c0f5a9bca85c87087fa1990d2f35cd32c4cf6e10eaa4d
+  metadata.gz: ba55b1d9a242f79fd7de2d364841c1a7fe660d338e973dfc0910e88248f25321cef6a9f03e4232d5e3f748ef4452883d081acada92cdf0a3a76c283a8350d23c
+  data.tar.gz: 2c421eeef8d843bfe5dc0d8d3d91c2c0f90cda817e27a9a9f96b036dfc21e742f26d349ada86a5f3a655a2d870ab333293735b6d24dd776ff47da7c83bf89540

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+tags

data/README.md

@@ -1,8 +1,9 @@
 # ShopifyUnlimited
 
 1. Allows API calls like products = Product.find(:all, params:{limit: false}), which will make as many requests as needed to fetch all Products in batches of 250. The result set can be queried for the number of requests required: ```products.requests_made```
-2. 
-2. Attempts to record api limits a little better by adding ```ShopifyAPI::Base.connection.shopify_credit```. This object contains the credit used since the first request (in a given process) and the time of the first request. It resets when the credit used goes down or the time elapsed has been more than 5 minutes.
+2.Contains a throttle tuned for 4-8 workers consuming a single shop's api limit. It will keep all workers busy while limiting the number of refused requests.
+3. Retries 404's once. This is because Shopify in the past has occasionally 404'd on count queries.
+
 
 ## Installation
 
@@ -18,6 +19,11 @@ Or install it yourself as:
 
     $ gem install shopify_unlimited
 
+
+## Usage
+1. to fetch all records using behind the scenes batching, use find with limit: false
+2. to see the throttle in action, set ENV['SHOPIFY_UNLIMITED_DEBUG'] and it will log info to STDOUT
+
 ## Contributing
 
 1. Fork it

data/lib/shopify_unlimited/active_resource/base_extensions.rb

@@ -7,19 +7,36 @@ end
 
 
 module ActiveResource
-  class Base     
+  class Base
     SHOPIFY_MAX_RECORDS_PER_REQUEST = 250
-    
+
+    def self.find_all(params = {}, &block)
+      records = 0
+      params[:limit] ||= 50
+      params[:page] = 1
+
+      begin
+        page = find(:all, :params => params)
+        return records if page.nil?
+        page.each do |value|
+          records += 1
+          block.call(value)
+        end
+        params[:page] += 1
+      end until page.length < params[:limit]
+      records
+    end
+
     class << self
       # get reference to unbound class-method #find_every
       find_every = self.instance_method(:find_every)
 
       define_method(:find_every) do |options|
         options[:params] ||= {}
-        
+
         # Determine number of ShopifyAPI requests to stitch together all records of this query.
         limit = options[:params][:limit]
-        
+
 
         results = []
         results.singleton_class.class_eval do
@@ -30,7 +47,7 @@ module ActiveResource
         # Bail out to default functionality unless limit == false
         # NOTE: the algorithm was switched from doing a count and pre-calculating pages
         # because Shopify 404s on some count requests
-        if limit == false          
+        if limit == false
           options[:params].update(:limit => SHOPIFY_MAX_RECORDS_PER_REQUEST)
 
           limit = SHOPIFY_MAX_RECORDS_PER_REQUEST
@@ -51,8 +68,8 @@ module ActiveResource
           results.requests_made += 1
         end
 
-        results                  
+        results
       end
-    end      
+    end
   end
 end

data/lib/shopify_unlimited/throttle.rb

@@ -12,19 +12,21 @@ module ShopifyUnlimited
       @throttle_increment = @throttle
       @requests_threshold = 10
       @debug = ! ENV['SHOPIFY_UNLIMITED_DEBUG'].blank?
+      @not_found_retries = Integer(ENV['SHOPIFY_UNLIMITED_NOTFOUND_RETRIES']) rescue 1
     end
-    
+
     def dbg(msg)
       puts msg if @debug
     end
-    
+
     def run
       if @running
         return yield
       end
       @running = true
       value = nil
-      retries ||= 0
+      tries ||= 0
+      not_found_tries ||= 0
       orig_logger = ActiveResource::Base.logger
       begin
         dbg "--- throttle.run ---"
@@ -45,9 +47,9 @@ module ShopifyUnlimited
         case e.response.code
         when '404'
           dbg "  404"
-          sleep 5 + retries + (rand * rand * 5)
-          retries += 1
-          if retries < 2
+          sleep 5 + not_found_tries + (rand * rand * 5)
+          not_found_tries += 1
+          if not_found_tries <= @not_found_retries
             ActiveResource::Base.logger ||= Logger.new(STDOUT)
             ActiveResource::Base.logger.info "Shopify returned not found: #{e.message}. Retrying"
             retry
@@ -59,9 +61,9 @@ module ShopifyUnlimited
           dbg "  429"
           ActiveResource::Base.logger.info "Shopify hit api limit" if ActiveResource::Base.logger
           @throttle += rand/5
-          retries += 1
-          sleep (@throttle * 4 * retries) + rand/10
-          if retries < 10
+          tries += 1
+          sleep (@throttle * 4 * tries) + rand/10
+          if tries < 10
             retry
           else
             raise
@@ -79,8 +81,6 @@ module ShopifyUnlimited
         sleep fuzzy_sleep_time
       else
         @throttle = (0.94 + rand/20) * @throttle
-        dbg "  sleep #{rand/20}"
-        sleep rand/20
       end
       dbg "=== throttle.done ==="
       value

data/lib/shopify_unlimited/version.rb

@@ -1,3 +1,3 @@
 module ShopifyUnlimited
-  VERSION = "0.0.17.threadsafe"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: shopify_unlimited
 version: !ruby/object:Gem::Version
-  version: 0.0.17.threadsafe
+  version: 0.1.0
 platform: ruby
 authors:
 - Michael Johnston
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-10 00:00:00.000000000 Z
+date: 2017-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeresource
@@ -129,12 +129,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: 'allow use of limit: false in api requests'