parse-stack 1.6.0 → 1.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 521333b9d84cd7319911a0e9b368fbfa6133e98e
-  data.tar.gz: 543f3204ab4f4a9df14cbe88aa71be7bd0c69c47
+  metadata.gz: e6f2fcd731c3e5c535c2fcd5b6ad7de0411ee3b8
+  data.tar.gz: d43b5c15e4ca7ff3528d53c09d9f333ea5cca2cd
 SHA512:
-  metadata.gz: 08b3de4ccf9cd5fdaa9045676b1147600bd3698d0cb99bc526d5172defbedc93e039d994f28350c291ad384470b96830d0e5186dbc5b6710255a95445af57a3b
-  data.tar.gz: 206b597203c7df406463f4a778934c7d79cf3ac76a950803c843bd5fd8b9c5020bdda7b6cd69938c1c738034973114759307b25c6ce1f451e998ac3c15b0797a
+  metadata.gz: 7521b1712daddb91201f06062a327443f65467c4098441b68e06de9a53b862c5e2fe36d5f66abb68d974aae098b5a9218315e9de0ab494b47032ad02a81c0a94
+  data.tar.gz: 75e194bbed8d4aebbb8c1ab2fd08f1d4b6c1b1c58191df19bcaf356bfceac761c40874321049e2a83b622b02b68cdf1b167652cac9f796c3f2a1a59aca062297

data/Changes.md

@@ -1,5 +1,13 @@
 ## Parse-Stack Changelog
 
+### 1.6.1
+- NEW: Batch requests are now parallelized.
+- `skip` in queries no longer capped to 10,000.
+- `limit`  in queries no longer capped at 1000.
+- `all()` queries can now return as many results as possible.
+- NEW: `each()` method on Parse::Object subclasses to iterate
+  over all records in the colleciton.
+
 ### 1.6.0
 - NEW: Auto generate models based on your remote schema.
 - The default server url is now 'http://localhost:1337/parse'.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    parse-stack (1.6.0)
+    parse-stack (1.6.1)
       active_model_serializers (>= 0.9, < 1)
       activemodel (>= 4.2.1, < 6)
       activesupport (>= 4.2.1, < 6)
@@ -27,11 +27,10 @@ GEM
       erubis (~> 2.7.0)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    active_model_serializers (0.10.2)
+    active_model_serializers (0.10.3)
       actionpack (>= 4.1, < 6)
       activemodel (>= 4.1, < 6)
-      jsonapi (~> 0.1.1.beta2)
-      railties (>= 4.1, < 6)
+      jsonapi (= 0.1.1.beta2)
     activemodel (5.0.0.1)
       activesupport (= 5.0.0.1)
     activesupport (5.0.0.1)
@@ -53,11 +52,9 @@ GEM
     faraday_middleware (0.10.1)
       faraday (>= 0.7.4, < 1.0)
     i18n (0.7.0)
-    jsonapi (0.1.1.beta6)
-      jsonapi-parser (= 0.1.1.beta3)
-      jsonapi-renderer (= 0.1.1.beta1)
-    jsonapi-parser (0.1.1.beta3)
-    jsonapi-renderer (0.1.1.beta1)
+    json (1.8.3)
+    jsonapi (0.1.1.beta2)
+      json (~> 1.8)
     loofah (2.0.3)
       nokogiri (>= 1.5.9)
     method_source (0.8.2)
@@ -85,15 +82,8 @@ GEM
       nokogiri (~> 1.6.0)
     rails-html-sanitizer (1.0.3)
       loofah (~> 2.0)
-    railties (5.0.0.1)
-      actionpack (= 5.0.0.1)
-      activesupport (= 5.0.0.1)
-      method_source
-      rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
     rake (11.3.0)
     slop (3.6.0)
-    thor (0.19.1)
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)

data/lib/parse/client/batch.rb

@@ -36,7 +36,7 @@ module Parse
   # @see Array.destroy
   class BatchOperation
     include Enumerable
-    
+
     # @!attribute requests
     #  @return [Array] the set of requests in this batch.
 
@@ -130,10 +130,8 @@ module Parse
     def submit(segment = 50)
       @responses = []
       @requests.uniq!(&:signature)
-      @requests.each_slice(segment) do |slice|
-        @responses << client.batch_request( BatchOperation.new(slice) )
-        #throttle
-        # sleep (slice.count.to_f / MAX_REQ_SEC.to_f )
+      @responses = @requests.each_slice(segment).to_a.threaded_map(2) do |slice|
+         client.batch_request( BatchOperation.new(slice) )
       end
       @responses.flatten!
       #puts "Requests: #{@requests.count} == Response: #{@responses.count}"

data/lib/parse/client/caching.rb

@@ -148,7 +148,7 @@ module Parse
             @store.delete "mk:#{url.to_s}" # master key cache-key
             @store.delete @cache_key # final key
           end
-        rescue Errno::EINVAL, Redis::CannotConnectError => e
+        rescue Errno::EINVAL, Redis::CannotConnectError, Redis::TimeoutError => e
           # if the cache store fails to connect, catch the exception but proceed
           # with the regular request, but turn off caching for this request. It is possible
           # that the cache connection resumes at a later point, so this is temporary.
@@ -158,10 +158,15 @@ module Parse
 
         @app.call(env).on_complete do |response_env|
           # Only cache GET requests with valid HTTP status codes whose content-length
-          # is greater than 20. Otherwise they could be errors, successes and empty result sets.
+          # is between 20 bytes and 1MB. Otherwise they could be errors, successes and empty result sets.
+
           if @enabled && method == :get &&  CACHEABLE_HTTP_CODES.include?(response_env.status) &&
-             response_env.present? && response_env.response_headers[CONTENT_LENGTH_KEY].to_i > 20
+              response_env.present? && response_env.response_headers[CONTENT_LENGTH_KEY].to_i.between?(20,1_000_000)
+              begin
                 @store.store(@cache_key, response_env, expires: @expires) # ||= response_env.body
+              rescue => e
+                puts "[Parse::Cache] Store Error: #{e}"
+              end
           end # if
           # do something with the response
           # response_env[:response_headers].merge!(...)

data/lib/parse/model/core/actions.rb

@@ -169,6 +169,54 @@ module Parse
           obj
         end
 
+        # This methods allow you to efficiently iterate over all the records in the collection
+        # (lower memory cost) at a minor cost of performance. This method utilizes
+        # the `created_at` field of Parse records to order and iterate over all records,
+        # therefore you should not use this method if you want to perform a query
+        # with constraints against `created_at` field or need specific type of ordering.
+        # @param constraints [Hash] a set of query constraints.
+        # @yield a block which will iterate through each matching record.
+        # @example
+        #
+        #  post = Post.first
+        #  # iterate over all comments matching conditions
+        #  Comments.each( post: post) do |comment|
+        #    # ...
+        #  end
+        # @return [Parse::Object] the last Parse::Object record processed.
+        def each(constraints = {}, **opts, &block)
+          #anchor_date = opts[:anchor_date] || Parse::Date.now
+          batch_size = 250
+          start_cursor = first( order: :created_at.asc, keys: :created_at )
+          constraints.merge! cache: false, limit: batch_size, order: :created_at.asc
+          all_query = query(constraints)
+          cursor = start_cursor
+          # the exclusion set is a set of ids not to include the next query.
+          exclusion_set = []
+          loop do
+            _q = query(constraints.dup)
+            _q.where(:created_at.on_or_after => cursor.created_at)
+            # set of ids not to include in the next query. non-performant, but accurate.
+            _q.where(:id.nin => exclusion_set) unless exclusion_set.empty?
+            results = _q.results # get results
+
+            break cursor if results.empty? # break if no results
+            results.each(&block)
+            next_cursor = results.last
+            # break if we got less than the maximum requested
+            break next_cursor if results.count < batch_size
+            # break if the next object is the same as the current object.
+            break next_cursor if cursor.id == next_cursor.id
+            # The exclusion set is used in the case where multiple records have the exact
+            # same created_at date (down to the microsecond). This prevents getting the same
+            # record in the next query request.
+            exclusion_set = results.select { |r| r.created_at == next_cursor.created_at }.map(&:id)
+            results = nil
+            cursor = next_cursor
+          end
+
+        end
+
         # Auto save all objects matching the query constraints. This method is
         # meant to be used with a block. Any objects that are modified in the block
         # will be batched for a save operation. This uses the `updated_at` field to
@@ -185,7 +233,7 @@ module Parse
         # @return [Boolean] whether there were any errors.
         def save_all(constraints = {})
           force = false
-
+          batch_size = 250
           iterator_block = nil
           if block_given?
             iterator_block = Proc.new
@@ -202,7 +250,7 @@ module Parse
           constraints.merge! :updated_at.on_or_before => anchor_date
           constraints.merge! cache: false
           # oldest first, so we create a reduction-cycle
-          constraints.merge! order: :updated_at.asc, limit: 100
+          constraints.merge! order: :updated_at.asc, limit: batch_size
           update_query = query(constraints)
           #puts "Setting Anchor Date: #{anchor_date}"
           cursor = nil
@@ -231,6 +279,7 @@ module Parse
             # cursor_item = results.max_by(&updated_comparison_block).updated_at
             # puts "[Parse::SaveAll] Updated #{results.count} records updated <= #{cursor.updated_at}"
 
+            break if results.count < batch_size # we didn't hit a cap on results.
             if cursor.is_a?(Parse::Object)
               update_query.where :updated_at.gte => cursor.updated_at
 

data/lib/parse/query.rb

@@ -386,16 +386,14 @@ module Parse
       self #chaining
     end #order
 
-    # Use with limit to paginate through results. Default is 0 with
-    # maximum value being 10,000.
+    # Use with limit to paginate through results. Default is 0.
     # @example
     #  # get the next 3 songs after the first 10
     #  Song.all :limit => 3, :skip => 10
-    # @param count [Integer] The number of records to skip.
+    # @param amount [Integer] The number of records to skip.
     # @return [self]
-    def skip(count)
-      #  min <= count <= max
-      @skip = [ 0, count.to_i, 10_000].sort[1]
+    def skip(amount)
+      @skip = [0,amount.to_i].max
       @results = nil
       self #chaining
     end
@@ -403,21 +401,21 @@ module Parse
     # Limit the number of objects returned by the query. The default is 100, with
     # Parse allowing a maximum of 1000. The framework also allows a value of
     # `:max`. Utilizing this will have the framework continually intelligently
-    # utilize `:skip` to continue to paginate through results until an empty
-    # result set is received or the `:skip` limit is reached (10,000). When
-    # utilizing `all()`, `:max` is the default option for `:limit`.
+    # utilize `:skip` to continue to paginate through results until no more results
+    # match the query criteria. When utilizing `all()`, `:max` is the default
+    # option for `:limit`.
     # @example
     #  Song.all :limit => 1 # same as Song.first
-    #  Song.all :limit => 1000 # maximum allowed by Parse
-    #  Song.all :limit => :max # up to 11,000 records (theoretical).
+    #  Song.all :limit => 2025 # large limits supported.
+    #  Song.all :limit => :max # as many records as possible.
     # @param count [Integer,Symbol] The number of records to return. You may pass :max
     #  to get as many as 11_000 records with the aid if skipping.
     # @return [self]
     def limit(count)
-      if count == :max
-        @limit = 11_000
-      elsif count.is_a?(Numeric)
-        @limit = [ 0, count.to_i, 11_000].sort[1]
+      if count.is_a?(Numeric)
+        @limit = [ 0, count.to_i ].max
+      elsif count == :max
+        @limit = :max
       else
         @limit = nil
       end
@@ -643,32 +641,51 @@ module Parse
     # max_results is used to iterate through as many API requests as possible using
     # :skip and :limit paramter.
     # @!visibility private
-    def max_results(raw: false)
+    def max_results(raw: false, on_batch: nil, discard_results: false)
       compiled_query = compile
-      query_limit = compiled_query[:limit] ||= 1_000
-      query_skip =  compiled_query[:skip] ||= 0
-      compiled_query[:limit] = 1_000
-      iterations = (query_limit/1000.0).ceil
+      batch_size = 1_000
       results = []
+      # determine if there is a user provided hard limit
+      _limit = (@limit.is_a?(Numeric) && @limit > 0) ? @limit : nil
+      compiled_query[:skip] ||= 0
+
+      loop do
+        # always reset the batch size
+        compiled_query[:limit] = batch_size
+
+        # if a hard limit was set by the user, then if the remaining amount
+        # is less than the batch size, set the new limit to the remaining amount.
+        unless _limit.nil?
+          compiled_query[:limit] = _limit if _limit < batch_size
+        end
 
-      iterations.times do |idx|
         response = fetch!( compiled_query )
         break if response.error? || response.results.empty?
 
         items = response.results
         items = decode(items) unless raw
-
+        # if a block is provided, we do not keep the results after processing.
         if block_given?
           items.each(&Proc.new)
         else
-          results += items
+          # concat results unless discard_results is true
+          results += items unless discard_results
         end
+
+        on_batch.call(items) if on_batch.present?
         # if we get less than the maximum set of results, most likely the next
         # query will return emtpy results - no need to perform it.
         break if items.count < compiled_query[:limit]
+
+        # if we have a set limit, then subtract from the total amount the user requested
+        # from the total in the current result set. Break if we've reached our limit.
+        unless _limit.nil?
+          _limit -= items.count
+          break if _limit < 1
+        end
+
         # add to the skip count for the next iteration
-        compiled_query[:skip] += 1_000
-        break if compiled_query[:skip] > 10_000
+        compiled_query[:skip] += batch_size
       end
       results
     end
@@ -722,14 +739,14 @@ module Parse
     # @return [Array<Parse::Object>] if raw is set to false, a list of matching Parse::Object subclasses.
     def results(raw: false)
       if @results.nil?
-        if @limit.nil? || @limit.to_i <= 1_000
+        if block_given?
+          max_results(raw: raw, &Proc.new)
+        elsif @limit.is_a?(Numeric)
           response = fetch!( compile )
           return [] if response.error?
           items = raw ? response.results : decode(response.results)
           return items.each(&Proc.new) if block_given?
           @results = items
-        elsif block_given?
-          return max_results(raw: raw, &Proc.new)
         else
           @results = max_results(raw: raw)
         end
@@ -780,7 +797,6 @@ module Parse
     def compile(encode: true, includeClassName: false)
       run_callbacks :prepare do
         q = {} #query
-        q[:limit] = 11_000 if @limit == :max
         q[:limit] = @limit if @limit.is_a?(Numeric) && @limit > 0
         q[:skip] = @skip if @skip > 0
 

data/lib/parse/stack/version.rb

@@ -6,6 +6,6 @@ module Parse
   # The Parse Server SDK for Ruby
   module Stack
     # The current version.
-    VERSION = "1.6.0"
+    VERSION = "1.6.1"
   end
 end

data/lib/parse/webhooks/registration.rb

@@ -93,7 +93,7 @@ module Parse
           classNames = routes[trigger].keys.dup
           if include_wildcard && classNames.include?('*') #then create the list for all classes
             classNames.delete '*' #delete the wildcard before we expand it
-            classNames = classNames + Parse.registered_classes - ['_Session']
+            classNames = classNames + Parse.registered_classes
             classNames.uniq!
           end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parse-stack
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.6.1
 platform: ruby
 authors:
 - Anthony Persaud
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-24 00:00:00.000000000 Z
+date: 2016-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -274,7 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.6
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Parse-Server Ruby Client and Relational Mapper