hoodoo 1.6.1 → 1.7.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODNjYzEzYzhiMTY1OTlkMjBkNDBlMmQ0MzNhYzAyZDJhNTFlMzBhNA==
-  data.tar.gz: !binary |-
-    M2I5ZmQyMmU0YTljY2JkNmE1ZGU4YzhhMzdmMzcxMDVmMGZiNzY2Nw==
+SHA1:
+  metadata.gz: 21732b5cd34041936e018495404cb3950ff70b85
+  data.tar.gz: b37d1a9ef5e225653be451d8e775ea07cf5a9690
 SHA512:
-  metadata.gz: !binary |-
-    NGY3NjZhYzM5MDVkODhkMGQ5MzQ5NzY2MmQ1Y2I0YTRlYWI1NDI4MGU0YTc5
-    NjJkNDZmODE4NTRkODA3ZGY0N2M0OWY5NmZjNTFiNzk0YzQxODVmM2RlZDc4
-    YmViMDFkMWY0MzZlNThkMzk0MWY5ZDRjM2Q1MTRlMTI5YmY4NmE=
-  data.tar.gz: !binary |-
-    NTQ4OWZjYjA5Y2ZhOWIzN2NkODM4NWUwZTc5MDdiOGE0OTVhNGMxNjY5NzIy
-    MTgwYWExNDk4M2NjNWQ4MDU3ZDJjNzk2Y2ZjYmQyNmU1Y2M2NzhhMDAxNDc3
-    YjkyNmIxYTZjZDNlZGNhMDRmNTc2MDQxNWI3NjFlMDg3ZTIzNzQ=
+  metadata.gz: 39c52f8db587f906e2c377ab9fae3939697b872366c07aace735439538f6412fe12efae8996652b746ab31e8704334329e5d71125ca14c60be978f9352aa0308
+  data.tar.gz: bc4e9624bff7ad092b1ca7e0f86e64c2b8c634ee12649fd22bbb27de26d360dd870dfe4d7644639b5cc30fabbd30294e13c004e9efde77cff07f35fe3879a2eb

data/lib/hoodoo/active/active_record/finder.rb

@@ -53,6 +53,7 @@ module Hoodoo
         model.class_attribute(
           :nz_co_loyalty_hoodoo_show_id_fields,
           :nz_co_loyalty_hoodoo_show_id_substitute,
+          :nz_co_loyalty_hoodoo_estimate_counts_with,
           :nz_co_loyalty_hoodoo_search_with,
           :nz_co_loyalty_hoodoo_filter_with,
           {
@@ -373,7 +374,12 @@ module Hoodoo
         #     end
         #
         # Note the use of helper method #dataset_size to count the total
-        # amount of results in the dataset without pagination.
+        # amount of results in the dataset without pagination. A resource may
+        # alternatively choose to use #estimated_dataset_size for a fast count
+        # estimation, or neither (though this is generally not recommended) or
+        # - permissible but unusual - include both.
+        #
+        #     context.response.set_resources( results, nil, finder.estimated_dataset_size )
         #
         # The service middleware enforces sane values for things like list
         # offsets, sort keys and so-on according to service interface
@@ -495,7 +501,137 @@ module Hoodoo
         # +dataset_size+ parameter.
         #
         def dataset_size
-          return all.limit( nil ).offset( nil ).count
+          return all.limit( nil ).offset( nil ).count()
+        end
+
+        # As #dataset_size, but allows a configurable counting back-end via
+        # #estimated_count and #estimate_counts_with. This method is intended
+        # to be used for fast count estimations, usually for performance
+        # reasons if an accurate #dataset_size count is too slow to compute.
+        #
+        def estimated_dataset_size
+          return all.limit( nil ).offset( nil ).estimated_count()
+        end
+
+        # In absence of other configuration, this method just calls through
+        # to Active Record's #count, but you can override the counting
+        # mechanism with a Proc which gets called to do the counting instead.
+        #
+        # The use case is for databases where counting may be slow for some
+        # reason. For example, in PostgreSQL 9, the MVCC model means that big
+        # tables under heavy write load may take extremely long times to be
+        # counted as a full sequential row scan gets activated. In the case
+        # of PostgreSQL, there's an estimation available as an alternative;
+        # its accuracy depends on how often the +ANALYZE+ command is run, but
+        # at least its execution speed is always very small.
+        #
+        # The #estimated_dataset_size method runs through here for counting so
+        # you need to ensure that your count estimation method can cope with
+        # whatever queries that might arise from the scope chains involved in
+        # instances of the model at hand, within the service code that uses
+        # that model.
+        #
+        # Specify a count estimation Proc with #estimate_counts_with. Such
+        # blocks are permitted to return +nil+ if the estimation is considered
+        # to be wildly wrong or unobtainable; in that case, the returned value
+        # for the estimated count will be +nil+ too.
+        #
+        def estimated_count
+          counter = self.nz_co_loyalty_hoodoo_estimate_counts_with
+
+          if ( counter.nil? )
+            return all.count
+          else
+            return counter.call( all.to_sql )
+          end
+        end
+
+        # This method is related to #estimated_count, so read the documentation
+        # for that as an introduction first.
+        #
+        # In #estimated_count, a PostgreSQL example is given. Continuing with
+        # this, we could implement an estimation mechanism via Hoodoo's fast
+        # counter with something like the approach described here:
+        #
+        # * https://wiki.postgresql.org/wiki/Count_estimate
+        # * http://www.verygoodindicators.com/blog/2015/04/07/faster-count-queries/
+        #
+        # First, you would need a migration in your service to implement the
+        # estimation method as a PLPGSQL function:
+        #
+        #     class CreateFastCountFunction < ActiveRecord::Migration
+        #       def up
+        #         execute <<-SQL
+        #           CREATE FUNCTION estimated_count(query text) RETURNS integer AS
+        #           $func$
+        #           DECLARE
+        #               rec   record;
+        #               rows  integer;
+        #           BEGIN
+        #               FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        #                   rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        #                   EXIT WHEN rows IS NOT NULL;
+        #               END LOOP;
+        #
+        #               RETURN rows;
+        #           END
+        #           $func$ LANGUAGE plpgsql;
+        #         SQL
+        #       end
+        #
+        #       def down
+        #         execute "DROP FUNCTION estimated_count(query text);"
+        #       end
+        #     end
+        #
+        # This takes arbitrary query text so should cope with pretty much any
+        # kind of ActiveRecord query chain and resulting SQL. Run the database
+        # migration, then define a Proc which calls the new function:
+        #
+        #     counter = Proc.new do | sql |
+        #       begin
+        #         sql = sql.gsub( "'", "''" ) # Escape SQL for insertion below
+        #         ActiveRecord::Base.connection.execute(
+        #           "SELECT estimated_count('#{ sql }')"
+        #         ).first[ 'estimated_count' ].to_i
+        #       rescue
+        #         nil
+        #     end
+        #
+        # Suppose we have a model called +Purchase+; next tell this model to
+        # use the above Proc for fast counting and use it:
+        #
+        #     Purchase.estimate_counts_with( counter )
+        #
+        #     Purchase.estimated_count()
+        #     # => An integer; and you can use scope chains, just like #count:
+        #     Purchase.where(...conditions...).estimated_count()
+        #     # => An integer
+        #
+        # A real-life example showing how running PostgreSQL's +ANALYZE+
+        # command can make a difference:
+        #
+        #     [1] pry(main)> Purchase.estimated_count
+        #     => 68
+        #     [2] pry(main)> Purchase.count
+        #     => 76
+        #     [3] pry(main)> ActiveRecord::Base.connection.execute( 'ANALYZE' )
+        #     => #<PG::Result:0x007f89b62cdcc8 status=PGRES_COMMAND_OK ntuples=0 nfields=0 cmd_tuples=0>
+        #     [4] pry(main)> Purchase.estimated_count
+        #     => 76
+        #
+        # Parameters:
+        #
+        # +proc+:: The Proc to call. It must accept one parameter, which is the
+        #          SQL query for which the count is to be run, as a String. It
+        #          must evaluate to an Integer estimation, or +nil+ if it is
+        #          not able to provide any/useful estimations, in its opinion.
+        #
+        #          Pass +nil+ to remove the custom counter method and restore
+        #          default behaviour.
+        #
+        def estimate_counts_with( proc )
+          self.nz_co_loyalty_hoodoo_estimate_counts_with = proc
         end
 
         # Specify a search mapping for use by #list to automatically restrict

data/lib/hoodoo/client/augmented_array.rb

@@ -20,9 +20,18 @@ module Hoodoo
 
       # For lists, the (optional) total size of the data set, of which
       # the contents of this Array will often only represent a single
-      # page. If unknown, the value is +nil+.
+      # page. If unknown, the value is +nil+, but as an alternative, an
+      # estimated size may be available in #estimated_dataset_size.
       #
       attr_accessor :dataset_size
+
+      # For lists, the (optional) estimated size of the data set, of
+      # which the contents of this Array will often only represent a
+      # single page. If unknown, the value is +nil+. The accuracy of
+      # the estimation is unknown.
+      #
+      attr_accessor :estimated_dataset_size
+
     end
 
   end

data/lib/hoodoo/client/client.rb

@@ -76,7 +76,8 @@ module Hoodoo
       # call and supports Hoodoo::Client::AugmentedArray#dataset_size which
       # (if the called Resource endpoint implementation provides the
       # information) gives the total size of the data set at the time of
-      # calling.
+      # calling. Hoodoo::Client::AugmentedArray#estimated_dataset_size
+      # likewise gives access to the estimated count, if available.
       #
       # The other 4 methods return a Hoodoo::Client::AugmentedHash. This is a
       # Hash subclass. Both the Array and Hash subclasses provide a common

data/lib/hoodoo/client/endpoint/endpoints/http_based.rb

@@ -325,9 +325,12 @@ module Hoodoo
             # part, else the hash part.
 
             if ( parsed[ '_data' ].is_a?( ::Array ) )
-              size   = parsed[ '_dataset_size' ]
-              parsed = parsed[ '_data'         ]
-              parsed.dataset_size = size
+              size           = parsed[ '_dataset_size'           ]
+              estimated_size = parsed[ '_estimated_dataset_size' ]
+
+              parsed                        = parsed[ '_data' ]
+              parsed.dataset_size           = size
+              parsed.estimated_dataset_size = estimated_size
 
             elsif ( parsed[ 'kind' ] == 'Errors' )
 

data/lib/hoodoo/services/middleware/middleware.rb

@@ -923,8 +923,9 @@ module Hoodoo; module Services
         body = local_response.body
 
         if action == :list && body.is_a?( ::Array )
-          result              = Hoodoo::Client::AugmentedArray.new( body )
-          result.dataset_size = local_response.dataset_size
+          result                        = Hoodoo::Client::AugmentedArray.new( body )
+          result.dataset_size           = local_response.dataset_size
+          result.estimated_dataset_size = local_response.estimated_dataset_size
 
         elsif action != :list && body.is_a?( ::Hash )
           result = Hoodoo::Client::AugmentedHash[ body ]
@@ -1947,8 +1948,24 @@ module Hoodoo; module Services
     #                 describing the current interaction. Updated on exit.
     #
     def deal_with_content_type_header( interaction )
-      content_type      = interaction.rack_request.media_type
-      content_encoding  = interaction.rack_request.content_charset
+
+      # An in-the-wild Content-Type header value of
+      # "application/json; charset=utf-8, application/x-www-form-urlencoded"
+      # from Postman caused Rack 1.6.4 to break and raise an exception. Trap
+      # any exceptions from the Rack request calls below and assume that they
+      # indicate a malformed header.
+      #
+      begin
+        content_type     = interaction.rack_request.media_type
+        content_encoding = interaction.rack_request.content_charset
+      rescue
+        interaction.context.response.errors.add_error(
+          'platform.malformed',
+          'message' => "Content-Type '#{ interaction.rack_request.content_type || "<unknown>" }' is malformed"
+        )
+
+        return
+      end
 
       content_type.downcase!     unless content_type.nil?
       content_encoding.downcase! unless content_encoding.nil?

data/lib/hoodoo/services/services/response.rb

@@ -71,13 +71,21 @@ module Hoodoo; module Services
     attr_accessor :body
     alias_method  :set_resource, :body=
 
-    # Read back a the dataset size given by a prior call to #set_resources,
+    # Read back a dataset size given by a prior call to #set_resources,
     # or +nil+ if none has been provided (either the response contains no
     # list yet/at all, or an Array was given but the dataset size was not
-    # supplied).
+    # supplied). If the dataset size is absent, an estimation may be
+    # present; see #estimated_dataset_size.
     #
     attr_reader :dataset_size
 
+    # Read back an estimated dataset size given by a prior call to
+    # #set_resources, or +nil+ if none has been provided (either the
+    # response contains no list yet/at all, or an Array was given but
+    # a dataset size estimation was not supplied).
+    #
+    attr_reader :estimated_dataset_size
+
     # Create a new instance, ready to take on a response. The service
     # middleware is responsible for doing this.
     #
@@ -90,12 +98,13 @@ module Hoodoo; module Services
         raise "Hoodoo::Services::Response.new must be given a valid Interaction ID (got '#{ interaction_id.inspect }')"
       end
 
-      @interaction_id   = interaction_id
-      @errors           = Hoodoo::Errors.new()
-      @headers          = {}
-      @http_status_code = 200
-      @body             = {}
-      @dataset_size     = nil
+      @interaction_id         = interaction_id
+      @errors                 = Hoodoo::Errors.new()
+      @headers                = {}
+      @http_status_code       = 200
+      @body                   = {}
+      @dataset_size           = nil
+      @estimated_dataset_size = nil
 
     end
 
@@ -112,26 +121,67 @@ module Hoodoo; module Services
     # array of items. Although you can just assign an array to either of
     # #body or #set_resource, calling #set_resources is more semantically
     # correct and provides an additional feature; you can specify the total
-    # number of items in the dataset.
+    # number of items in the dataset either precisely, or as an estimation.
     #
     # For example, if you were listing a page of 50 resource instances but
     # the total matching dataset of that list included 344 instances, you
     # would pass 344 in the +dataset_size+ input parameter. This is optional
     # but highly recommended as it is often very useful for calling clients.
     #
+    # If for any reason you aren't able to quickly produce an accurate count
+    # but _can_ produce an estimation, call #set_estimated_resources instead.
+    #
     # +array+::        Array of resource representations (Ruby Array with
     #                  Ruby Hash entries representing rendered resources,
     #                  ideally through the Hoodoo::Presenters framework).
     #
     # +dataset_size+:: Optional _total_ number of items in the entire dataset
     #                  of which +array+ is, most likely, just a subset due to
-    #                  paginated lists via offset and limit parameters.
+    #                  paginated lists via offset and limit parameters. This
+    #                  value was accurate at the instant of counting.
     #
     def set_resources( array, dataset_size = nil )
       self.body = array
       @dataset_size = dataset_size
     end
 
+    # A companion to #set_resources. See the documentation of that method for
+    # background information.
+    #
+    # If the persistence layer in use and data volumes expected for a given
+    # resource make accurate counting too slow to compute, your persistence
+    # layer might support a mechanism for producing an _estimated_ count
+    # quickly instead. For example, PostgreSQL 9's row counting can be slow
+    # due to MVCC but there are PostgreSQL-specific ways of obtaining a row
+    # count estimation quickly. If this applies to you, call here to
+    # correctly specify the estimation in a way that makes it clear to the
+    # calling client that it's not an accurate result.
+    #
+    # Technically you could call *both* this *and* #set_resources to set both
+    # an accurate and an estimated count, though it's hard to imagine a use
+    # case for this outside of testing scenarios; but note that each call
+    # will override any previous setting of the #body property.
+    #
+    # If using the Hoodoo::ActiveRecord extensions for your persistence layer,
+    # then please also see
+    # Hoodoo::ActiveRecord::Finder::ClassMethods::estimated_dataset_size.
+    #
+    # +array+::                  Array of resource representations (Ruby Array
+    #                            with Ruby Hash entries representing rendered
+    #                            resources, ideally through the
+    #                            Hoodoo::Presenters framework).
+    #
+    # +estimated_dataset_size+:: Optional _total_ number of items in the
+    #                            entire dataset of which +array+ is, most
+    #                            likely, just a subset due to paginated lists
+    #                            via offset and limit parameters; this value
+    #                            is an estimation with undefined accuracy.
+    #
+    def set_estimated_resources( array, estimated_dataset_size = nil )
+      self.body = array
+      @estimated_dataset_size = estimated_dataset_size
+    end
+
     # Add an HTTP header to the internal collection that will be used for the
     # response. Trying to set data for the same HTTP header name more than once
     # will result in an exception being raised unless the +overwrite+ parameter
@@ -289,14 +339,19 @@ module Hoodoo; module Services
 
       if body_data.is_a?( ::Array )
         response_hash = { '_data' => body_data }
-        response_hash[ '_dataset_size' ] = @dataset_size unless @dataset_size.nil?
+        response_hash[ '_dataset_size'           ] = @dataset_size           unless @dataset_size.nil?
+        response_hash[ '_estimated_dataset_size' ] = @estimated_dataset_size unless @estimated_dataset_size.nil?
         response_string = ::JSON.generate( response_hash )
+
       elsif body_data.is_a?( ::Hash )
         response_string = ::JSON.generate( body_data )
+
       elsif body_data.is_a?( ::String )
         response_string = body_data
+
       else
         raise "Hoodoo::Services::Response\#for_rack given unrecognised body data class '#{ body_data.class.name }'"
+
       end
 
       rack_response.write( response_string )

data/lib/hoodoo/version.rb

@@ -12,6 +12,6 @@ module Hoodoo
   # The Hoodoo gem version. If this changes, ensure that the date in
   # "hoodoo.gemspec" is correct and run "bundle install" (or "update").
   #
-  VERSION = '1.6.1'
+  VERSION = '1.7.0'
 
 end

data/spec/active/active_record/finder_spec.rb

@@ -435,6 +435,8 @@ describe Hoodoo::ActiveRecord::Finder do
 
   context '#list' do
     it 'lists with pages, offsets and counts' do
+      expect_any_instance_of( RSpecModelFinderTest ).to_not receive( :estimated_dataset_size )
+
       @list_params.offset = 1 # 0 is first record
       @list_params.limit  = 1
 
@@ -453,6 +455,121 @@ describe Hoodoo::ActiveRecord::Finder do
 
   # ==========================================================================
 
+  context 'counting' do
+    it 'lists with a normal count' do
+      finder = RSpecModelFinderTest.list( @list_params )
+
+      expect( finder ).to receive( :dataset_size ).at_least( :once ).and_call_original
+      expect( finder ).to receive( :count        ).at_least( :once ).and_call_original
+
+      expect( finder ).to_not receive( :estimated_dataset_size )
+      expect( finder ).to_not receive( :estimated_count        )
+
+      result = finder.dataset_size()
+
+      expect( result ).to_not be_nil
+    end
+
+    it 'lists with an estimated count' do
+      finder = RSpecModelFinderTest.list( @list_params )
+
+      expect( finder ).to_not receive( :dataset_size )
+
+      expect( finder ).to receive( :estimated_dataset_size ).at_least( :once ).and_call_original
+      expect( finder ).to receive( :estimated_count        ).at_least( :once ).and_call_original
+      expect( finder ).to receive( :count                  ).at_least( :once ).and_call_original
+
+      result = finder.estimated_dataset_size
+
+      expect( result ).to_not be_nil
+    end
+
+    context 'RDoc-recommended PostgreSQL migration example' do
+      before :each do
+        ActiveRecord::Base.connection.execute <<-SQL
+          CREATE FUNCTION estimated_count(query text) RETURNS integer AS
+          $func$
+          DECLARE
+              rec  record;
+              rows integer;
+          BEGIN
+              FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+                  rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+                  EXIT WHEN rows IS NOT NULL;
+              END LOOP;
+
+              RETURN rows;
+          END
+          $func$ LANGUAGE plpgsql;
+        SQL
+
+        counter = Proc.new do | sql |
+          begin
+            ActiveRecord::Base.connection.execute(
+              "SELECT estimated_count('#{ sql}')"
+            ).first[ 'estimated_count' ].to_i
+          rescue
+            nil
+          end
+        end
+
+        RSpecModelFinderTest.estimate_counts_with( counter )
+
+        # Tests start by ensuring the database knows about the current object count.
+        #
+        ActiveRecord::Base.connection.execute( 'ANALYZE' )
+      end
+
+      after :each do
+        ActiveRecord::Base.connection.execute "DROP FUNCTION estimated_count(query text);"
+        RSpecModelFinderTest.estimate_counts_with( nil )
+      end
+
+      context 'estimate' do
+        before :each do
+          @initial_accurate_count = RSpecModelFinderTest.count
+
+          # The outer 'before' code ensures an accurate initial count of 3,
+          # but we're going add in a few more unestimated items.
+          #
+          @uncounted1 = RSpecModelFinderTest.new.save!
+          @uncounted2 = RSpecModelFinderTest.new.save!
+          @uncounted3 = RSpecModelFinderTest.new.save!
+
+          @subsequent_accurate_count = RSpecModelFinderTest.count
+        end
+
+        it 'is initially inaccurate' do
+          finder = RSpecModelFinderTest.list( @list_params )
+          result = finder.estimated_dataset_size
+          expect( result ).to eq( @initial_accurate_count )
+        end
+
+        # The outer 'before' code kind of already tests this anyway since if
+        # the analyze call therein didn't work, prerequisites in the tests
+        # would be wrong and other tests would fail. It's useful to
+        # double-check something this important though.
+        #
+        it 'is accurate after ANALYZE' do
+          ActiveRecord::Base.connection.execute( 'ANALYZE' )
+
+          finder = RSpecModelFinderTest.list( @list_params )
+          result = finder.estimated_dataset_size
+          expect( result ).to eq( @subsequent_accurate_count )
+        end
+
+        it 'is "nil" if the Proc evaluates thus' do
+          RSpecModelFinderTest.estimate_counts_with( Proc.new() { | sql | nil } )
+          finder = RSpecModelFinderTest.list( @list_params )
+          result = finder.estimated_dataset_size
+          expect( result ).to be_nil
+        end
+      end
+    end
+  end
+
+  # ==========================================================================
+
   context 'search' do
     it 'searches without chain' do
       @list_params.search_data = {

data/spec/client/client_spec.rb

@@ -80,10 +80,13 @@ class RSpecClientTestTargetImplementation < Hoodoo::Services::Implementation
         return
       end
 
-      context.response.set_resources(
-        [ mock( context ), mock( context ), mock( context ) ],
-        3
-      )
+      resources = [ mock( context ), mock( context ), mock( context ) ]
+
+      if context.request.embeds.include?( 'estimated_counts_please' )
+        context.response.set_estimated_resources( resources, resources.count )
+      else
+        context.response.set_resources( resources, resources.count )
+      end
     end
 
     def create( context )
@@ -171,7 +174,7 @@ class RSpecClientTestTargetInterface < Hoodoo::Services::Interface
     endpoint :r_spec_client_test_targets, RSpecClientTestTargetImplementation
     public_actions :show
     actions :list, :create, :update, :delete
-    embeds :foo, :bar, :baz
+    embeds :foo, :bar, :baz, :estimated_counts_please
   end
 end
 
@@ -588,6 +591,7 @@ describe Hoodoo::Client do
         result = @endpoint.list( query_hash )
         expect( result.platform_errors.has_errors? ).to eq( false )
         expect( result.dataset_size ).to eq( result.size )
+        expect( result.estimated_dataset_size ).to be_nil
 
         expect( result[ 0 ][ 'embeds'   ] ).to eq( embeds )
         expect( result[ 0 ][ 'language' ] ).to eq( @expected_locale )
@@ -636,6 +640,19 @@ describe Hoodoo::Client do
 
         option_based_expectations( result )
       end
+
+      it "provides estimations" do
+        query_hash = { '_embed' => 'estimated_counts_please' }
+
+        result = @endpoint.list( query_hash )
+        expect( result.platform_errors.has_errors? ).to eq( false )
+        expect( result.dataset_size ).to be_nil
+        expect( result.estimated_dataset_size ).to eq( result.size )
+
+        expect( result[ 0 ][ 'language' ] ).to eq( @expected_locale )
+
+        option_based_expectations( result )
+      end
     end
 
     before :each do
@@ -854,6 +871,7 @@ describe Hoodoo::Client do
         result = @endpoint.list( query_hash )
         expect( result.platform_errors.has_errors? ).to eq( false )
         expect( result.dataset_size ).to eq( result.size )
+        expect( result.estimated_dataset_size ).to be_nil
 
         expect( result[ 0 ][ 'embeds'   ] ).to eq( embeds )
         expect( result[ 0 ][ 'language' ] ).to eq( @expected_locale )
@@ -903,6 +921,19 @@ describe Hoodoo::Client do
         option_based_expectations( result )
       end
 
+      it "provides estimations" do
+        query_hash = { '_embed' => 'estimated_counts_please' }
+
+        result = @endpoint.list( query_hash )
+        expect( result.platform_errors.has_errors? ).to eq( false )
+        expect( result.dataset_size ).to be_nil
+        expect( result.estimated_dataset_size ).to eq( result.size )
+
+        expect( result[ 0 ][ 'language' ] ).to eq( @expected_locale )
+
+        option_based_expectations( result )
+      end
+
       it 'automatically retries' do
         result = @endpoint.list()
         expect( result.platform_errors.has_errors? ).to eq( false )

data/spec/services/middleware/middleware_exotic_communication_spec.rb

@@ -8,6 +8,7 @@ describe Hoodoo::Services::Middleware do
 
   class RSpecTestServiceExoticStubImplementation < Hoodoo::Services::Implementation
     def list( context )
+      context.response.set_estimated_resources( [], 88 )
       context.response.set_resources( [], 99 )
     end
   end
@@ -151,11 +152,13 @@ describe Hoodoo::Services::Middleware do
         endpoint = @mw.inter_resource_endpoint_for( 'Version', 2, @interaction )
 
         # The endpoint should've been called locally; the implementation at
-        # the top of this file sets an empty array with dataset size 99.
+        # the top of this file sets an empty array with dataset size 99 *and*
+        # an estimated dataset size of 88.
 
         mock_result = endpoint.list()
         expect( mock_result ).to be_empty
         expect( mock_result.dataset_size ).to eq( 99 )
+        expect( mock_result.estimated_dataset_size ).to eq( 88 )
       end
 
       it 'complains about a missing Alchemy instance' do
@@ -558,6 +561,7 @@ describe Hoodoo::Services::Middleware do
 
       expect( mock_result ).to eq( Hoodoo::Client::AugmentedArray.new )
       expect( mock_result.dataset_size ).to eq(99)
+      expect( mock_result.estimated_dataset_size ).to eq( 88 )
       expect( mock_result.platform_errors.has_errors? ).to eq( false )
     end
   end

data/spec/services/middleware/middleware_multi_local_spec.rb

@@ -23,7 +23,9 @@ class RSpecTestInterResourceCallsAImplementation < Hoodoo::Services::Implementat
     if search_offset > 0
       context.response.add_error( 'service_calls_a.triggered', 'reference' => { :offset => search_offset } )
     else
-      context.response.set_resources( [1,2,3,4], 4321 )
+      array = [ 1, 2, 3, 4 ]
+      context.response.set_estimated_resources( array, 1234 )
+      context.response.set_resources( array, 4321 )
       expectable_hook( context )
     end
   end
@@ -415,6 +417,7 @@ describe Hoodoo::Services::Middleware::InterResourceLocal do
     expect_any_instance_of(RSpecTestInterResourceCallsBImplementation).to receive(:expectable_result_hook).once do | ignored_rspec_mock_instance, result |
       expect(result).to eq([1,2,3,4])
       expect(result.dataset_size).to eq(4321)
+      expect(result.estimated_dataset_size).to eq(1234)
     end
 
     get '/v1/rspec_test_inter_resource_calls_b',

data/spec/services/middleware/middleware_multi_remote_spec.rb

@@ -28,14 +28,15 @@ class TestEchoImplementation < Hoodoo::Services::Implementation
         return
       end
 
-      context.response.set_resources(
-        [
-          { 'list0' => TestEchoImplementation.to_h( context ) },
-          { 'list1' => TestEchoImplementation.to_h( context ) },
-          { 'list2' => TestEchoImplementation.to_h( context ) }
-        ],
-        49
-      )
+      array =
+      [
+        { 'list0' => TestEchoImplementation.to_h( context ) },
+        { 'list1' => TestEchoImplementation.to_h( context ) },
+        { 'list2' => TestEchoImplementation.to_h( context ) }
+      ]
+
+      context.response.set_resources( array, 49 )
+      context.response.set_estimated_resources( array, 50 )
     end
 
     def show( context )
@@ -189,15 +190,16 @@ class TestCallImplementation < Hoodoo::Services::Implementation
 
     return if result.adds_errors_to?( context.response.errors )
 
-    context.response.set_resources(
-      [
-        { 'listA'   => result                  },
-        { 'listB'   => result                  },
-        { 'listC'   => result                  },
-        { 'options' => result.response_options }
-      ],
-      ( result.dataset_size || 0 ) + 2
-    )
+    array =
+    [
+      { 'listA'   => result                  },
+      { 'listB'   => result                  },
+      { 'listC'   => result                  },
+      { 'options' => result.response_options }
+    ]
+
+    context.response.set_resources( array, ( result.dataset_size || 0 ) + 2 )
+    context.response.set_estimated_resources( array, ( result.estimated_dataset_size || 0 ) + 2 )
   end
 
   def show( context )
@@ -434,6 +436,7 @@ describe Hoodoo::Services::Middleware do
         }
       )
       expect( parsed[ '_dataset_size' ] ).to eq( 49 )
+      expect( parsed[ '_estimated_dataset_size' ] ).to eq( 50 )
     end
 
     it 'lists things with callbacks', :check_callbacks => true do
@@ -957,6 +960,7 @@ describe Hoodoo::Services::Middleware do
       )
 
       expect( parsed[ '_dataset_size' ]).to eq( 51 )
+      expect( parsed[ '_estimated_dataset_size' ] ).to eq( 52 )
 
       expect( parsed[ '_data' ][ 3 ] ).to_not be_nil
       expect_response_options_for( parsed[ '_data' ][ 3 ][ 'options' ] )

data/spec/services/middleware/middleware_spec.rb

@@ -225,7 +225,7 @@ describe Hoodoo::Services::Middleware do
       expect(result['errors'][0]['message']).to eq("Content-Type 'application/json' does not match supported types '[\"application/json\"]' and/or encodings '[\"utf-8\"]'")
     end
 
-    it 'should complain about incorrect content type' do
+    it 'complains about incorrect content types' do
       get '/v2/rspec_test_service_stub', nil, { 'CONTENT_TYPE' => 'some/thing; charset=utf-8' }
 
       expect(last_response.status).to eq(422)
@@ -235,7 +235,7 @@ describe Hoodoo::Services::Middleware do
       expect(result['errors'][0]['message']).to eq("Content-Type 'some/thing; charset=utf-8' does not match supported types '[\"application/json\"]' and/or encodings '[\"utf-8\"]'")
     end
 
-    it 'should complain about incorrect content type' do
+    it 'complains about incorrect content type charsets' do
       get '/v2/rspec_test_service_stub', nil, { 'CONTENT_TYPE' => 'application/json; charset=madeup' }
 
       expect(last_response.status).to eq(422)
@@ -245,6 +245,25 @@ describe Hoodoo::Services::Middleware do
       expect(result['errors'][0]['message']).to eq("Content-Type 'application/json; charset=madeup' does not match supported types '[\"application/json\"]' and/or encodings '[\"utf-8\"]'")
     end
 
+    it 'rejects malformed attempts to specify a list of options' do
+      types =
+      [
+        'application/json; charset=utf-8, application/x-www-form-urlencoded',
+        'application/x-www-form-urlencoded, application/json; charset=utf-8',
+        'application/x-www-form-urlencoded, application/json; charset=utf-8, application/json; charset=madeup'
+      ]
+
+      types.each do | type |
+        get '/v2/rspec_test_service_stub', nil, { 'CONTENT_TYPE' => type }
+
+        expect(last_response.status).to eq(422)
+
+        result = JSON.parse(last_response.body)
+        expect(result['errors'][0]['code']).to eq('platform.malformed')
+        expect(result['errors'][0]['message']).to eq("Content-Type '#{ type }' is malformed")
+      end
+    end
+
     it 'should generate interaction IDs and other standard headers even for error states' do
       get '/v2/rspec_test_service_stub'
 

data/spec/services/services/response_spec.rb

@@ -202,7 +202,7 @@ describe Hoodoo::Services::Response do
       expect(body.body).to eq([expected])
     end
 
-    it 'should return non-error condition Rack data correctly with an Array body' do
+    it 'returns non-error condition Rack data correctly with an Array body' do
       response_array = [ { this: 'should not be ignored' }, { neither: 'should this' } ]
       @r.body = response_array
 
@@ -214,6 +214,47 @@ describe Hoodoo::Services::Response do
       expect(body.body).to eq([expected])
     end
 
+    it 'returns non-error condition Rack data correctly with a dataset size' do
+      response_array = [ { this: 'should not be ignored' }, { neither: 'should this' } ]
+      @r.set_resources( response_array, response_array.count )
+
+      status, headers, body = @r.for_rack
+
+      expected = JSON.generate( { '_data' => response_array, '_dataset_size' => response_array.count } )
+      expect( status    ).to eq( 200 )
+      expect( headers   ).to eq( { 'Content-Length' => expected.length.to_s } )
+      expect( body.body ).to eq( [ expected ] )
+    end
+
+    it 'returns non-error condition Rack data correctly with an estimated dataset size' do
+      response_array = [ { this: 'should not be ignored' }, { neither: 'should this' } ]
+      @r.set_estimated_resources( response_array, response_array.count )
+
+      status, headers, body = @r.for_rack
+
+      expected = JSON.generate( { '_data' => response_array, '_estimated_dataset_size' => response_array.count } )
+      expect( status    ).to eq( 200 )
+      expect( headers   ).to eq( { 'Content-Length' => expected.length.to_s } )
+      expect( body.body ).to eq( [ expected ] )
+    end
+
+    it 'returns non-error condition Rack data correctly with both an accurate and an estimated dataset size' do
+      response_array = [ { this: 'should not be ignored' }, { neither: 'should this' } ]
+
+                @r.set_resources( response_array, response_array.count )
+      @r.set_estimated_resources( response_array, response_array.count )
+
+      status, headers, body = @r.for_rack
+
+      expected = JSON.generate( { '_data'                   => response_array,
+                                  '_dataset_size'           => response_array.count,
+                                  '_estimated_dataset_size' => response_array.count } )
+
+      expect( status    ).to eq( 200 )
+      expect( headers   ).to eq( { 'Content-Length' => expected.length.to_s } )
+      expect( body.body ).to eq( [ expected ] )
+    end
+
     it 'should allow pre-encoded strings in the body' do
       @r.body = 'Hello World!'
 
@@ -232,7 +273,7 @@ describe Hoodoo::Services::Response do
     end
   end
 
-  context "#not_found" do
+  context '#not_found' do
 
     let(:ident) { 'an_ident' }
     before { @r.not_found(ident) }
@@ -247,4 +288,34 @@ describe Hoodoo::Services::Response do
       expect(@r.halt_processing?).to eq(true)
     end
   end
+
+  context '#set_resources and #set_estimated_resources' do
+    it '#set_resources sets #body and #dataset_size' do
+      array = [ 1, 2, 3, 4 ]
+      @r.set_resources( array, 4321 )
+      expect( @r.body ).to match_array( array )
+      expect( @r.dataset_size ).to eq( 4321 )
+      expect( @r.estimated_dataset_size ).to be_nil
+    end
+
+    it '#set_estimated_resources sets #body and #estimated_dataset_size' do
+      array = [ 4, 3, 2, 1 ]
+      @r.set_estimated_resources( array, 1234 )
+      expect( @r.body ).to match_array( array )
+      expect( @r.dataset_size ).to be_nil
+      expect( @r.estimated_dataset_size ).to eq( 1234 )
+    end
+
+    it 'both together set all properties with the most recent call setting #body' do
+      array_1 = [ 1, 2, 3, 4 ]
+      @r.set_resources( array_1, 4321 )
+
+      array_2 = [ 4, 3, 2, 1 ]
+      @r.set_estimated_resources( array_2, 1234 )
+
+      expect( @r.body ).to match_array( array_2 )
+      expect( @r.dataset_size ).to eq( 4321 )
+      expect( @r.estimated_dataset_size ).to eq( 1234 )
+    end
+  end
 end

metadata

@@ -1,265 +1,265 @@
 --- !ruby/object:Gem::Specification
 name: hoodoo
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.7.0
 platform: ruby
 authors:
 - Loyalty New Zealand
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-29 00:00:00.000000000 Z
+date: 2016-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kgio
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.9'
 - !ruby/object:Gem::Dependency
   name: dalli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.7'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.4'
 - !ruby/object:Gem::Dependency
   name: simplecov-rcov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
 - !ruby/object:Gem::Dependency
   name: alchemy-flux
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.3'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.2'
 - !ruby/object:Gem::Dependency
   name: database_cleaner
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.18'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.5'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.8'
 - !ruby/object:Gem::Dependency
   name: raygun4ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.1'
 - !ruby/object:Gem::Dependency
   name: airbrake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.3'
 - !ruby/object:Gem::Dependency
   name: le
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.6'
 description: Simplify the implementation of consistent services within an API-based
@@ -495,17 +495,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: Opinionated APIs