simple-sql 0.5.36 → 0.5.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5185777df8efad3d0d17334a70d30b35551f4abd7adf0fded3549dd2d1b1325
-  data.tar.gz: e34ee800440259bfbe8ef88f9f70fb1675dee4a91e441c162a8bdc6313e5d88f
+  metadata.gz: eb13412c57185b5bd356c4dc94f52232b22bdef58e032d67ceb79127811ec639
+  data.tar.gz: adc67bad73619d640834d8b821676bfbc5530ca003adfcabe3012535350cea39
 SHA512:
-  metadata.gz: 9b03b9f204d9bd0e87e0c349f1f2ab962bc94101f14ab33cb2522c0287861e801ddca3cd98e97f4317716df7a07a054f6acf26837309f18bc14b8a50f95eac4a
-  data.tar.gz: 5eb4fe4c83ca847aef2263251b56f95b3ea450c15abc1e8289ed92bbcaaaa5d5d8a3683814c009c2dd4928d679a4072d10717a616e3cbac2978d59f2d2f453e8
+  metadata.gz: 1c5597472257a4b487c2b9b1a87545cb37d70e1e625cdd31ced80d3aaf650f547ccd6931cac3e9e81e97df25caaa6fe3be519ae905ea206b5a573717fcbc3a18
+  data.tar.gz: a69336914733676aa4465489b71576b1274284337d9295cc786d5e050a72942fe8b54f90cb6eccad7165e8f24ad635e6376342e49dd63bf4d2cbe174fc173620

data/VERSION

@@ -1 +1 @@
-0.5.36
+0.5.37

data/lib/simple/sql/connection/scope/count_by_groups.rb

@@ -19,55 +19,113 @@ class Simple::SQL::Connection::Scope
   #
   def enumerate_groups(sql_fragment)
     sql = order_by(nil).to_sql(pagination: false)
+    @connection.all "SELECT DISTINCT #{sql_fragment} FROM (#{sql}) sq", *args
+  end
 
-    cost = @connection.estimate_cost "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
-
-    # cost estimates are good, but are hard to check against a hard coded value.
-    # see https://issues.mediafellows.com/issues/75232
-    #
-    # if cost > 10_000
-    #   raise "enumerate_groups(#{sql_fragment.inspect}) takes too much time. Make sure to create a suitable index"
-    # end
+  def count_by(sql_fragment)
+    expect! sql_fragment => String
 
-    groups = []
-    var_name = "$#{@args.count + 1}"
-    cur = @connection.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+    sql = order_by(nil).to_sql(pagination: false)
 
-    while cur
-      groups << cur
-      cur = @connection.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq"" WHERE #{sql_fragment} > #{var_name}", *args, cur
+    recs = @connection.all "SELECT COUNT(*) AS count, #{sql_fragment} AS group FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+
+    # if we count by a single value (e.g. `count_by("role_id")`) each entry in recs consists of an array [group_value, count].
+    # The resulting Hash will have entries of group_value => count.
+    if recs.first&.length == 2
+      recs.each_with_object({}) do |count_and_group, hsh|
+        count, group = *count_and_group
+        hsh[group] = count
+      end
+    else
+      recs.each_with_object({}) do |count_and_group, hsh|
+        count, *group = *count_and_group
+        hsh[group] = count
+      end
     end
-
-    groups
   end
 
-  def count_by(sql_fragment)
-    sql = order_by(nil).to_sql(pagination: false)
+  private
 
-    recs = @connection.all "SELECT #{sql_fragment} AS group, COUNT(*) AS count FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
-    Hash[recs]
+  # cost estimate threshold for count_by method. Can be set to false, true, or
+  # a number.
+  #
+  # Note that cost estimates are problematic, since they are not reported in
+  # any "real" unit, meaning any comparison really is a bit pointless.
+  COUNT_BY_ESTIMATE_COST_THRESHOLD = 10_000
+
+  # estimates the cost to run a sql query. If COUNT_BY_ESTIMATE_COST_THRESHOLD
+  # is set and the cost estimate is less than COUNT_BY_ESTIMATE_COST_THRESHOLD
+  # \a count_by_estimate is using the estimating code path.
+  def use_count_by_estimate?(sql_group_by_fragment)
+    case COUNT_BY_ESTIMATE_COST_THRESHOLD
+    when true then true
+    when false then false
+    else
+      # estimate the effort to exact counting over all groups.
+      base_sql  = order_by(nil).to_sql(pagination: false)
+      count_sql = "SELECT COUNT(*) FROM (#{base_sql}) sq GROUP BY #{sql_group_by_fragment}"
+      cost      = @connection.estimate_cost count_sql, *args
+
+      cost >= COUNT_BY_ESTIMATE_COST_THRESHOLD
+    end
   end
 
+  public
+
   def count_by_estimate(sql_fragment)
-    return count_by(sql_fragment)
+    expect! sql_fragment => String
 
-    # The code below runs an estimate on the effort to count by a group. This is currently
-    # disabled (see https://issues.mediafellows.com/issues/75237).
+    return count_by(sql_fragment) unless use_count_by_estimate?(sql_fragment)
 
-    sql = order_by(nil).to_sql(pagination: false)
-    cost = @connection.estimate_cost "SELECT COUNT(*) FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+    # iterate over all groups, estimating the count for each.
+    #
+    # For larger groups we'll use that estimate - preventing a full table scan.
+    # Groups smaller than EXACT_COUNT_THRESHOLD are counted exactly - in the
+    # hope that this query can be answered from an index.
+
+    #
+    # Usually Simple::SQL.all normalizes each result row into its first value,
+    # if the row only consists of a single value. Here, however, we don't
+    # know the width of a group; so to understand this we just add a dummy
+    # value to the sql_fragment and then remove it again.
+    #
+    groups = enumerate_groups("1 AS __dummy__, #{sql_fragment}")
+    groups = groups.each(&:shift)
 
-    return count_by(sql_fragment) if cost < 10_000
+    # no groups? well, then...
+    return {} if groups.empty?
 
-    # iterate over all groups, estimating the count for each. If the count is
-    # less than EXACT_COUNT_THRESHOLD we ask for the exact count in that and
-    # similarily sparse groups.
-    var_name = "$#{@args.count + 1}"
+    #
+    # The estimating code only works for groups of size 1. This is a limitation
+    # of simple-sql - for larger groups we would have to be able to encode arrays
+    # of arrays on their way to the postgres server. We are not able to do that
+    # currently.
+    #
+    group_size = groups.first&.length
+    if group_size > 1
+      return count_by(sql_fragment)
+    end
+
+    # The code below only works for groups of size 1
+    groups = groups.map(&:first)
+
+    #
+    # Now we estimate the count of entries in each group. For large groups we
+    # just use the estimate - because it is usually pretty close to being correct.
+    # Small groups are collected in the `sparse_groups` array, to be counted
+    # exactly later on.
+    #
 
     counts = {}
+
     sparse_groups = []
-    enumerate_groups(sql_fragment).each do |group|
-      scope = @connection.scope("SELECT * FROM (#{sql}) sq WHERE #{sql_fragment}=#{var_name}", *args, group)
+    base_sql = order_by(nil).to_sql(pagination: false)
+
+    var_name = "$#{@args.count + 1}"
+
+    groups.each do |group|
+      scope = @connection.scope("SELECT * FROM (#{base_sql}) sq WHERE #{sql_fragment}=#{var_name}", args + [group])
+
       estimated_count = scope.send(:estimated_count)
       counts[group] = estimated_count
       sparse_groups << group if estimated_count < EXACT_COUNT_THRESHOLD
@@ -77,7 +135,7 @@ class Simple::SQL::Connection::Scope
     unless sparse_groups.empty?
       sparse_counts = @connection.all <<~SQL, *args, sparse_groups
         SELECT #{sql_fragment} AS group, COUNT(*) AS count
-        FROM (#{sql}) sq
+        FROM (#{base_sql}) sq
         WHERE #{sql_fragment} = ANY(#{var_name})
         GROUP BY #{sql_fragment}
       SQL

data/spec/simple/sql/count_by_groups_spec.rb

@@ -1,44 +1,63 @@
 require "spec_helper"
 
 describe "Simple::SQL::Connection::Scope#count_by" do
-  let!(:users)        { 1.upto(10).map { |i| create(:user, role_id: i) } }
-  let(:all_role_ids)  { SQL.all("SELECT DISTINCT role_id FROM users") }
-  let(:scope)         { SQL.scope("SELECT * FROM users") }
+  let!(:users)                  { 1.upto(10).map { |i| create(:user, role_id: i) } }
+  let(:scope)                   { SQL.scope("SELECT * FROM users") }
+
+  let(:all_role_ids)            { 1.upto(10).to_a }
+  let(:all_role_ids_w_squares)  { all_role_ids.map { |role_id| [role_id, role_id*role_id] } } 
+
+  before do
+    # initially we have 10 users, one per role_id in the range 1 .. 10
+    # This adds another 3 users with role_id of 1.
+    create(:user, role_id: 1)
+    create(:user, role_id: 1)
+    create(:user, role_id: 1)
+  end
 
   describe "enumerate_groups" do
-    it "returns all groups" do
+    it "returns all groups by a single column" do
       expect(scope.enumerate_groups("role_id")).to contain_exactly(*all_role_ids)
-      expect(scope.where("role_id < 4").enumerate_groups("role_id")).to contain_exactly(*(1.upto(3).to_a))
+    end
+
+    it "obeys where conditions" do
+      expect(scope.where("role_id < $1", 4).enumerate_groups("role_id")).to contain_exactly(1,2,3)
+    end
+
+    it "counts all groups by multiple columns" do
+      expect(scope.where("role_id < $1", 4).enumerate_groups("role_id, role_id * role_id")).to contain_exactly([1, 1], [2, 4], [3, 9])
     end
   end
 
   describe "count_by" do
-    it "counts all groups" do
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-
+    it "counts all groups by a single column" do
       expect(scope.count_by("role_id")).to include(1 => 4)
       expect(scope.count_by("role_id")).to include(2 => 1)
       expect(scope.count_by("role_id").keys).to contain_exactly(*all_role_ids)
     end
+
+    it "counts all groups by multiple columns" do
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id")).to include([1,1] => 4)
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id")).to include([2, 4] => 1)
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id").keys).to contain_exactly([1, 1], [2, 4], [3, 9])
+    end
   end
 
   describe "count_by_estimate" do
     before do
-      # 10_000 is chosen "magically". It is large enough to switch to the fast algorithm,
-      # but 
-      allow(::Simple::SQL).to receive(:costs).and_return([0, 10_000])
+      expect_any_instance_of(Simple::SQL::Connection).to receive(:estimate_cost).at_least(:once).and_return(10_000)
     end
-    
-    it "counts all groups" do
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
 
+    it "counts all groups by a single column" do
       expect(scope.count_by_estimate("role_id")).to include(1 => 4)
       expect(scope.count_by_estimate("role_id")).to include(2 => 1)
       expect(scope.count_by_estimate("role_id").keys).to contain_exactly(*all_role_ids)
     end
+
+    it "counts all groups by multiple columns and conditions" do
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id")).to include([1,1] => 4)
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id")).to include([2, 4] => 1)
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id").keys).to contain_exactly([1, 1], [2, 4], [3, 9])
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple-sql
 version: !ruby/object:Gem::Version
-  version: 0.5.36
+  version: 0.5.37
 platform: ruby
 authors:
 - radiospiel
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-06 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg_array_parser