simple-sql 0.5.32 → 0.5.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb4af8eb0802b84e83370417d134ff18f82c812dbd9a053853739d9570f92a95
-  data.tar.gz: 0b023420320187d12f32cb6e1a4a093775a5bd2849015443768b0177392729fb
+  metadata.gz: eb13412c57185b5bd356c4dc94f52232b22bdef58e032d67ceb79127811ec639
+  data.tar.gz: adc67bad73619d640834d8b821676bfbc5530ca003adfcabe3012535350cea39
 SHA512:
-  metadata.gz: 996461571d090b5cf7589dfc6f1d12925b11b54d047fc2ec2395bbb6a2d75e583a7f82d0cc92985982891d3254e3e26a83260f6a38be5bee0eac5872effb4609
-  data.tar.gz: 57decbf1a834422bdfc2987056c3fa37f04988de7059ed8c6eb8aa64ffc16c445a19f1dc9e91f880eac8a2ea7e8772dcfb8b1aeaf43b57ca7f13d4662ceaf177
+  metadata.gz: 1c5597472257a4b487c2b9b1a87545cb37d70e1e625cdd31ced80d3aaf650f547ccd6931cac3e9e81e97df25caaa6fe3be519ae905ea206b5a573717fcbc3a18
+  data.tar.gz: a69336914733676aa4465489b71576b1274284337d9295cc786d5e050a72942fe8b54f90cb6eccad7165e8f24ad635e6376342e49dd63bf4d2cbe174fc173620

data/VERSION

@@ -1 +1 @@
-0.5.32
+0.5.37

data/lib/simple/sql/connection/scope/count_by_groups.rb

@@ -19,45 +19,113 @@ class Simple::SQL::Connection::Scope
   #
   def enumerate_groups(sql_fragment)
     sql = order_by(nil).to_sql(pagination: false)
+    @connection.all "SELECT DISTINCT #{sql_fragment} FROM (#{sql}) sq", *args
+  end
 
-    cost = @connection.estimate_cost "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
-    raise "enumerate_groups(#{sql_fragment.inspect}) takes too much time. Make sure to create a suitable index" if cost > 10_000
+  def count_by(sql_fragment)
+    expect! sql_fragment => String
 
-    groups = []
-    var_name = "$#{@args.count + 1}"
-    cur = @connection.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+    sql = order_by(nil).to_sql(pagination: false)
 
-    while cur
-      groups << cur
-      cur = @connection.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq"" WHERE #{sql_fragment} > #{var_name}", *args, cur
-    end
+    recs = @connection.all "SELECT COUNT(*) AS count, #{sql_fragment} AS group FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
 
-    groups
+    # if we count by a single value (e.g. `count_by("role_id")`) each entry in recs consists of an array [group_value, count].
+    # The resulting Hash will have entries of group_value => count.
+    if recs.first&.length == 2
+      recs.each_with_object({}) do |count_and_group, hsh|
+        count, group = *count_and_group
+        hsh[group] = count
+      end
+    else
+      recs.each_with_object({}) do |count_and_group, hsh|
+        count, *group = *count_and_group
+        hsh[group] = count
+      end
+    end
   end
 
-  def count_by(sql_fragment)
-    sql = order_by(nil).to_sql(pagination: false)
+  private
+
+  # cost estimate threshold for count_by method. Can be set to false, true, or
+  # a number.
+  #
+  # Note that cost estimates are problematic, since they are not reported in
+  # any "real" unit, meaning any comparison really is a bit pointless.
+  COUNT_BY_ESTIMATE_COST_THRESHOLD = 10_000
 
-    recs = @connection.all "SELECT #{sql_fragment} AS group, COUNT(*) AS count FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
-    Hash[recs]
+  # estimates the cost to run a sql query. If COUNT_BY_ESTIMATE_COST_THRESHOLD
+  # is set and the cost estimate is less than COUNT_BY_ESTIMATE_COST_THRESHOLD
+  # \a count_by_estimate is using the estimating code path.
+  def use_count_by_estimate?(sql_group_by_fragment)
+    case COUNT_BY_ESTIMATE_COST_THRESHOLD
+    when true then true
+    when false then false
+    else
+      # estimate the effort to exact counting over all groups.
+      base_sql  = order_by(nil).to_sql(pagination: false)
+      count_sql = "SELECT COUNT(*) FROM (#{base_sql}) sq GROUP BY #{sql_group_by_fragment}"
+      cost      = @connection.estimate_cost count_sql, *args
+
+      cost >= COUNT_BY_ESTIMATE_COST_THRESHOLD
+    end
   end
 
+  public
+
   def count_by_estimate(sql_fragment)
-    sql = order_by(nil).to_sql(pagination: false)
+    expect! sql_fragment => String
 
-    cost = @connection.estimate_cost "SELECT COUNT(*) FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+    return count_by(sql_fragment) unless use_count_by_estimate?(sql_fragment)
 
-    return count_by(sql_fragment) if cost < 10_000
+    # iterate over all groups, estimating the count for each.
+    #
+    # For larger groups we'll use that estimate - preventing a full table scan.
+    # Groups smaller than EXACT_COUNT_THRESHOLD are counted exactly - in the
+    # hope that this query can be answered from an index.
 
-    # iterate over all groups, estimating the count for each. If the count is
-    # less than EXACT_COUNT_THRESHOLD we ask for the exact count in that and
-    # similarily sparse groups.
-    var_name = "$#{@args.count + 1}"
+    #
+    # Usually Simple::SQL.all normalizes each result row into its first value,
+    # if the row only consists of a single value. Here, however, we don't
+    # know the width of a group; so to understand this we just add a dummy
+    # value to the sql_fragment and then remove it again.
+    #
+    groups = enumerate_groups("1 AS __dummy__, #{sql_fragment}")
+    groups = groups.each(&:shift)
+
+    # no groups? well, then...
+    return {} if groups.empty?
+
+    #
+    # The estimating code only works for groups of size 1. This is a limitation
+    # of simple-sql - for larger groups we would have to be able to encode arrays
+    # of arrays on their way to the postgres server. We are not able to do that
+    # currently.
+    #
+    group_size = groups.first&.length
+    if group_size > 1
+      return count_by(sql_fragment)
+    end
+
+    # The code below only works for groups of size 1
+    groups = groups.map(&:first)
+
+    #
+    # Now we estimate the count of entries in each group. For large groups we
+    # just use the estimate - because it is usually pretty close to being correct.
+    # Small groups are collected in the `sparse_groups` array, to be counted
+    # exactly later on.
+    #
 
     counts = {}
+
     sparse_groups = []
-    enumerate_groups(sql_fragment).each do |group|
-      scope = @connection.scope("SELECT * FROM (#{sql}) sq WHERE #{sql_fragment}=#{var_name}", *args, group)
+    base_sql = order_by(nil).to_sql(pagination: false)
+
+    var_name = "$#{@args.count + 1}"
+
+    groups.each do |group|
+      scope = @connection.scope("SELECT * FROM (#{base_sql}) sq WHERE #{sql_fragment}=#{var_name}", args + [group])
+
       estimated_count = scope.send(:estimated_count)
       counts[group] = estimated_count
       sparse_groups << group if estimated_count < EXACT_COUNT_THRESHOLD
@@ -67,7 +135,7 @@ class Simple::SQL::Connection::Scope
     unless sparse_groups.empty?
       sparse_counts = @connection.all <<~SQL, *args, sparse_groups
         SELECT #{sql_fragment} AS group, COUNT(*) AS count
-        FROM (#{sql}) sq
+        FROM (#{base_sql}) sq
         WHERE #{sql_fragment} = ANY(#{var_name})
         GROUP BY #{sql_fragment}
       SQL

data/lib/simple/sql/monkey_patches.rb

@@ -2,12 +2,14 @@
 
 module Simple::SQL::MonkeyPatches
   def self.warn(msg)
+    return if ENV["SIMPLE_SQL_SILENCE"] == "1"
+
     @@warned ||= {}
     return if @@warned[msg]
 
     @@warned[msg] = true
 
-    STDERR.puts "== monkeypatch warning: #{msg}"
+    STDERR.puts "== monkeypatch warning: #{msg} (set SIMPLE_SQL_SILENCE=1 to disable)"
   end
 end
 

data/scripts/benchmark1.rb

@@ -0,0 +1,29 @@
+require "benchmark"
+require "simple-sql"
+
+ENV["DATABASE_URL"] = "postgres://admin:admin@localhost/um_development"
+
+Simple::SQL.connect
+# require 'stackprof'# added
+
+Benchmark.bmbm do |x|
+  x.report("1000x Simple::SQL HStore performance for 100 users") do
+    1000.times { Simple::SQL.all("SELECT id, meta_data FROM users limit 100") }
+  end
+
+  x.report("1000x Simple::SQL HStore::varchar performance for 100 users") do
+    1000.times { Simple::SQL.all("SELECT id, meta_data::varchar FROM users limit 100") }
+  end
+
+  x.report("1000x Simple::SQL HStore as jsonb performance for 100 users") do
+    1000.times { Simple::SQL.all("SELECT id, to_jsonb(meta_data) FROM users limit 100") }
+  end
+
+  x.report("1000x Simple::SQL timestamp performance for 100 users") do
+    1000.times { Simple::SQL.all("SELECT id, created_at FROM users limit 100") }
+  end
+
+  x.report("1000x Simple::SQL timestamp::varchar performance for 100 users") do
+    1000.times { Simple::SQL.all("SELECT id, created_at::varchar FROM users limit 100") }
+  end
+end

data/simple-sql.gemspec

@@ -33,16 +33,16 @@ Gem::Specification.new do |gem|
   # during tests we check the SIMPLE_SQL_ACTIVERECORD_SPECS environment setting.
   # Run make tests to run all tests
   if ENV["SIMPLE_SQL_ACTIVERECORD_SPECS"]
-    gem.add_dependency 'activerecord', '> 4.2', *(ENV["SIMPLE_SQL_ACTIVERECORD_SPECS"].split(","))
+    gem.add_dependency 'activerecord', '>= 5.2.4.5', *(ENV["SIMPLE_SQL_ACTIVERECORD_SPECS"].split(","))
   else
-    gem.add_dependency 'activerecord', '> 4.2', '< 7'
+    gem.add_dependency 'activerecord', '>= 5.2.4.5', '< 6.1'
   end
 
   # optional gems (required by some of the parts)
 
   # development gems
   gem.add_development_dependency 'pg', '0.20'
-  gem.add_development_dependency 'rake', '~> 11'
+  gem.add_development_dependency 'rake', '>= 12.3.3'
   gem.add_development_dependency 'rspec', '~> 3.7'
   gem.add_development_dependency 'rubocop', '~> 0.61.1'
   gem.add_development_dependency 'simplecov', '~> 0'

data/spec/simple/sql/count_by_groups_spec.rb

@@ -1,44 +1,63 @@
 require "spec_helper"
 
 describe "Simple::SQL::Connection::Scope#count_by" do
-  let!(:users)        { 1.upto(10).map { |i| create(:user, role_id: i) } }
-  let(:all_role_ids)  { SQL.all("SELECT DISTINCT role_id FROM users") }
-  let(:scope)         { SQL.scope("SELECT * FROM users") }
+  let!(:users)                  { 1.upto(10).map { |i| create(:user, role_id: i) } }
+  let(:scope)                   { SQL.scope("SELECT * FROM users") }
+
+  let(:all_role_ids)            { 1.upto(10).to_a }
+  let(:all_role_ids_w_squares)  { all_role_ids.map { |role_id| [role_id, role_id*role_id] } } 
+
+  before do
+    # initially we have 10 users, one per role_id in the range 1 .. 10
+    # This adds another 3 users with role_id of 1.
+    create(:user, role_id: 1)
+    create(:user, role_id: 1)
+    create(:user, role_id: 1)
+  end
 
   describe "enumerate_groups" do
-    it "returns all groups" do
+    it "returns all groups by a single column" do
       expect(scope.enumerate_groups("role_id")).to contain_exactly(*all_role_ids)
-      expect(scope.where("role_id < 4").enumerate_groups("role_id")).to contain_exactly(*(1.upto(3).to_a))
+    end
+
+    it "obeys where conditions" do
+      expect(scope.where("role_id < $1", 4).enumerate_groups("role_id")).to contain_exactly(1,2,3)
+    end
+
+    it "counts all groups by multiple columns" do
+      expect(scope.where("role_id < $1", 4).enumerate_groups("role_id, role_id * role_id")).to contain_exactly([1, 1], [2, 4], [3, 9])
     end
   end
 
   describe "count_by" do
-    it "counts all groups" do
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-
+    it "counts all groups by a single column" do
       expect(scope.count_by("role_id")).to include(1 => 4)
       expect(scope.count_by("role_id")).to include(2 => 1)
       expect(scope.count_by("role_id").keys).to contain_exactly(*all_role_ids)
     end
+
+    it "counts all groups by multiple columns" do
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id")).to include([1,1] => 4)
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id")).to include([2, 4] => 1)
+      expect(scope.where("role_id < $1", 4).count_by("role_id, role_id * role_id").keys).to contain_exactly([1, 1], [2, 4], [3, 9])
+    end
   end
 
   describe "count_by_estimate" do
     before do
-      # 10_000 is chosen "magically". It is large enough to switch to the fast algorithm,
-      # but 
-      allow(::Simple::SQL).to receive(:costs).and_return([0, 10_000])
+      expect_any_instance_of(Simple::SQL::Connection).to receive(:estimate_cost).at_least(:once).and_return(10_000)
     end
-    
-    it "counts all groups" do
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
-      create(:user, role_id: 1)
 
+    it "counts all groups by a single column" do
       expect(scope.count_by_estimate("role_id")).to include(1 => 4)
       expect(scope.count_by_estimate("role_id")).to include(2 => 1)
       expect(scope.count_by_estimate("role_id").keys).to contain_exactly(*all_role_ids)
     end
+
+    it "counts all groups by multiple columns and conditions" do
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id")).to include([1,1] => 4)
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id")).to include([2, 4] => 1)
+      expect(scope.where("role_id < $1", 4).count_by_estimate("role_id, role_id * role_id").keys).to contain_exactly([1, 1], [2, 4], [3, 9])
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple-sql
 version: !ruby/object:Gem::Version
-  version: 0.5.32
+  version: 0.5.37
 platform: ruby
 authors:
 - radiospiel
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-02 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg_array_parser
@@ -91,22 +91,22 @@ dependencies:
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 5.2.4.5
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '6.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: 5.2.4.5
     - - "<"
       - !ruby/object:Gem::Version
-        version: '7'
+        version: '6.1'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -125,16 +125,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '11'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -235,6 +235,7 @@ files:
 - lib/simple/sql/table_print.rb
 - lib/simple/sql/version.rb
 - log/.gitkeep
+- scripts/benchmark1.rb
 - scripts/release
 - scripts/release.rb
 - scripts/stats