simple-sql 0.4.37 → 0.4.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb4ff06bf7f2aeba2ded053177b20b1db9630ee1f2534672e9bb5616a1d6ad93
-  data.tar.gz: 9ec867c270c8e4b263e61b0e222748d663021a40da6e650f2b072165e66eef21
+  metadata.gz: 4c46da1c022c4c166ffa76a597ab2c2357de1f93ab2c62f979a9dbac983d763f
+  data.tar.gz: e2b1af9258ad35e64fa845c233e0c8fac11e19d5be3d715255c7b4d56eaf980c
 SHA512:
-  metadata.gz: 4ecf49aa237ba87bb1cab4aa026eb3ce7dbaf123e0b161a0d9fd997e47aaf46bdba454a1f85f864ac8e85bd733ffd805fdc97df0d6981764ddb32642b5e24829
-  data.tar.gz: 8d54d93bcbfb5c06ce8be25264bf3b1f10cf0eb0039b60ea05350d0b2da23d1888d561ead99eddd984400b0cb3c6d48ada81f4bcc81a9ba6859c4a482383c904
+  metadata.gz: 235ff8fa9d55221c93d347c4c0f497f75cac530f49ef74f58697fbb9e9c056a5befad61a7cf365ebeae49c7895f06ef5bb01150950cfa0140bf1d35bb4d35191
+  data.tar.gz: 849232ea61c8478088977240e8d2cc7c987ead8bcbd9e909b14084bc566e55caa6b6d423e642be3f46d5b5a6240f470fe28bdd90e33389e141c00cdcda05e1a5

data/lib/simple/sql.rb

@@ -21,7 +21,7 @@ module Simple
     extend self
 
     extend Forwardable
-    delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify] => :default_connection
+    delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify, :costs] => :default_connection
 
     delegate [:logger, :logger=] => ::Simple::SQL::Logging
 

data/lib/simple/sql/connection_adapter.rb

@@ -90,6 +90,16 @@ module Simple::SQL::ConnectionAdapter
     end
   end
 
+  # returns an Array [min_cost, max_cost] based on the database's estimation
+  def costs(sql, *args)
+    explanation_first = Simple::SQL.ask "EXPLAIN #{sql}", *args
+    unless explanation_first =~ /cost=(\d+(\.\d+))\.+(\d+(\.\d+))/
+      raise "Cannot determine cost"
+    end
+
+    [Float($1), Float($3)]
+  end
+
   # Executes a block, usually of db insert code, while holding an
   # advisory lock.
   #

data/lib/simple/sql/logging.rb

@@ -50,7 +50,7 @@ module Simple
       end
 
       def slow_query_treshold=(slow_query_treshold)
-        expect! slow_query_treshold > 0
+        expect! slow_query_treshold.nil? || slow_query_treshold > 0
         @slow_query_treshold = slow_query_treshold
       end
 

data/lib/simple/sql/result.rb

@@ -30,52 +30,49 @@ class ::Simple::SQL::Result < Array
     replace(records)
   end
 
-  # returns a fast estimate for the total_count of search hits
+  # returns the (potentialy estimated) total count of results
   #
-  # This is filled in when resolving a paginated scope.
-  def total_count_estimate
-    return @total_count_estimate if instance_variable_defined?(:@total_count_estimate)
-
-    @total_count_estimate = _total_count_estimate
+  # This is only available for paginated scopes
+  def total_fast_count
+    @total_fast_count ||= pagination_scope.fast_count
   end
 
-  # returns the estimated total number of pages of search hits
+  # returns the (potentialy estimated) total number of pages
   #
-  # This is filled in when resolving a paginated scope.
-  def total_pages_estimate
-    return @total_pages_estimate if instance_variable_defined?(:@total_pages_estimate)
-
-    @total_pages_estimate = (total_count_estimate * 1.0 / @pagination_scope.per).ceil if @pagination_scope
+  # This is only available for paginated scopes
+  def total_fast_pages
+    @total_fast_pages ||= (total_fast_count * 1.0 / pagination_scope.per).ceil
   end
 
-  # returns the total_count of search hits
+  # returns the (potentialy slow) exact total count of results
   #
-  # This is filled in when resolving a paginated scope.
+  # This is only available for paginated scopes
   def total_count
-    return @total_count if instance_variable_defined?(:@total_count)
-
-    @total_count = _total_count
+    @total_count ||= pagination_scope.count
   end
 
-  # returns the total number of pages of search hits
+  # returns the (potentialy estimated) total number of pages
   #
-  # This is filled in when resolving a paginated scope. It takes
-  # into account the scope's "per" option.
+  # This is only available for paginated scopes
   def total_pages
-    return @total_pages if instance_variable_defined?(:@total_pages)
-
-    @total_pages = (total_count * 1.0 / @pagination_scope.per).ceil if @pagination_scope
+    @total_pages ||= (total_count * 1.0 / pagination_scope.per).ceil
   end
 
   # returns the current page number in a paginated search
   #
-  # This is filled in when resolving a paginated scope.
+  # This is only available for paginated scopes
   def current_page
-    @current_page ||= @pagination_scope.page
+    @current_page ||= pagination_scope.page
   end
 
   private
 
+  def pagination_scope
+    return @pagination_scope if @pagination_scope
+
+    raise "Only available only for paginated scopes"
+  end
+
   def set_pagination_info(scope)
     raise ArgumentError, "per must be > 0" unless scope.per > 0
 
@@ -86,29 +83,10 @@ class ::Simple::SQL::Result < Array
       @current_page = 1
       @total_count  = 0
       @total_pages  = 1
-      @total_count_estimate  = 0
-      @total_pages_estimate  = 1
+      @total_fast_count  = 0
+      @total_fast_pages  = 1
     else
       @pagination_scope = scope
     end
   end
-
-  def _total_count_estimate
-    return unless @pagination_scope
-
-    sql = @pagination_scope.order_by(nil).to_sql(pagination: false)
-    ::Simple::SQL.each("EXPLAIN #{sql}", *@pagination_scope.args) do |line|
-      next unless line =~ /\brows=(\d+)/
-      return Integer($1)
-    end
-
-    -1
-  end
-
-  def _total_count
-    return unless @pagination_scope
-
-    sql = @pagination_scope.order_by(nil).to_sql(pagination: false)
-    ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *@pagination_scope.args)
-  end
 end

data/lib/simple/sql/scope.rb

@@ -3,6 +3,8 @@
 require_relative "scope/filters.rb"
 require_relative "scope/order.rb"
 require_relative "scope/pagination.rb"
+require_relative "scope/count.rb"
+require_relative "scope/count_by_groups.rb"
 
 # The Simple::SQL::Scope class helps building scopes; i.e. objects
 # that start as a quite basic SQL query, and allow one to add

data/lib/simple/sql/scope/count.rb

@@ -0,0 +1,33 @@
+class Simple::SQL::Scope
+  EXACT_COUNT_THRESHOLD = 10_000
+
+  # Returns the exact count of matching records
+  def count
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
+  end
+
+  # Returns the fast count of matching records
+  #
+  # For counts larger than EXACT_COUNT_THRESHOLD this returns an estimate
+  def fast_count
+    estimate = estimated_count
+    return estimate if estimate > EXACT_COUNT_THRESHOLD
+
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
+  end
+
+  private
+
+  def estimated_count
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.each("EXPLAIN #{sql}", *args) do |line|
+      next unless line =~ /\brows=(\d+)/
+
+      return Integer($1)
+    end
+
+    -1
+  end
+end

data/lib/simple/sql/scope/count_by_groups.rb

@@ -0,0 +1,79 @@
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/MethodLength
+
+class Simple::SQL::Scope
+  # Potentially fast implementation of returning all different values for a specific group.
+  #
+  # For example:
+  #
+  #   Scope.new("SELECT * FROM users").enumerate_groups("gender") -> [ "female", "male" ]
+  #
+  # It is possible to enumerate over multiple attributes, for example:
+  #
+  #   scope.enumerate_groups fragment: "ARRAY[workflow, queue]"
+  #
+  # In any case it is important that an index exists that the database can use to group
+  # by the +sql_fragment+, for example:
+  #
+  #   CREATE INDEX ix3 ON table((ARRAY[workflow, queue]));
+  #
+  def enumerate_groups(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    _, max_cost = ::Simple::SQL.costs "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+    raise "enumerate_groups: takes too much time. Make sure to create a suitable index" if max_cost > 10_000
+
+    groups = []
+    var_name = "$#{@args.count + 1}"
+    cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+
+    while cur
+      groups << cur
+      cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq"" WHERE #{sql_fragment} > #{var_name}", *args, cur
+    end
+
+    groups
+  end
+
+  def count_by(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    recs = ::Simple::SQL.all "SELECT #{sql_fragment} AS group, COUNT(*) AS count FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+    Hash[recs]
+  end
+
+  def fast_count_by(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    _, max_cost = ::Simple::SQL.costs "SELECT COUNT(*) FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+
+    return count_by(sql_fragment) if max_cost < 10_000
+
+    # iterate over all groups, estimating the count for each. If the count is
+    # less than EXACT_COUNT_THRESHOLD we ask for the exact count in that and
+    # similarily sparse groups.
+    var_name = "$#{@args.count + 1}"
+
+    counts = {}
+    sparse_groups = []
+    enumerate_groups(sql_fragment).each do |group|
+      scope = ::Simple::SQL::Scope.new("SELECT * FROM (#{sql}) sq WHERE #{sql_fragment}=#{var_name}", *args, group)
+      counts[group] = scope.send(:estimated_count)
+      sparse_groups << group if estimated_count < EXACT_COUNT_THRESHOLD
+    end
+
+    # fetch exact counts in all sparse_groups
+    unless sparse_groups.empty?
+      sparse_counts = ::Simple::SQL.all <<~SQL, *args, sparse_groups
+        SELECT #{sql_fragment} AS group, COUNT(*) AS count
+        FROM (#{sql}) sq
+        WHERE #{sql_fragment} = ANY(#{var_name})
+        GROUP BY #{sql_fragment}
+      SQL
+
+      counts.update Hash[sparse_counts]
+    end
+
+    counts
+  end
+end

data/lib/simple/sql/version.rb

@@ -1,5 +1,5 @@
 module Simple
   module SQL
-    VERSION = "0.4.37"
+    VERSION = "0.4.38"
   end
 end

data/spec/simple/sql/count_by_groups_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+
+describe "Simple::SQL::Scope#count_by" do
+  let!(:users)        { 1.upto(10).map { |i| create(:user, role_id: i) } }
+  let(:all_role_ids)  { SQL.all("SELECT DISTINCT role_id FROM users") }
+  let(:scope)         { SQL::Scope.new("SELECT * FROM users") }
+
+  describe "enumerate_groups" do
+    it "returns all groups" do
+      expect(scope.enumerate_groups("role_id")).to contain_exactly(*all_role_ids)
+      expect(scope.where("role_id < 4").enumerate_groups("role_id")).to contain_exactly(*(1.upto(3).to_a))
+    end
+  end
+
+  describe "count_by" do
+    it "counts all groups" do
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+
+      expect(scope.count_by("role_id")).to include(1 => 4)
+      expect(scope.count_by("role_id")).to include(2 => 1)
+      expect(scope.count_by("role_id").keys).to contain_exactly(*all_role_ids)
+    end
+  end
+
+  describe "fast_count_by" do
+    before do
+      # 10_000 is chosen "magically". It is large enough to switch to the fast algorithm,
+      # but 
+      allow(::Simple::SQL).to receive(:costs).and_return([0, 10_000])
+    end
+    
+    it "counts all groups" do
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+
+      expect(scope.fast_count_by("role_id")).to include(1 => 4)
+      expect(scope.fast_count_by("role_id")).to include(2 => 1)
+      expect(scope.fast_count_by("role_id").keys).to contain_exactly(*all_role_ids)
+    end
+  end
+end

data/spec/simple/sql/count_spec.rb

@@ -0,0 +1,29 @@
+require "spec_helper"
+
+describe "Simple::SQL::Scope#count" do
+  let!(:users)      { 1.upto(USER_COUNT).map { create(:user) } }
+  let(:min_user_id) { SQL.ask "SELECT min(id) FROM users" }
+  let(:scope)       { SQL::Scope.new("SELECT * FROM users") }
+
+  describe "exact count" do
+    it "counts" do
+      expect(scope.count).to eq(USER_COUNT)
+    end
+
+    it "evaluates conditions" do
+      expect(scope.where("id < $1", min_user_id).count).to eq(0)
+      expect(scope.where("id <= $1", min_user_id).count).to eq(1)
+    end
+  end
+
+  describe "fast count" do
+    it "counts" do
+      expect(scope.fast_count).to eq(USER_COUNT)
+    end
+
+    it "evaluates conditions" do
+      expect(scope.where("id < $1", min_user_id).fast_count).to eq(0)
+      expect(scope.where("id <= $1", min_user_id).fast_count).to eq(1)
+    end
+  end
+end

data/spec/simple/sql/logging_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe "Simple::SQL logging" do
+  context 'when running a slow query' do
+    before do
+      SQL::Logging.slow_query_treshold = 0.05
+    end
+    after do
+      SQL::Logging.slow_query_treshold = nil
+    end
+
+    it "does not crash" do
+      SQL.ask "SELECT pg_sleep(0.1)"
+    end
+  end
+end

data/spec/simple/sql/result_count_spec.rb

@@ -0,0 +1,57 @@
+require "spec_helper"
+
+describe "Simple::SQL::Result counts" do
+  let!(:users)          { 1.upto(USER_COUNT).map { create(:user) } }
+  let(:min_user_id)     { SQL.ask "SELECT min(id) FROM users" }
+  let(:scope)           { SQL::Scope.new("SELECT * FROM users") }
+  let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
+
+  describe "exact counting" do
+    it "counts" do
+      result = SQL.all(paginated_scope)
+      expect(result.total_count).to eq(USER_COUNT)
+      expect(result.total_pages).to eq(USER_COUNT)
+      expect(result.current_page).to eq(1)
+    end
+  end
+  
+  describe "fast counting" do
+    it "counts fast" do
+      result = SQL.all(paginated_scope)
+
+      expect(result.total_fast_count).to eq(USER_COUNT)
+      expect(result.total_fast_pages).to eq(USER_COUNT)
+      expect(result.current_page).to eq(1)
+    end
+  end
+
+  context 'when running with a non-paginated paginated_scope' do
+    it "raises errors" do
+      result = SQL.all(scope)
+
+      expect { result.total_count }.to raise_error(RuntimeError)
+      expect { result.total_pages }.to raise_error(RuntimeError)
+      expect { result.current_page }.to raise_error(RuntimeError)
+      expect { result.total_fast_count }.to raise_error(RuntimeError)
+      expect { result.total_fast_pages }.to raise_error(RuntimeError)
+    end
+  end
+
+
+  context 'when running with an empty, paginated paginated_scope' do
+    let(:scope)           { SQL::Scope.new("SELECT * FROM users WHERE FALSE") }
+    let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
+
+    it "returns correct results" do
+      result = SQL.all(paginated_scope)
+
+      expect(result.total_count).to eq(0)
+      expect(result.total_pages).to eq(1)
+
+      expect(result.total_fast_count).to eq(0)
+      expect(result.total_fast_pages).to eq(1)
+
+      expect(result.current_page).to eq(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -14,7 +14,9 @@ Dir.glob("./spec/support/**/*.rb").sort.each { |path| load path }
 require "simple/sql"
 
 unless ENV["USE_ACTIVE_RECORD"]
-  Simple::SQL.connect!
+  database_url = Simple::SQL::Config.determine_url
+
+  Simple::SQL.connect! database_url
   Simple::SQL.ask "DELETE FROM users"
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple-sql
 version: !ruby/object:Gem::Version
-  version: 0.4.37
+  version: 0.4.38
 platform: ruby
 authors:
 - radiospiel
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-22 00:00:00.000000000 Z
+date: 2019-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg_array_parser
@@ -218,6 +218,8 @@ files:
 - lib/simple/sql/result/association_loader.rb
 - lib/simple/sql/result/records.rb
 - lib/simple/sql/scope.rb
+- lib/simple/sql/scope/count.rb
+- lib/simple/sql/scope/count_by_groups.rb
 - lib/simple/sql/scope/filters.rb
 - lib/simple/sql/scope/order.rb
 - lib/simple/sql/scope/pagination.rb
@@ -232,11 +234,15 @@ files:
 - spec/simple/sql/associations_spec.rb
 - spec/simple/sql/config_spec.rb
 - spec/simple/sql/conversion_spec.rb
+- spec/simple/sql/count_by_groups_spec.rb
+- spec/simple/sql/count_spec.rb
 - spec/simple/sql/duplicate_spec.rb
 - spec/simple/sql/duplicate_unique_spec.rb
 - spec/simple/sql/each_spec.rb
 - spec/simple/sql/insert_spec.rb
+- spec/simple/sql/logging_spec.rb
 - spec/simple/sql/reflection_spec.rb
+- spec/simple/sql/result_count_spec.rb
 - spec/simple/sql/scope_spec.rb
 - spec/simple/sql/version_spec.rb
 - spec/simple/sql_locked_spec.rb
@@ -278,11 +284,15 @@ test_files:
 - spec/simple/sql/associations_spec.rb
 - spec/simple/sql/config_spec.rb
 - spec/simple/sql/conversion_spec.rb
+- spec/simple/sql/count_by_groups_spec.rb
+- spec/simple/sql/count_spec.rb
 - spec/simple/sql/duplicate_spec.rb
 - spec/simple/sql/duplicate_unique_spec.rb
 - spec/simple/sql/each_spec.rb
 - spec/simple/sql/insert_spec.rb
+- spec/simple/sql/logging_spec.rb
 - spec/simple/sql/reflection_spec.rb
+- spec/simple/sql/result_count_spec.rb
 - spec/simple/sql/scope_spec.rb
 - spec/simple/sql/version_spec.rb
 - spec/simple/sql_locked_spec.rb