simple-sql 0.4.37 → 0.4.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: eb4ff06bf7f2aeba2ded053177b20b1db9630ee1f2534672e9bb5616a1d6ad93
4
- data.tar.gz: 9ec867c270c8e4b263e61b0e222748d663021a40da6e650f2b072165e66eef21
3
+ metadata.gz: 4c46da1c022c4c166ffa76a597ab2c2357de1f93ab2c62f979a9dbac983d763f
4
+ data.tar.gz: e2b1af9258ad35e64fa845c233e0c8fac11e19d5be3d715255c7b4d56eaf980c
5
5
  SHA512:
6
- metadata.gz: 4ecf49aa237ba87bb1cab4aa026eb3ce7dbaf123e0b161a0d9fd997e47aaf46bdba454a1f85f864ac8e85bd733ffd805fdc97df0d6981764ddb32642b5e24829
7
- data.tar.gz: 8d54d93bcbfb5c06ce8be25264bf3b1f10cf0eb0039b60ea05350d0b2da23d1888d561ead99eddd984400b0cb3c6d48ada81f4bcc81a9ba6859c4a482383c904
6
+ metadata.gz: 235ff8fa9d55221c93d347c4c0f497f75cac530f49ef74f58697fbb9e9c056a5befad61a7cf365ebeae49c7895f06ef5bb01150950cfa0140bf1d35bb4d35191
7
+ data.tar.gz: 849232ea61c8478088977240e8d2cc7c987ead8bcbd9e909b14084bc566e55caa6b6d423e642be3f46d5b5a6240f470fe28bdd90e33389e141c00cdcda05e1a5
data/lib/simple/sql.rb CHANGED
@@ -21,7 +21,7 @@ module Simple
21
21
  extend self
22
22
 
23
23
  extend Forwardable
24
- delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify] => :default_connection
24
+ delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify, :costs] => :default_connection
25
25
 
26
26
  delegate [:logger, :logger=] => ::Simple::SQL::Logging
27
27
 
@@ -90,6 +90,16 @@ module Simple::SQL::ConnectionAdapter
90
90
  end
91
91
  end
92
92
 
93
+ # returns an Array [min_cost, max_cost] based on the database's estimation
94
+ def costs(sql, *args)
95
+ explanation_first = Simple::SQL.ask "EXPLAIN #{sql}", *args
96
+ unless explanation_first =~ /cost=(\d+(\.\d+))\.+(\d+(\.\d+))/
97
+ raise "Cannot determine cost"
98
+ end
99
+
100
+ [Float($1), Float($3)]
101
+ end
102
+
93
103
  # Executes a block, usually of db insert code, while holding an
94
104
  # advisory lock.
95
105
  #
@@ -50,7 +50,7 @@ module Simple
50
50
  end
51
51
 
52
52
  def slow_query_treshold=(slow_query_treshold)
53
- expect! slow_query_treshold > 0
53
+ expect! slow_query_treshold.nil? || slow_query_treshold > 0
54
54
  @slow_query_treshold = slow_query_treshold
55
55
  end
56
56
 
@@ -30,52 +30,49 @@ class ::Simple::SQL::Result < Array
30
30
  replace(records)
31
31
  end
32
32
 
33
- # returns a fast estimate for the total_count of search hits
33
+ # returns the (potentialy estimated) total count of results
34
34
  #
35
- # This is filled in when resolving a paginated scope.
36
- def total_count_estimate
37
- return @total_count_estimate if instance_variable_defined?(:@total_count_estimate)
38
-
39
- @total_count_estimate = _total_count_estimate
35
+ # This is only available for paginated scopes
36
+ def total_fast_count
37
+ @total_fast_count ||= pagination_scope.fast_count
40
38
  end
41
39
 
42
- # returns the estimated total number of pages of search hits
40
+ # returns the (potentialy estimated) total number of pages
43
41
  #
44
- # This is filled in when resolving a paginated scope.
45
- def total_pages_estimate
46
- return @total_pages_estimate if instance_variable_defined?(:@total_pages_estimate)
47
-
48
- @total_pages_estimate = (total_count_estimate * 1.0 / @pagination_scope.per).ceil if @pagination_scope
42
+ # This is only available for paginated scopes
43
+ def total_fast_pages
44
+ @total_fast_pages ||= (total_fast_count * 1.0 / pagination_scope.per).ceil
49
45
  end
50
46
 
51
- # returns the total_count of search hits
47
+ # returns the (potentialy slow) exact total count of results
52
48
  #
53
- # This is filled in when resolving a paginated scope.
49
+ # This is only available for paginated scopes
54
50
  def total_count
55
- return @total_count if instance_variable_defined?(:@total_count)
56
-
57
- @total_count = _total_count
51
+ @total_count ||= pagination_scope.count
58
52
  end
59
53
 
60
- # returns the total number of pages of search hits
54
+ # returns the (potentialy estimated) total number of pages
61
55
  #
62
- # This is filled in when resolving a paginated scope. It takes
63
- # into account the scope's "per" option.
56
+ # This is only available for paginated scopes
64
57
  def total_pages
65
- return @total_pages if instance_variable_defined?(:@total_pages)
66
-
67
- @total_pages = (total_count * 1.0 / @pagination_scope.per).ceil if @pagination_scope
58
+ @total_pages ||= (total_count * 1.0 / pagination_scope.per).ceil
68
59
  end
69
60
 
70
61
  # returns the current page number in a paginated search
71
62
  #
72
- # This is filled in when resolving a paginated scope.
63
+ # This is only available for paginated scopes
73
64
  def current_page
74
- @current_page ||= @pagination_scope.page
65
+ @current_page ||= pagination_scope.page
75
66
  end
76
67
 
77
68
  private
78
69
 
70
+ def pagination_scope
71
+ return @pagination_scope if @pagination_scope
72
+
73
+ raise "Only available only for paginated scopes"
74
+ end
75
+
79
76
  def set_pagination_info(scope)
80
77
  raise ArgumentError, "per must be > 0" unless scope.per > 0
81
78
 
@@ -86,29 +83,10 @@ class ::Simple::SQL::Result < Array
86
83
  @current_page = 1
87
84
  @total_count = 0
88
85
  @total_pages = 1
89
- @total_count_estimate = 0
90
- @total_pages_estimate = 1
86
+ @total_fast_count = 0
87
+ @total_fast_pages = 1
91
88
  else
92
89
  @pagination_scope = scope
93
90
  end
94
91
  end
95
-
96
- def _total_count_estimate
97
- return unless @pagination_scope
98
-
99
- sql = @pagination_scope.order_by(nil).to_sql(pagination: false)
100
- ::Simple::SQL.each("EXPLAIN #{sql}", *@pagination_scope.args) do |line|
101
- next unless line =~ /\brows=(\d+)/
102
- return Integer($1)
103
- end
104
-
105
- -1
106
- end
107
-
108
- def _total_count
109
- return unless @pagination_scope
110
-
111
- sql = @pagination_scope.order_by(nil).to_sql(pagination: false)
112
- ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *@pagination_scope.args)
113
- end
114
92
  end
@@ -3,6 +3,8 @@
3
3
  require_relative "scope/filters.rb"
4
4
  require_relative "scope/order.rb"
5
5
  require_relative "scope/pagination.rb"
6
+ require_relative "scope/count.rb"
7
+ require_relative "scope/count_by_groups.rb"
6
8
 
7
9
  # The Simple::SQL::Scope class helps building scopes; i.e. objects
8
10
  # that start as a quite basic SQL query, and allow one to add
@@ -0,0 +1,33 @@
1
+ class Simple::SQL::Scope
2
+ EXACT_COUNT_THRESHOLD = 10_000
3
+
4
+ # Returns the exact count of matching records
5
+ def count
6
+ sql = order_by(nil).to_sql(pagination: false)
7
+ ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
8
+ end
9
+
10
+ # Returns the fast count of matching records
11
+ #
12
+ # For counts larger than EXACT_COUNT_THRESHOLD this returns an estimate
13
+ def fast_count
14
+ estimate = estimated_count
15
+ return estimate if estimate > EXACT_COUNT_THRESHOLD
16
+
17
+ sql = order_by(nil).to_sql(pagination: false)
18
+ ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
19
+ end
20
+
21
+ private
22
+
23
+ def estimated_count
24
+ sql = order_by(nil).to_sql(pagination: false)
25
+ ::Simple::SQL.each("EXPLAIN #{sql}", *args) do |line|
26
+ next unless line =~ /\brows=(\d+)/
27
+
28
+ return Integer($1)
29
+ end
30
+
31
+ -1
32
+ end
33
+ end
@@ -0,0 +1,79 @@
1
+ # rubocop:disable Metrics/AbcSize
2
+ # rubocop:disable Metrics/MethodLength
3
+
4
+ class Simple::SQL::Scope
5
+ # Potentially fast implementation of returning all different values for a specific group.
6
+ #
7
+ # For example:
8
+ #
9
+ # Scope.new("SELECT * FROM users").enumerate_groups("gender") -> [ "female", "male" ]
10
+ #
11
+ # It is possible to enumerate over multiple attributes, for example:
12
+ #
13
+ # scope.enumerate_groups fragment: "ARRAY[workflow, queue]"
14
+ #
15
+ # In any case it is important that an index exists that the database can use to group
16
+ # by the +sql_fragment+, for example:
17
+ #
18
+ # CREATE INDEX ix3 ON table((ARRAY[workflow, queue]));
19
+ #
20
+ def enumerate_groups(sql_fragment)
21
+ sql = order_by(nil).to_sql(pagination: false)
22
+
23
+ _, max_cost = ::Simple::SQL.costs "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
24
+ raise "enumerate_groups: takes too much time. Make sure to create a suitable index" if max_cost > 10_000
25
+
26
+ groups = []
27
+ var_name = "$#{@args.count + 1}"
28
+ cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
29
+
30
+ while cur
31
+ groups << cur
32
+ cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq"" WHERE #{sql_fragment} > #{var_name}", *args, cur
33
+ end
34
+
35
+ groups
36
+ end
37
+
38
+ def count_by(sql_fragment)
39
+ sql = order_by(nil).to_sql(pagination: false)
40
+
41
+ recs = ::Simple::SQL.all "SELECT #{sql_fragment} AS group, COUNT(*) AS count FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
42
+ Hash[recs]
43
+ end
44
+
45
+ def fast_count_by(sql_fragment)
46
+ sql = order_by(nil).to_sql(pagination: false)
47
+
48
+ _, max_cost = ::Simple::SQL.costs "SELECT COUNT(*) FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
49
+
50
+ return count_by(sql_fragment) if max_cost < 10_000
51
+
52
+ # iterate over all groups, estimating the count for each. If the count is
53
+ # less than EXACT_COUNT_THRESHOLD we ask for the exact count in that and
54
+ # similarily sparse groups.
55
+ var_name = "$#{@args.count + 1}"
56
+
57
+ counts = {}
58
+ sparse_groups = []
59
+ enumerate_groups(sql_fragment).each do |group|
60
+ scope = ::Simple::SQL::Scope.new("SELECT * FROM (#{sql}) sq WHERE #{sql_fragment}=#{var_name}", *args, group)
61
+ counts[group] = scope.send(:estimated_count)
62
+ sparse_groups << group if estimated_count < EXACT_COUNT_THRESHOLD
63
+ end
64
+
65
+ # fetch exact counts in all sparse_groups
66
+ unless sparse_groups.empty?
67
+ sparse_counts = ::Simple::SQL.all <<~SQL, *args, sparse_groups
68
+ SELECT #{sql_fragment} AS group, COUNT(*) AS count
69
+ FROM (#{sql}) sq
70
+ WHERE #{sql_fragment} = ANY(#{var_name})
71
+ GROUP BY #{sql_fragment}
72
+ SQL
73
+
74
+ counts.update Hash[sparse_counts]
75
+ end
76
+
77
+ counts
78
+ end
79
+ end
@@ -1,5 +1,5 @@
1
1
  module Simple
2
2
  module SQL
3
- VERSION = "0.4.37"
3
+ VERSION = "0.4.38"
4
4
  end
5
5
  end
@@ -0,0 +1,44 @@
1
+ require "spec_helper"
2
+
3
+ describe "Simple::SQL::Scope#count_by" do
4
+ let!(:users) { 1.upto(10).map { |i| create(:user, role_id: i) } }
5
+ let(:all_role_ids) { SQL.all("SELECT DISTINCT role_id FROM users") }
6
+ let(:scope) { SQL::Scope.new("SELECT * FROM users") }
7
+
8
+ describe "enumerate_groups" do
9
+ it "returns all groups" do
10
+ expect(scope.enumerate_groups("role_id")).to contain_exactly(*all_role_ids)
11
+ expect(scope.where("role_id < 4").enumerate_groups("role_id")).to contain_exactly(*(1.upto(3).to_a))
12
+ end
13
+ end
14
+
15
+ describe "count_by" do
16
+ it "counts all groups" do
17
+ create(:user, role_id: 1)
18
+ create(:user, role_id: 1)
19
+ create(:user, role_id: 1)
20
+
21
+ expect(scope.count_by("role_id")).to include(1 => 4)
22
+ expect(scope.count_by("role_id")).to include(2 => 1)
23
+ expect(scope.count_by("role_id").keys).to contain_exactly(*all_role_ids)
24
+ end
25
+ end
26
+
27
+ describe "fast_count_by" do
28
+ before do
29
+ # 10_000 is chosen "magically". It is large enough to switch to the fast algorithm,
30
+ # but
31
+ allow(::Simple::SQL).to receive(:costs).and_return([0, 10_000])
32
+ end
33
+
34
+ it "counts all groups" do
35
+ create(:user, role_id: 1)
36
+ create(:user, role_id: 1)
37
+ create(:user, role_id: 1)
38
+
39
+ expect(scope.fast_count_by("role_id")).to include(1 => 4)
40
+ expect(scope.fast_count_by("role_id")).to include(2 => 1)
41
+ expect(scope.fast_count_by("role_id").keys).to contain_exactly(*all_role_ids)
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,29 @@
1
+ require "spec_helper"
2
+
3
+ describe "Simple::SQL::Scope#count" do
4
+ let!(:users) { 1.upto(USER_COUNT).map { create(:user) } }
5
+ let(:min_user_id) { SQL.ask "SELECT min(id) FROM users" }
6
+ let(:scope) { SQL::Scope.new("SELECT * FROM users") }
7
+
8
+ describe "exact count" do
9
+ it "counts" do
10
+ expect(scope.count).to eq(USER_COUNT)
11
+ end
12
+
13
+ it "evaluates conditions" do
14
+ expect(scope.where("id < $1", min_user_id).count).to eq(0)
15
+ expect(scope.where("id <= $1", min_user_id).count).to eq(1)
16
+ end
17
+ end
18
+
19
+ describe "fast count" do
20
+ it "counts" do
21
+ expect(scope.fast_count).to eq(USER_COUNT)
22
+ end
23
+
24
+ it "evaluates conditions" do
25
+ expect(scope.where("id < $1", min_user_id).fast_count).to eq(0)
26
+ expect(scope.where("id <= $1", min_user_id).fast_count).to eq(1)
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,16 @@
1
+ require "spec_helper"
2
+
3
+ describe "Simple::SQL logging" do
4
+ context 'when running a slow query' do
5
+ before do
6
+ SQL::Logging.slow_query_treshold = 0.05
7
+ end
8
+ after do
9
+ SQL::Logging.slow_query_treshold = nil
10
+ end
11
+
12
+ it "does not crash" do
13
+ SQL.ask "SELECT pg_sleep(0.1)"
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,57 @@
1
+ require "spec_helper"
2
+
3
+ describe "Simple::SQL::Result counts" do
4
+ let!(:users) { 1.upto(USER_COUNT).map { create(:user) } }
5
+ let(:min_user_id) { SQL.ask "SELECT min(id) FROM users" }
6
+ let(:scope) { SQL::Scope.new("SELECT * FROM users") }
7
+ let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
8
+
9
+ describe "exact counting" do
10
+ it "counts" do
11
+ result = SQL.all(paginated_scope)
12
+ expect(result.total_count).to eq(USER_COUNT)
13
+ expect(result.total_pages).to eq(USER_COUNT)
14
+ expect(result.current_page).to eq(1)
15
+ end
16
+ end
17
+
18
+ describe "fast counting" do
19
+ it "counts fast" do
20
+ result = SQL.all(paginated_scope)
21
+
22
+ expect(result.total_fast_count).to eq(USER_COUNT)
23
+ expect(result.total_fast_pages).to eq(USER_COUNT)
24
+ expect(result.current_page).to eq(1)
25
+ end
26
+ end
27
+
28
+ context 'when running with a non-paginated paginated_scope' do
29
+ it "raises errors" do
30
+ result = SQL.all(scope)
31
+
32
+ expect { result.total_count }.to raise_error(RuntimeError)
33
+ expect { result.total_pages }.to raise_error(RuntimeError)
34
+ expect { result.current_page }.to raise_error(RuntimeError)
35
+ expect { result.total_fast_count }.to raise_error(RuntimeError)
36
+ expect { result.total_fast_pages }.to raise_error(RuntimeError)
37
+ end
38
+ end
39
+
40
+
41
+ context 'when running with an empty, paginated paginated_scope' do
42
+ let(:scope) { SQL::Scope.new("SELECT * FROM users WHERE FALSE") }
43
+ let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
44
+
45
+ it "returns correct results" do
46
+ result = SQL.all(paginated_scope)
47
+
48
+ expect(result.total_count).to eq(0)
49
+ expect(result.total_pages).to eq(1)
50
+
51
+ expect(result.total_fast_count).to eq(0)
52
+ expect(result.total_fast_pages).to eq(1)
53
+
54
+ expect(result.current_page).to eq(1)
55
+ end
56
+ end
57
+ end
data/spec/spec_helper.rb CHANGED
@@ -14,7 +14,9 @@ Dir.glob("./spec/support/**/*.rb").sort.each { |path| load path }
14
14
  require "simple/sql"
15
15
 
16
16
  unless ENV["USE_ACTIVE_RECORD"]
17
- Simple::SQL.connect!
17
+ database_url = Simple::SQL::Config.determine_url
18
+
19
+ Simple::SQL.connect! database_url
18
20
  Simple::SQL.ask "DELETE FROM users"
19
21
  end
20
22
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: simple-sql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.37
4
+ version: 0.4.38
5
5
  platform: ruby
6
6
  authors:
7
7
  - radiospiel
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-03-22 00:00:00.000000000 Z
12
+ date: 2019-03-23 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: pg_array_parser
@@ -218,6 +218,8 @@ files:
218
218
  - lib/simple/sql/result/association_loader.rb
219
219
  - lib/simple/sql/result/records.rb
220
220
  - lib/simple/sql/scope.rb
221
+ - lib/simple/sql/scope/count.rb
222
+ - lib/simple/sql/scope/count_by_groups.rb
221
223
  - lib/simple/sql/scope/filters.rb
222
224
  - lib/simple/sql/scope/order.rb
223
225
  - lib/simple/sql/scope/pagination.rb
@@ -232,11 +234,15 @@ files:
232
234
  - spec/simple/sql/associations_spec.rb
233
235
  - spec/simple/sql/config_spec.rb
234
236
  - spec/simple/sql/conversion_spec.rb
237
+ - spec/simple/sql/count_by_groups_spec.rb
238
+ - spec/simple/sql/count_spec.rb
235
239
  - spec/simple/sql/duplicate_spec.rb
236
240
  - spec/simple/sql/duplicate_unique_spec.rb
237
241
  - spec/simple/sql/each_spec.rb
238
242
  - spec/simple/sql/insert_spec.rb
243
+ - spec/simple/sql/logging_spec.rb
239
244
  - spec/simple/sql/reflection_spec.rb
245
+ - spec/simple/sql/result_count_spec.rb
240
246
  - spec/simple/sql/scope_spec.rb
241
247
  - spec/simple/sql/version_spec.rb
242
248
  - spec/simple/sql_locked_spec.rb
@@ -278,11 +284,15 @@ test_files:
278
284
  - spec/simple/sql/associations_spec.rb
279
285
  - spec/simple/sql/config_spec.rb
280
286
  - spec/simple/sql/conversion_spec.rb
287
+ - spec/simple/sql/count_by_groups_spec.rb
288
+ - spec/simple/sql/count_spec.rb
281
289
  - spec/simple/sql/duplicate_spec.rb
282
290
  - spec/simple/sql/duplicate_unique_spec.rb
283
291
  - spec/simple/sql/each_spec.rb
284
292
  - spec/simple/sql/insert_spec.rb
293
+ - spec/simple/sql/logging_spec.rb
285
294
  - spec/simple/sql/reflection_spec.rb
295
+ - spec/simple/sql/result_count_spec.rb
286
296
  - spec/simple/sql/scope_spec.rb
287
297
  - spec/simple/sql/version_spec.rb
288
298
  - spec/simple/sql_locked_spec.rb