postgresql_cursor 0.6.4 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/.travis.yml +27 -0
- data/Appraisals +12 -0
- data/Rakefile +1 -0
- data/gemfiles/activerecord_4.gemfile +8 -0
- data/gemfiles/activerecord_5.gemfile +7 -0
- data/gemfiles/activerecord_6.gemfile +7 -0
- data/lib/postgresql_cursor/cursor.rb +77 -80
- data/lib/postgresql_cursor/version.rb +1 -1
- data/postgresql_cursor.gemspec +21 -25
- data/test/helper.rb +6 -0
- data/test/test_postgresql_cursor.rb +72 -38
- data/test-app/Gemfile +4 -11
- metadata +30 -14
- data/Gemfile.lock +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c85acb2e98683cf55a34048f3b4b20ee78b210741f22dcb634bd5c4a2fe53395
|
4
|
+
data.tar.gz: 2c9c6bef23d6a18d6e878b8bf0cd5f85ec94c4fe013238ca95fa1a95f9af8ada
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 707d3922478b915d36b503a9e3ef4973e2064d7e1b0388b2b31191e91e4bef1e118ac3d08eb8aaeae4a7b94e6c099c522116d7651aa23f0645f16541e8b68793
|
7
|
+
data.tar.gz: 7bda9582106b1d8472a7c09d3ce9db8e66a519f2f22f2903c9e29b5f9814083f81ffa3df3c2aa171b5e2f8445c2bf05f0fac5025176c77ef74ae7c7733c0dec6
|
data/.gitignore
CHANGED
data/.travis.yml
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 2.6.5
|
4
|
+
- 2.7.1
|
5
|
+
before_install:
|
6
|
+
- sudo apt-get update
|
7
|
+
- sudo apt-get --yes remove postgresql\*
|
8
|
+
- sudo apt-get install -y postgresql-12 postgresql-client-12
|
9
|
+
- sudo cp /etc/postgresql/{9.6,12}/main/pg_hba.conf
|
10
|
+
- sudo service postgresql restart 12
|
11
|
+
gemfile:
|
12
|
+
- gemfiles/activerecord_4.gemfile
|
13
|
+
- gemfiles/activerecord_5.gemfile
|
14
|
+
- gemfiles/activerecord_6.gemfile
|
15
|
+
matrix:
|
16
|
+
exclude:
|
17
|
+
- rvm: 2.7.1
|
18
|
+
gemfile: gemfiles/activerecord_4.gemfile
|
19
|
+
services:
|
20
|
+
- postgresql
|
21
|
+
before_script:
|
22
|
+
- psql -c 'create database postgresql_cursor_test;' -U postgres
|
23
|
+
- psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
|
24
|
+
- psql -c 'create table products ( id serial primary key, data varchar);' -U postgres -d postgresql_cursor_test
|
25
|
+
- psql -c 'create table prices ( id serial primary key, data varchar, product_id integer);' -U postgres -d postgresql_cursor_test
|
26
|
+
addons:
|
27
|
+
postgresql: '12.3'
|
data/Appraisals
ADDED
data/Rakefile
CHANGED
@@ -21,4 +21,5 @@ desc "Setup testing database and table"
|
|
21
21
|
task :setup do
|
22
22
|
sh %q(createdb postgresql_cursor_test)
|
23
23
|
sh %Q<echo "create table products ( id serial primary key, data varchar);" | psql postgresql_cursor_test>
|
24
|
+
sh %Q<echo "create table prices ( id serial primary key, data varchar, product_id integer);" | psql postgresql_cursor_test>
|
24
25
|
end
|
@@ -41,29 +41,32 @@ module PostgreSQLCursor
|
|
41
41
|
# PostgreSQLCursor::Cursor.new("select ....")
|
42
42
|
#
|
43
43
|
# Returns the cursor object when called with new.
|
44
|
-
def initialize(sql, options={})
|
45
|
-
@sql
|
46
|
-
@options
|
44
|
+
def initialize(sql, options = {})
|
45
|
+
@sql = sql
|
46
|
+
@options = options
|
47
47
|
@connection = @options.fetch(:connection) { ::ActiveRecord::Base.connection }
|
48
|
-
@count
|
49
|
-
@iterate
|
50
|
-
@batched
|
48
|
+
@count = 0
|
49
|
+
@iterate = options[:instances] ? :each_instance : :each_row
|
50
|
+
@batched = false
|
51
51
|
end
|
52
52
|
|
53
|
-
# Specify the type to instantiate, or reset to return a Hash
|
54
|
-
|
55
|
-
|
53
|
+
# Specify the type to instantiate, or reset to return a Hash.
|
54
|
+
#
|
55
|
+
# Explicitly check for type class to prevent calling equality
|
56
|
+
# operator on active record relation, which will load it.
|
57
|
+
def iterate_type(type = nil)
|
58
|
+
if type.nil? || (type.instance_of?(Class) && type == Hash)
|
56
59
|
@iterate = :each_row
|
57
|
-
elsif type == Array
|
60
|
+
elsif type.instance_of?(Class) && type == Array
|
58
61
|
@iterate = :each_array
|
59
62
|
else
|
60
63
|
@iterate = :each_instance
|
61
|
-
@type
|
64
|
+
@type = type
|
62
65
|
end
|
63
66
|
self
|
64
67
|
end
|
65
68
|
|
66
|
-
def iterate_batched(batched=true)
|
69
|
+
def iterate_batched(batched = true)
|
67
70
|
@batched = batched
|
68
71
|
self
|
69
72
|
end
|
@@ -76,16 +79,16 @@ module PostgreSQLCursor
|
|
76
79
|
# Returns the count of rows processed
|
77
80
|
def each(&block)
|
78
81
|
if @iterate == :each_row
|
79
|
-
@batched ?
|
82
|
+
@batched ? each_row_batch(&block) : each_row(&block)
|
80
83
|
elsif @iterate == :each_array
|
81
|
-
@batched ?
|
84
|
+
@batched ? each_array_batch(&block) : each_array(&block)
|
82
85
|
else
|
83
|
-
@batched ?
|
86
|
+
@batched ? each_instance_batch(@type, &block) : each_instance(@type, &block)
|
84
87
|
end
|
85
88
|
end
|
86
89
|
|
87
90
|
def each_row(&block)
|
88
|
-
|
91
|
+
each_tuple do |row|
|
89
92
|
row = row.symbolize_keys if @options[:symbolize_keys]
|
90
93
|
block.call(row)
|
91
94
|
end
|
@@ -95,7 +98,7 @@ module PostgreSQLCursor
|
|
95
98
|
old_iterate = @iterate
|
96
99
|
@iterate = :each_array
|
97
100
|
begin
|
98
|
-
rv =
|
101
|
+
rv = each_tuple do |row|
|
99
102
|
block.call(row)
|
100
103
|
end
|
101
104
|
ensure
|
@@ -104,11 +107,11 @@ module PostgreSQLCursor
|
|
104
107
|
rv
|
105
108
|
end
|
106
109
|
|
107
|
-
def each_instance(klass=nil, &block)
|
110
|
+
def each_instance(klass = nil, &block)
|
108
111
|
klass ||= @type
|
109
|
-
|
112
|
+
each_tuple do |row|
|
110
113
|
if ::ActiveRecord::VERSION::MAJOR < 4
|
111
|
-
model = klass.send(:instantiate,row)
|
114
|
+
model = klass.send(:instantiate, row)
|
112
115
|
else
|
113
116
|
@column_types ||= column_types
|
114
117
|
model = klass.send(:instantiate, row, @column_types)
|
@@ -118,7 +121,7 @@ module PostgreSQLCursor
|
|
118
121
|
end
|
119
122
|
|
120
123
|
def each_row_batch(&block)
|
121
|
-
|
124
|
+
each_batch do |batch|
|
122
125
|
batch.map!(&:symbolize_keys) if @options[:symbolize_keys]
|
123
126
|
block.call(batch)
|
124
127
|
end
|
@@ -128,7 +131,7 @@ module PostgreSQLCursor
|
|
128
131
|
old_iterate = @iterate
|
129
132
|
@iterate = :each_array
|
130
133
|
begin
|
131
|
-
rv =
|
134
|
+
rv = each_batch do |batch|
|
132
135
|
block.call(batch)
|
133
136
|
end
|
134
137
|
ensure
|
@@ -137,15 +140,15 @@ module PostgreSQLCursor
|
|
137
140
|
rv
|
138
141
|
end
|
139
142
|
|
140
|
-
def each_instance_batch(klass=nil, &block)
|
143
|
+
def each_instance_batch(klass = nil, &block)
|
141
144
|
klass ||= @type
|
142
|
-
|
145
|
+
each_batch do |batch|
|
143
146
|
models = batch.map do |row|
|
144
147
|
if ::ActiveRecord::VERSION::MAJOR < 4
|
145
|
-
|
148
|
+
klass.send(:instantiate, row)
|
146
149
|
else
|
147
150
|
@column_types ||= column_types
|
148
|
-
|
151
|
+
klass.send(:instantiate, row, @column_types)
|
149
152
|
end
|
150
153
|
end
|
151
154
|
block.call(models)
|
@@ -160,11 +163,11 @@ module PostgreSQLCursor
|
|
160
163
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
161
164
|
@options.merge!(options)
|
162
165
|
@options[:symbolize_keys] = true
|
163
|
-
|
164
|
-
cols
|
165
|
-
result
|
166
|
+
iterate_type(options[:class]) if options[:class]
|
167
|
+
cols = cols.map { |c| c.to_sym }
|
168
|
+
result = []
|
166
169
|
|
167
|
-
|
170
|
+
each do |row|
|
168
171
|
row = row.symbolize_keys if row.is_a?(Hash)
|
169
172
|
result << cols.map { |c| row[c] }
|
170
173
|
end
|
@@ -173,48 +176,44 @@ module PostgreSQLCursor
|
|
173
176
|
result
|
174
177
|
end
|
175
178
|
|
176
|
-
def each_tuple(&block)
|
177
|
-
has_do_until
|
178
|
-
has_do_while
|
179
|
-
@count
|
179
|
+
def each_tuple(&block) # :nodoc:
|
180
|
+
has_do_until = @options.has_key?(:until)
|
181
|
+
has_do_while = @options.has_key?(:while)
|
182
|
+
@count = 0
|
180
183
|
@column_types = nil
|
181
184
|
with_optional_transaction do
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
break if has_do_while && rc != @options[:while]
|
190
|
-
end
|
191
|
-
rescue Exception => e
|
192
|
-
raise e
|
193
|
-
ensure
|
194
|
-
close if @block
|
185
|
+
open
|
186
|
+
while (row = fetch)
|
187
|
+
break if row.size == 0
|
188
|
+
@count += 1
|
189
|
+
rc = block.call(row)
|
190
|
+
break if has_do_until && rc == @options[:until]
|
191
|
+
break if has_do_while && rc != @options[:while]
|
195
192
|
end
|
193
|
+
rescue => e
|
194
|
+
raise e
|
195
|
+
ensure
|
196
|
+
close if @block && connection.active?
|
196
197
|
end
|
197
198
|
@count
|
198
199
|
end
|
199
200
|
|
200
|
-
def each_batch(&block)
|
201
|
+
def each_batch(&block) # :nodoc:
|
201
202
|
has_do_until = @options.key?(:until)
|
202
203
|
has_do_while = @options.key?(:while)
|
203
204
|
@count = 0
|
204
205
|
@column_types = nil
|
205
206
|
with_optional_transaction do
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
break if has_do_while && rc != @options[:while]
|
214
|
-
end
|
215
|
-
ensure
|
216
|
-
close if @block
|
207
|
+
open
|
208
|
+
while (batch = fetch_block)
|
209
|
+
break if batch.empty?
|
210
|
+
@count += 1
|
211
|
+
rc = block.call(batch)
|
212
|
+
break if has_do_until && rc == @options[:until]
|
213
|
+
break if has_do_while && rc != @options[:while]
|
217
214
|
end
|
215
|
+
ensure
|
216
|
+
close if @block && connection.active?
|
218
217
|
end
|
219
218
|
@count
|
220
219
|
end
|
@@ -230,16 +229,14 @@ module PostgreSQLCursor
|
|
230
229
|
types = {}
|
231
230
|
fields = @result.fields
|
232
231
|
fields.each_with_index do |fname, i|
|
233
|
-
ftype = @result.ftype
|
234
|
-
fmod
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
end
|
242
|
-
end
|
232
|
+
ftype = @result.ftype(i)
|
233
|
+
fmod = @result.fmod(i)
|
234
|
+
|
235
|
+
# From @netrusov 2023-01-18. This is the same call used in the PostgreSQL Adapter
|
236
|
+
types[fname] = @connection.send(:get_oid_type, ftype, fmod, fname)
|
237
|
+
|
238
|
+
# # From @simi 2023-01-18 (Works as well, used old calling method)
|
239
|
+
# types[fname] = @connection.get_type_map.fetch(ftype)
|
243
240
|
end
|
244
241
|
|
245
242
|
@column_types = types
|
@@ -248,8 +245,8 @@ module PostgreSQLCursor
|
|
248
245
|
# Public: Opens (actually, "declares") the cursor. Call this before fetching
|
249
246
|
def open
|
250
247
|
set_cursor_tuple_fraction
|
251
|
-
@cursor = @options[:cursor_name] || ("cursor_" + SecureRandom.uuid.
|
252
|
-
hold = @options[:with_hold] ?
|
248
|
+
@cursor = @options[:cursor_name] || ("cursor_" + SecureRandom.uuid.delete("-"))
|
249
|
+
hold = @options[:with_hold] ? "with hold " : ""
|
253
250
|
@result = @connection.execute("declare #{@cursor} no scroll cursor #{hold}for #{@sql}")
|
254
251
|
@block = []
|
255
252
|
end
|
@@ -257,23 +254,23 @@ module PostgreSQLCursor
|
|
257
254
|
# Public: Returns the next row from the cursor, or empty hash if end of results
|
258
255
|
#
|
259
256
|
# Returns a row as a hash of {'colname'=>value,...}
|
260
|
-
def fetch(options={})
|
257
|
+
def fetch(options = {})
|
261
258
|
open unless @block
|
262
|
-
fetch_block if @block.size==0
|
259
|
+
fetch_block if @block.size == 0
|
263
260
|
row = @block.shift
|
264
261
|
row = row.symbolize_keys if row && options[:symbolize_keys]
|
265
262
|
row
|
266
263
|
end
|
267
264
|
|
268
265
|
# Private: Fetches the next block of rows into @block
|
269
|
-
def fetch_block(block_size=nil)
|
270
|
-
block_size ||= @block_size ||= @options.fetch(:block_size
|
266
|
+
def fetch_block(block_size = nil)
|
267
|
+
block_size ||= @block_size ||= @options.fetch(:block_size, 1000)
|
271
268
|
@result = @connection.execute("fetch #{block_size} from #{@cursor}")
|
272
269
|
|
273
|
-
if @iterate == :each_array
|
274
|
-
@
|
270
|
+
@block = if @iterate == :each_array
|
271
|
+
@result.each_row.collect { |row| row }
|
275
272
|
else
|
276
|
-
@
|
273
|
+
@result.collect { |row| row }
|
277
274
|
end
|
278
275
|
end
|
279
276
|
|
@@ -295,8 +292,8 @@ module PostgreSQLCursor
|
|
295
292
|
# This is a value between 0.1 and 1.0 (PostgreSQL defaults to 0.1, this library defaults to 1.0)
|
296
293
|
# used to determine the expected fraction (percent) of result rows returned the the caller.
|
297
294
|
# This value determines the access path by the query planner.
|
298
|
-
def set_cursor_tuple_fraction(frac=1.0)
|
299
|
-
@cursor_tuple_fraction ||= @options.fetch(:fraction
|
295
|
+
def set_cursor_tuple_fraction(frac = 1.0)
|
296
|
+
@cursor_tuple_fraction ||= @options.fetch(:fraction, 1.0)
|
300
297
|
return @cursor_tuple_fraction if frac == @cursor_tuple_fraction
|
301
298
|
@cursor_tuple_fraction = frac
|
302
299
|
@result = @connection.execute("set cursor_tuple_fraction to #{frac}")
|
data/postgresql_cursor.gemspec
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path(
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
|
-
require
|
5
|
+
require "postgresql_cursor/version"
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
|
-
spec.name
|
9
|
-
spec.version
|
10
|
-
spec.authors
|
11
|
-
spec.email
|
12
|
-
spec.summary
|
8
|
+
spec.name = "postgresql_cursor"
|
9
|
+
spec.version = PostgresqlCursor::VERSION
|
10
|
+
spec.authors = ["Allen Fair"]
|
11
|
+
spec.email = ["allen.fair@gmail.com"]
|
12
|
+
spec.summary = <<-SUMMARY
|
13
13
|
ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
14
14
|
large result set
|
15
15
|
SUMMARY
|
@@ -20,25 +20,21 @@ Gem::Specification.new do |spec|
|
|
20
20
|
rows in 'chunks' (default of 1_000 rows), buffers them, and returns the rows
|
21
21
|
one at a time.
|
22
22
|
DESCRIPTION
|
23
|
-
spec.homepage
|
24
|
-
spec.license
|
23
|
+
spec.homepage = "http://github.com/afair/postgresql_cursor"
|
24
|
+
spec.license = "MIT"
|
25
25
|
|
26
|
-
spec.files
|
27
|
-
spec.executables
|
28
|
-
spec.
|
29
|
-
spec.require_paths = ['lib']
|
26
|
+
spec.files = `git ls-files -z`.split("\x0")
|
27
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
28
|
+
spec.require_paths = ["lib"]
|
30
29
|
|
31
|
-
|
32
|
-
# spec.
|
30
|
+
spec.add_dependency "activerecord", ">= 6.0"
|
31
|
+
# spec.add_development_dependency "activerecord", "~> 6.1"
|
32
|
+
# spec.add_development_dependency "activerecord", "~> 7.0"
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
spec.add_development_dependency 'irb'
|
41
|
-
spec.add_development_dependency 'minitest'
|
42
|
-
spec.add_development_dependency 'pg'
|
43
|
-
spec.add_development_dependency 'rake'
|
34
|
+
# PG dependency held back for Jruby. See pg_jruby or jdbc-postgres-adapter gems.
|
35
|
+
spec.add_development_dependency "pg" unless RUBY_PLATFORM == "java"
|
36
|
+
spec.add_development_dependency "irb"
|
37
|
+
spec.add_development_dependency "minitest"
|
38
|
+
spec.add_development_dependency "rake"
|
39
|
+
spec.add_development_dependency "appraisal"
|
44
40
|
end
|
data/test/helper.rb
CHANGED
@@ -10,6 +10,8 @@ ActiveRecord::Base.establish_connection(adapter: 'postgresql',
|
|
10
10
|
username: ENV['TEST_USER'] || ENV['USER'] || 'postgresql_cursor')
|
11
11
|
|
12
12
|
class Product < ActiveRecord::Base
|
13
|
+
has_many :prices
|
14
|
+
|
13
15
|
# create table records (id serial primary key);
|
14
16
|
def self.generate(max=1_000)
|
15
17
|
max.times do |i|
|
@@ -18,5 +20,9 @@ class Product < ActiveRecord::Base
|
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
23
|
+
class Price < ActiveRecord::Base
|
24
|
+
belongs_to :product
|
25
|
+
end
|
26
|
+
|
21
27
|
Product.destroy_all
|
22
28
|
Product.generate(1000)
|
@@ -3,16 +3,15 @@
|
|
3
3
|
# rake setup
|
4
4
|
# or create the database manually if your environment doesn't permit
|
5
5
|
################################################################################
|
6
|
-
require_relative
|
7
|
-
require
|
8
|
-
require
|
6
|
+
require_relative "helper"
|
7
|
+
require "minitest/autorun"
|
8
|
+
require "minitest/pride"
|
9
9
|
|
10
10
|
class TestPostgresqlCursor < Minitest::Test
|
11
|
-
|
12
11
|
def test_each
|
13
12
|
c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
14
13
|
nn = 0
|
15
|
-
n = c.each { nn += 1}
|
14
|
+
n = c.each { nn += 1 }
|
16
15
|
assert_equal nn, n
|
17
16
|
end
|
18
17
|
|
@@ -29,22 +28,22 @@ class TestPostgresqlCursor < Minitest::Test
|
|
29
28
|
end
|
30
29
|
|
31
30
|
def test_each_while_until
|
32
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until:true)
|
33
|
-
n = c.each { |r| r[
|
31
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until: true)
|
32
|
+
n = c.each { |r| r["id"].to_i > 100 }
|
34
33
|
assert_equal 101, n
|
35
34
|
|
36
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while:true)
|
37
|
-
n = c.each { |r| r[
|
35
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while: true)
|
36
|
+
n = c.each { |r| r["id"].to_i < 100 }
|
38
37
|
assert_equal 100, n
|
39
38
|
end
|
40
39
|
|
41
40
|
def test_each_batch_while_until
|
42
41
|
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", until: true, block_size: 50)
|
43
|
-
n = c.each_batch { |b| b.last[
|
42
|
+
n = c.each_batch { |b| b.last["id"].to_i > 100 }
|
44
43
|
assert_equal 3, n
|
45
44
|
|
46
45
|
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", while: true, block_size: 50)
|
47
|
-
n = c.each_batch { |b| b.last[
|
46
|
+
n = c.each_batch { |b| b.last["id"].to_i < 100 }
|
48
47
|
assert_equal 2, n
|
49
48
|
end
|
50
49
|
|
@@ -68,20 +67,26 @@ class TestPostgresqlCursor < Minitest::Test
|
|
68
67
|
|
69
68
|
def test_relation
|
70
69
|
nn = 0
|
71
|
-
Product.where("id>0").each_row {|r| nn += 1 }
|
70
|
+
Product.where("id>0").each_row { |r| nn += 1 }
|
72
71
|
assert_equal 1000, nn
|
73
72
|
end
|
74
73
|
|
75
74
|
def test_relation_batch
|
76
75
|
nn = 0
|
77
76
|
row = nil
|
78
|
-
Product.where("id>0").each_row_batch(block_size: 100) { |b|
|
77
|
+
Product.where("id>0").each_row_batch(block_size: 100) { |b|
|
78
|
+
row = b.last
|
79
|
+
nn += 1
|
80
|
+
}
|
79
81
|
assert_equal 10, nn
|
80
82
|
assert_equal Hash, row.class
|
81
83
|
|
82
84
|
nn = 0
|
83
85
|
row = nil
|
84
|
-
Product.where("id>0").each_instance_batch(block_size: 100) { |b|
|
86
|
+
Product.where("id>0").each_instance_batch(block_size: 100) { |b|
|
87
|
+
row = b.last
|
88
|
+
nn += 1
|
89
|
+
}
|
85
90
|
assert_equal 10, nn
|
86
91
|
assert_equal Product, row.class
|
87
92
|
end
|
@@ -89,12 +94,18 @@ class TestPostgresqlCursor < Minitest::Test
|
|
89
94
|
def test_activerecord
|
90
95
|
nn = 0
|
91
96
|
row = nil
|
92
|
-
Product.each_row_by_sql("select * from products") {|r|
|
97
|
+
Product.each_row_by_sql("select * from products") { |r|
|
98
|
+
row = r
|
99
|
+
nn += 1
|
100
|
+
}
|
93
101
|
assert_equal 1000, nn
|
94
102
|
assert_equal Hash, row.class
|
95
103
|
|
96
104
|
nn = 0
|
97
|
-
Product.each_instance_by_sql("select * from products") {|r|
|
105
|
+
Product.each_instance_by_sql("select * from products") { |r|
|
106
|
+
row = r
|
107
|
+
nn += 1
|
108
|
+
}
|
98
109
|
assert_equal 1000, nn
|
99
110
|
assert_equal Product, row.class
|
100
111
|
end
|
@@ -102,32 +113,52 @@ class TestPostgresqlCursor < Minitest::Test
|
|
102
113
|
def test_activerecord_batch
|
103
114
|
nn = 0
|
104
115
|
row = nil
|
105
|
-
Product.each_row_batch_by_sql("select * from products", block_size: 100) { |b|
|
116
|
+
Product.each_row_batch_by_sql("select * from products", block_size: 100) { |b|
|
117
|
+
row = b.last
|
118
|
+
nn += 1
|
119
|
+
}
|
106
120
|
assert_equal 10, nn
|
107
121
|
assert_equal Hash, row.class
|
108
122
|
|
109
123
|
nn = 0
|
110
|
-
Product.each_instance_batch_by_sql("select * from products", block_size: 100) { |b|
|
124
|
+
Product.each_instance_batch_by_sql("select * from products", block_size: 100) { |b|
|
125
|
+
row = b.last
|
126
|
+
nn += 1
|
127
|
+
}
|
111
128
|
assert_equal 10, nn
|
112
129
|
assert_equal Product, row.class
|
113
130
|
end
|
114
131
|
|
115
132
|
def test_exception
|
116
|
-
|
117
|
-
|
118
|
-
raise "Oops"
|
119
|
-
end
|
120
|
-
rescue Exception => e
|
121
|
-
assert_equal e.message, 'Oops'
|
133
|
+
Product.each_row_by_sql("select * from products") do |r|
|
134
|
+
raise "Oops"
|
122
135
|
end
|
136
|
+
rescue => e
|
137
|
+
assert_equal e.message, "Oops"
|
123
138
|
end
|
124
139
|
|
125
140
|
def test_batch_exception
|
126
141
|
Product.each_row_batch_by_sql("select * from products") do |r|
|
127
|
-
raise
|
142
|
+
raise "Oops"
|
143
|
+
end
|
144
|
+
rescue => e
|
145
|
+
assert_equal e.message, "Oops"
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_exception_in_failed_transaction
|
149
|
+
Product.each_row_by_sql("select * from products") do |r|
|
150
|
+
Product.connection.execute("select kaboom")
|
151
|
+
end
|
152
|
+
rescue => e
|
153
|
+
assert_match(/PG::(InFailedSqlTransaction|UndefinedColumn)/, e.message)
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_batch_exception_in_failed_transaction
|
157
|
+
Product.each_row_batch_by_sql("select * from products") do |r|
|
158
|
+
Product.connection.execute("select kaboom")
|
128
159
|
end
|
129
160
|
rescue => e
|
130
|
-
|
161
|
+
assert_match(/PG::(InFailedSqlTransaction|UndefinedColumn)/, e.message)
|
131
162
|
end
|
132
163
|
|
133
164
|
def test_cursor
|
@@ -144,11 +175,11 @@ class TestPostgresqlCursor < Minitest::Test
|
|
144
175
|
def test_batched_cursor
|
145
176
|
cursor = Product.all.each_row_batch(block_size: 100)
|
146
177
|
assert cursor.respond_to?(:each)
|
147
|
-
b = cursor.map { |batch| batch.map { |r| r[
|
178
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
148
179
|
assert_equal 10, b.size
|
149
180
|
cursor = Product.each_row_batch_by_sql("select * from products", block_size: 100)
|
150
181
|
assert cursor.respond_to?(:each)
|
151
|
-
b = cursor.map { |batch| batch.map { |r| r[
|
182
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
152
183
|
assert_equal 10, b.size
|
153
184
|
end
|
154
185
|
|
@@ -162,9 +193,9 @@ class TestPostgresqlCursor < Minitest::Test
|
|
162
193
|
|
163
194
|
def test_with_hold
|
164
195
|
items = 0
|
165
|
-
Product.where("id < 4")
|
196
|
+
Product.where("id < 4").each_instance(with_hold: true, block_size: 1) do |row|
|
166
197
|
Product.transaction do
|
167
|
-
row.update(data:Time.now.to_f.to_s)
|
198
|
+
row.update(data: Time.now.to_f.to_s)
|
168
199
|
items += 1
|
169
200
|
end
|
170
201
|
end
|
@@ -173,22 +204,25 @@ class TestPostgresqlCursor < Minitest::Test
|
|
173
204
|
|
174
205
|
def test_fetch_symbolize_keys
|
175
206
|
Product.transaction do
|
176
|
-
#cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
207
|
+
# cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
177
208
|
cursor = Product.all.each_row
|
178
209
|
r = cursor.fetch
|
179
210
|
assert r.has_key?("id")
|
180
|
-
r = cursor.fetch(symbolize_keys:true)
|
211
|
+
r = cursor.fetch(symbolize_keys: true)
|
181
212
|
assert r.has_key?(:id)
|
182
213
|
cursor.close
|
183
214
|
end
|
184
215
|
end
|
185
216
|
|
186
217
|
def test_bad_sql
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
218
|
+
ActiveRecord::Base.each_row_by_sql("select * from bad_table") {}
|
219
|
+
raise "Did Not Raise Expected Exception"
|
220
|
+
rescue => e
|
221
|
+
assert_match(/bad_table/, e.message)
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_relation_association_is_not_loaded
|
225
|
+
cursor = Product.first.prices.each_instance
|
226
|
+
refute cursor.instance_variable_get(:@type).loaded?
|
193
227
|
end
|
194
228
|
end
|
data/test-app/Gemfile
CHANGED
@@ -1,15 +1,8 @@
|
|
1
1
|
# A sample Gemfile
|
2
2
|
source "https://rubygems.org"
|
3
3
|
|
4
|
-
gem
|
5
|
-
#gem 'activerecord', '~>
|
6
|
-
#gem 'activerecord', '~> 4.0.0'
|
7
|
-
#gem 'activerecord', '~> 4.1.0'
|
8
|
-
#gem 'activerecord', '4.1.2.rc1'
|
4
|
+
gem "activerecord", "~> 7.0.4.1"
|
5
|
+
# gem 'activerecord', '~> 6.1.7.1'
|
9
6
|
|
10
|
-
|
11
|
-
|
12
|
-
#gem 'arel', github: 'rails/arel', branch: 'master'
|
13
|
-
|
14
|
-
gem 'pg'
|
15
|
-
gem 'postgresql_cursor', path:"../"
|
7
|
+
gem "pg"
|
8
|
+
gem "postgresql_cursor", path: "../"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgresql_cursor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Allen Fair
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-01-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -16,14 +16,28 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: '6.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version:
|
26
|
+
version: '6.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: pg
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: irb
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -53,7 +67,7 @@ dependencies:
|
|
53
67
|
- !ruby/object:Gem::Version
|
54
68
|
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
70
|
+
name: rake
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - ">="
|
@@ -67,7 +81,7 @@ dependencies:
|
|
67
81
|
- !ruby/object:Gem::Version
|
68
82
|
version: '0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
|
-
name:
|
84
|
+
name: appraisal
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
72
86
|
requirements:
|
73
87
|
- - ">="
|
@@ -94,11 +108,15 @@ extra_rdoc_files: []
|
|
94
108
|
files:
|
95
109
|
- ".document"
|
96
110
|
- ".gitignore"
|
111
|
+
- ".travis.yml"
|
112
|
+
- Appraisals
|
97
113
|
- Gemfile
|
98
|
-
- Gemfile.lock
|
99
114
|
- LICENSE
|
100
115
|
- README.md
|
101
116
|
- Rakefile
|
117
|
+
- gemfiles/activerecord_4.gemfile
|
118
|
+
- gemfiles/activerecord_5.gemfile
|
119
|
+
- gemfiles/activerecord_6.gemfile
|
102
120
|
- lib/postgresql_cursor.rb
|
103
121
|
- lib/postgresql_cursor/active_record/connection_adapters/postgresql_type_map.rb
|
104
122
|
- lib/postgresql_cursor/active_record/relation/cursor_iterators.rb
|
@@ -116,7 +134,7 @@ homepage: http://github.com/afair/postgresql_cursor
|
|
116
134
|
licenses:
|
117
135
|
- MIT
|
118
136
|
metadata: {}
|
119
|
-
post_install_message:
|
137
|
+
post_install_message:
|
120
138
|
rdoc_options: []
|
121
139
|
require_paths:
|
122
140
|
- lib
|
@@ -131,11 +149,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
131
149
|
- !ruby/object:Gem::Version
|
132
150
|
version: '0'
|
133
151
|
requirements: []
|
134
|
-
rubygems_version: 3.
|
135
|
-
signing_key:
|
152
|
+
rubygems_version: 3.3.7
|
153
|
+
signing_key:
|
136
154
|
specification_version: 4
|
137
155
|
summary: ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
138
156
|
large result set
|
139
|
-
test_files:
|
140
|
-
- test/helper.rb
|
141
|
-
- test/test_postgresql_cursor.rb
|
157
|
+
test_files: []
|
data/Gemfile.lock
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
postgresql_cursor (0.6.4)
|
5
|
-
activerecord (>= 3.1.0)
|
6
|
-
|
7
|
-
GEM
|
8
|
-
remote: https://rubygems.org/
|
9
|
-
specs:
|
10
|
-
activemodel (6.0.0)
|
11
|
-
activesupport (= 6.0.0)
|
12
|
-
activerecord (6.0.0)
|
13
|
-
activemodel (= 6.0.0)
|
14
|
-
activesupport (= 6.0.0)
|
15
|
-
activesupport (6.0.0)
|
16
|
-
concurrent-ruby (~> 1.0, >= 1.0.2)
|
17
|
-
i18n (>= 0.7, < 2)
|
18
|
-
minitest (~> 5.1)
|
19
|
-
tzinfo (~> 1.1)
|
20
|
-
zeitwerk (~> 2.1, >= 2.1.8)
|
21
|
-
concurrent-ruby (1.1.5)
|
22
|
-
i18n (1.6.0)
|
23
|
-
concurrent-ruby (~> 1.0)
|
24
|
-
irb (1.0.0)
|
25
|
-
minitest (5.12.0)
|
26
|
-
pg (1.1.4)
|
27
|
-
rake (13.0.0)
|
28
|
-
thread_safe (0.3.6)
|
29
|
-
tzinfo (1.2.5)
|
30
|
-
thread_safe (~> 0.1)
|
31
|
-
zeitwerk (2.1.10)
|
32
|
-
|
33
|
-
PLATFORMS
|
34
|
-
ruby
|
35
|
-
|
36
|
-
DEPENDENCIES
|
37
|
-
irb
|
38
|
-
minitest
|
39
|
-
pg
|
40
|
-
postgresql_cursor!
|
41
|
-
rake
|
42
|
-
|
43
|
-
BUNDLED WITH
|
44
|
-
1.17.3
|