postgresql_cursor 0.6.0 → 0.6.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.travis.yml +27 -0
- data/Appraisals +12 -0
- data/README.md +45 -28
- data/Rakefile +2 -1
- data/gemfiles/activerecord_4.gemfile +8 -0
- data/gemfiles/activerecord_5.gemfile +7 -0
- data/gemfiles/activerecord_6.gemfile +7 -0
- data/lib/postgresql_cursor/active_record/relation/cursor_iterators.rb +44 -0
- data/lib/postgresql_cursor/active_record/sql_cursor.rb +76 -2
- data/lib/postgresql_cursor/cursor.rb +154 -54
- data/lib/postgresql_cursor/version.rb +3 -1
- data/lib/postgresql_cursor.rb +15 -10
- data/postgresql_cursor.gemspec +33 -21
- data/test/helper.rb +6 -0
- data/test/test_postgresql_cursor.rb +146 -31
- data/test-app/Gemfile +2 -2
- metadata +47 -16
- data/Gemfile.lock +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 8aed2634af7f5f9be7ea31ff6a9b0757eadead1df98b48326e02716453b1a2ff
|
4
|
+
data.tar.gz: a6187a116e1f4e3151d39d834c9b00f665594fd94e80182eccbceb23fbce73e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 04fd5c71657bb4702af5d73efb7ae55c0f3e0e59b625d4331c0c202766faca9f2f3effcfc56bbb26e5aafc9cace30b719c2db7232413d600a3970b39d46393da
|
7
|
+
data.tar.gz: a3dff976f877302d6f0ffa8bfee195eb509c36c18ec3e8c4c81c07ec6bc57b0434d9fd9b63528b39da43ade976c61b47888d3a3451df9c1eedf24f8561e35aa0
|
data/.gitignore
CHANGED
data/.travis.yml
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 2.6.5
|
4
|
+
- 2.7.1
|
5
|
+
before_install:
|
6
|
+
- sudo apt-get update
|
7
|
+
- sudo apt-get --yes remove postgresql\*
|
8
|
+
- sudo apt-get install -y postgresql-12 postgresql-client-12
|
9
|
+
- sudo cp /etc/postgresql/{9.6,12}/main/pg_hba.conf
|
10
|
+
- sudo service postgresql restart 12
|
11
|
+
gemfile:
|
12
|
+
- gemfiles/activerecord_4.gemfile
|
13
|
+
- gemfiles/activerecord_5.gemfile
|
14
|
+
- gemfiles/activerecord_6.gemfile
|
15
|
+
matrix:
|
16
|
+
exclude:
|
17
|
+
- rvm: 2.7.1
|
18
|
+
gemfile: gemfiles/activerecord_4.gemfile
|
19
|
+
services:
|
20
|
+
- postgresql
|
21
|
+
before_script:
|
22
|
+
- psql -c 'create database postgresql_cursor_test;' -U postgres
|
23
|
+
- psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
|
24
|
+
- psql -c 'create table products ( id serial primary key, data varchar);' -U postgres -d postgresql_cursor_test
|
25
|
+
- psql -c 'create table prices ( id serial primary key, data varchar, product_id integer);' -U postgres -d postgresql_cursor_test
|
26
|
+
addons:
|
27
|
+
postgresql: '12.3'
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#PostgreSQLCursor for handling large Result Sets
|
1
|
+
# PostgreSQLCursor for handling large Result Sets
|
2
2
|
|
3
3
|
[](http://badge.fury.io/rb/postgresql_cursor)
|
4
4
|
|
@@ -13,16 +13,12 @@ set is exhausted. By fetching a smaller chunk of data, this reduces the
|
|
13
13
|
amount of memory your application uses and prevents the potential crash
|
14
14
|
of running out of memory.
|
15
15
|
|
16
|
-
This extension is not intended to support the "FOR UPDATE / WHERE
|
17
|
-
CURRENT OF" syntax to process and update each row in place. The primary
|
18
|
-
goal is to read a large number of rows using buffering.
|
19
|
-
|
20
16
|
Supports Rails/ActiveRecord v3.1 (v3.2 recommended) higher (including
|
21
17
|
v5.0) and Ruby 1.9 and higher. Not all features work in ActiveRecord v3.1.
|
22
18
|
Support for this gem will only be for officially supported versions of
|
23
19
|
ActiveRecord and Ruby; others can try older versions of the gem.
|
24
20
|
|
25
|
-
##Using Cursors
|
21
|
+
## Using Cursors
|
26
22
|
|
27
23
|
PostgreSQLCursor was developed to take advantage of PostgreSQL's cursors. Cursors allow the program
|
28
24
|
to declare a cursor to run a given query returning "chunks" of rows to the application program while
|
@@ -71,6 +67,7 @@ All these methods take an options hash to control things more:
|
|
71
67
|
This library uses 1.0 (Optimize for 100% of the result set)
|
72
68
|
Do not override this value unless you understand it.
|
73
69
|
with_hold:boolean Keep the cursor "open" even after a commit.
|
70
|
+
cursor_name:string Give your cursor a name.
|
74
71
|
|
75
72
|
Notes:
|
76
73
|
|
@@ -79,7 +76,7 @@ Notes:
|
|
79
76
|
* Aliases each_hash and each_hash_by_sql are provided for each_row and each_row_by_sql
|
80
77
|
if you prefer to express what types are being returned.
|
81
78
|
|
82
|
-
###PostgreSQLCursor is an Enumerable
|
79
|
+
### PostgreSQLCursor is an Enumerable
|
83
80
|
|
84
81
|
If you do not pass in a block, the cursor is returned, which mixes in the Enumerable
|
85
82
|
libary. With that, you can pass it around, or chain in the awesome enumerable things
|
@@ -91,7 +88,7 @@ Product.each_row.map {|r| r["id"].to_i } #=> [1, 2, 3, ...]
|
|
91
88
|
Product.each_instance.map {|r| r.id }.each {|id| p id } #=> [1, 2, 3, ...]
|
92
89
|
Product.each_instance.lazy.inject(0) {|sum,r| sum + r.quantity } #=> 499500
|
93
90
|
```
|
94
|
-
###Hashes vs. Instances
|
91
|
+
### Hashes vs. Instances
|
95
92
|
|
96
93
|
The each_row method returns the Hash of strings for speed (as this allows you to process a lot of rows).
|
97
94
|
Hashes are returned with String values, and you must take care of any type conversion.
|
@@ -103,7 +100,7 @@ If you find you need the types cast for your attributes, consider using each_ins
|
|
103
100
|
insead. ActiveRecord's read casting algorithm will only cast the values you need and
|
104
101
|
has become more efficient over time.
|
105
102
|
|
106
|
-
###Select and Pluck
|
103
|
+
### Select and Pluck
|
107
104
|
|
108
105
|
To limit the columns returned to just those you need, use `.select(:id, :name)`
|
109
106
|
query method.
|
@@ -128,7 +125,7 @@ Product.pluck_rows(:id) #=> ["1", "2", ...]
|
|
128
125
|
Product.pluck_instances(:id, :quantity) #=> [[1, 503], [2, 932], ...]
|
129
126
|
```
|
130
127
|
|
131
|
-
###Associations and Eager Loading
|
128
|
+
### Associations and Eager Loading
|
132
129
|
|
133
130
|
ActiveRecord performs some magic when eager-loading associated row. It
|
134
131
|
will usually not join the tables, and prefers to load the data in
|
@@ -138,24 +135,50 @@ This library hooks onto the `to_sql` feature of the query builder. As a
|
|
138
135
|
result, it can't do the join if ActiveRecord decided not to join, nor
|
139
136
|
can it construct the association objects eagerly.
|
140
137
|
|
141
|
-
##
|
138
|
+
## Locking and Updating Each Row (FOR UPDATE Queries)
|
139
|
+
|
140
|
+
When you use the AREL `lock` method, a "FOR UPDATE" clause is added to
|
141
|
+
the query. This causes the block of rows returned from each FETCH
|
142
|
+
operation (see the `block_size` option) to be locked for you to update.
|
143
|
+
The lock is released on those rows once the block is exhausted and the
|
144
|
+
next FETCH or CLOSE statement is executed.
|
145
|
+
|
146
|
+
This example will run through a large table and potentially update each
|
147
|
+
row, locking only a set of rows at a time to allow concurrent use.
|
148
|
+
|
149
|
+
```ruby
|
150
|
+
Product.lock.each_instance(block_size:100) do |p|
|
151
|
+
p.update(price: p.price * 1.05)
|
152
|
+
end
|
153
|
+
```
|
154
|
+
|
155
|
+
Also, pay attention to the `block_size` you request. Locking large
|
156
|
+
blocks of rows for an extended time can cause deadlocks or other
|
157
|
+
performance issues in your application. On a busy table, or if the
|
158
|
+
processing of each row consumes a lot of time or resources, try a
|
159
|
+
`block_size` <= 10.
|
160
|
+
|
161
|
+
See the [PostgreSQL Select Documentation](https://www.postgresql.org/docs/current/static/sql-select.html)
|
162
|
+
for more information and limitations when using "FOR UPDATE" locking.
|
163
|
+
|
164
|
+
## Background: Why PostgreSQL Cursors?
|
142
165
|
|
143
166
|
ActiveRecord is designed and optimized for web performance. In a web transaction, only a "page" of
|
144
167
|
around 20 rows is returned to the user. When you do this
|
145
168
|
|
146
169
|
```ruby
|
147
|
-
Product.
|
170
|
+
Product.where("id>0").each { |product| product.process }
|
148
171
|
```
|
149
172
|
|
150
173
|
The database returns all matching result set rows to ActiveRecord, which instantiates each row with
|
151
174
|
the data returned. This function returns an array of all these rows to the caller.
|
152
175
|
|
153
|
-
|
176
|
+
Asynchronous, Background, or Offline processing may require processing a large amount of data.
|
154
177
|
When there is a very large number of rows, this requires a lot more memory to hold the data. Ruby
|
155
178
|
does not return that memory after processing the array, and the causes your process to "bloat". If you
|
156
179
|
don't have enough memory, it will cause an exception.
|
157
180
|
|
158
|
-
###ActiveRecord.find_each and find_in_batches
|
181
|
+
### ActiveRecord.find_each and find_in_batches
|
159
182
|
|
160
183
|
To solve this problem, ActiveRecord gives us two alternative methods that work in "chunks" of your data:
|
161
184
|
|
@@ -179,7 +202,7 @@ There are drawbacks with these methods:
|
|
179
202
|
### How it works
|
180
203
|
|
181
204
|
Under the covers, the library calls the PostgreSQL cursor operations
|
182
|
-
with the
|
205
|
+
with the pseudo-code:
|
183
206
|
|
184
207
|
SET cursor_tuple_fraction TO 1.0;
|
185
208
|
DECLARE cursor_1 CURSOR WITH HOLD FOR select * from widgets;
|
@@ -189,17 +212,11 @@ with the psuedo-code:
|
|
189
212
|
until rows.size < 100;
|
190
213
|
CLOSE cursor_1;
|
191
214
|
|
192
|
-
##Meta
|
193
|
-
###Author
|
194
|
-
Allen Fair, [@allenfair](https://twitter.com/allenfair),
|
195
|
-
|
196
|
-
Thanks to:
|
197
|
-
|
198
|
-
* Iulian Dogariu, http://github.com/iulianu (Fixes)
|
199
|
-
* Julian Mehnle, julian@mehnle.net (Suggestions)
|
200
|
-
* ...And all the other contributers!
|
215
|
+
## Meta
|
216
|
+
### Author
|
217
|
+
Allen Fair, [@allenfair](https://twitter.com/allenfair), [github://afair](https://github.com/afair)
|
201
218
|
|
202
|
-
###Note on Patches/Pull Requests
|
219
|
+
### Note on Patches/Pull Requests
|
203
220
|
|
204
221
|
* Fork the project.
|
205
222
|
* Make your feature addition or bug fix.
|
@@ -209,11 +226,11 @@ Thanks to:
|
|
209
226
|
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
210
227
|
* Send me a pull request. Bonus points for topic branches.
|
211
228
|
|
212
|
-
###Code of Conduct
|
229
|
+
### Code of Conduct
|
213
230
|
|
214
231
|
This project adheres to the [Open Code of Conduct](http://todogroup.org/opencodeofconduct/#postgresql_cursor/2016@allenfair.com).
|
215
232
|
By participating, you are expected to honor this code.
|
216
233
|
|
217
|
-
###Copyright
|
234
|
+
### Copyright
|
218
235
|
|
219
|
-
Copyright (c) 2010-
|
236
|
+
Copyright (c) 2010-2017 Allen Fair. See (MIT) LICENSE for details.
|
data/Rakefile
CHANGED
@@ -11,7 +11,7 @@ end
|
|
11
11
|
|
12
12
|
desc "Open and IRB Console with the gem and test-app loaded"
|
13
13
|
task :console do
|
14
|
-
sh "bundle exec irb -Ilib -I . -r postgresql_cursor -r test-app/app"
|
14
|
+
sh "bundle exec irb -Ilib -I . -r pg -r postgresql_cursor -r test-app/app"
|
15
15
|
#require 'irb'
|
16
16
|
#ARGV.clear
|
17
17
|
#IRB.start
|
@@ -21,4 +21,5 @@ desc "Setup testing database and table"
|
|
21
21
|
task :setup do
|
22
22
|
sh %q(createdb postgresql_cursor_test)
|
23
23
|
sh %Q<echo "create table products ( id serial primary key, data varchar);" | psql postgresql_cursor_test>
|
24
|
+
sh %Q<echo "create table prices ( id serial primary key, data varchar, product_id integer);" | psql postgresql_cursor_test>
|
24
25
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Defines extension to ActiveRecord/AREL to use this library
|
2
4
|
module PostgreSQLCursor
|
3
5
|
module ActiveRecord
|
@@ -12,6 +14,7 @@ module PostgreSQLCursor
|
|
12
14
|
# block_size: 1..n - The number of rows to fetch per db block fetch
|
13
15
|
# while: value - Exits loop when block does not return this value.
|
14
16
|
# until: value - Exits loop when block returns this value.
|
17
|
+
# cursor_name: string - Allows you to name your cursor.
|
15
18
|
#
|
16
19
|
# Example:
|
17
20
|
# Post.where(user_id:123).each_row { |hash| Post.process(hash) }
|
@@ -42,6 +45,47 @@ module PostgreSQLCursor
|
|
42
45
|
cursor.iterate_type(self)
|
43
46
|
end
|
44
47
|
|
48
|
+
# Public: Executes the query, yielding each batch of up to block_size
|
49
|
+
# rows where each row is a hash to the given block.
|
50
|
+
#
|
51
|
+
# Parameters: same as each_row
|
52
|
+
#
|
53
|
+
# Example:
|
54
|
+
# Post.where(user_id:123).each_row_batch do |batch|
|
55
|
+
# Post.process_batch(batch)
|
56
|
+
# end
|
57
|
+
# Post.each_row_batch.map { |batch| Post.transform_batch(batch) }
|
58
|
+
#
|
59
|
+
# Returns the number of rows yielded to the block
|
60
|
+
def each_row_batch(options={}, &block)
|
61
|
+
options = {:connection => self.connection}.merge(options)
|
62
|
+
cursor = PostgreSQLCursor::Cursor.new(to_unprepared_sql, options)
|
63
|
+
return cursor.each_row_batch(&block) if block_given?
|
64
|
+
cursor.iterate_batched
|
65
|
+
end
|
66
|
+
alias :each_hash_batch :each_row_batch
|
67
|
+
|
68
|
+
# Public: Like each_row, but yields an array of instantiated model
|
69
|
+
# objects to the block
|
70
|
+
#
|
71
|
+
# Parameters: same as each_row
|
72
|
+
#
|
73
|
+
# Example:
|
74
|
+
# Post.where(user_id:123).each_instance_batch do |batch|
|
75
|
+
# Post.process_batch(batch)
|
76
|
+
# end
|
77
|
+
# Post.where(user_id:123).each_instance_batch.map do |batch|
|
78
|
+
# Post.transform_batch(batch)
|
79
|
+
# end
|
80
|
+
#
|
81
|
+
# Returns the number of rows yielded to the block
|
82
|
+
def each_instance_batch(options={}, &block)
|
83
|
+
options = {:connection => self.connection}.merge(options)
|
84
|
+
cursor = PostgreSQLCursor::Cursor.new(to_unprepared_sql, options)
|
85
|
+
return cursor.each_instance_batch(self, &block) if block_given?
|
86
|
+
cursor.iterate_type(self).iterate_batched
|
87
|
+
end
|
88
|
+
|
45
89
|
# Plucks the column names from the rows, and return them in an array
|
46
90
|
def pluck_rows(*cols)
|
47
91
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -74,7 +74,81 @@ module PostgreSQLCursor
|
|
74
74
|
cursor.iterate_type(self)
|
75
75
|
end
|
76
76
|
|
77
|
-
#
|
77
|
+
# Public: Executes the query, yielding an array of up to block_size rows
|
78
|
+
# where each row is a hash to the given block.
|
79
|
+
#
|
80
|
+
# Parameters: same as each_row
|
81
|
+
#
|
82
|
+
# Example:
|
83
|
+
# Post.each_row_batch { |batch| Post.process_batch(batch) }
|
84
|
+
#
|
85
|
+
# Returns the number of rows yielded to the block
|
86
|
+
def each_row_batch(options={}, &block)
|
87
|
+
options = {:connection => self.connection}.merge(options)
|
88
|
+
all.each_row_batch(options, &block)
|
89
|
+
end
|
90
|
+
alias :each_hash_batch :each_row_batch
|
91
|
+
|
92
|
+
# Public: Like each_row_batch, but yields an array of instantiated model
|
93
|
+
# objects to the block
|
94
|
+
#
|
95
|
+
# Parameters: same as each_row
|
96
|
+
#
|
97
|
+
# Example:
|
98
|
+
# Post.each_instance_batch { |batch| Post.process_batch(batch) }
|
99
|
+
#
|
100
|
+
# Returns the number of rows yielded to the block
|
101
|
+
def each_instance_batch(options={}, &block)
|
102
|
+
options = {:connection => self.connection}.merge(options)
|
103
|
+
all.each_instance_batch(options, &block)
|
104
|
+
end
|
105
|
+
|
106
|
+
# Public: Yields each batch of up to block_size rows as an array of rows
|
107
|
+
# where each row as a hash to the given block
|
108
|
+
#
|
109
|
+
# Parameters: see each_row_by_sql
|
110
|
+
#
|
111
|
+
# Example:
|
112
|
+
# Post.each_row_batch_by_sql("select * from posts") do |batch|
|
113
|
+
# Post.process_batch(batch)
|
114
|
+
# end
|
115
|
+
# Post.each_row_batch_by_sql("select * from posts").map do |batch|
|
116
|
+
# Post.transform_batch(batch)
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# Returns the number of rows yielded to the block
|
120
|
+
def each_row_batch_by_sql(sql, options={}, &block)
|
121
|
+
options = {:connection => self.connection}.merge(options)
|
122
|
+
cursor = PostgreSQLCursor::Cursor.new(sql, options)
|
123
|
+
return cursor.each_row_batch(&block) if block_given?
|
124
|
+
cursor.iterate_batched
|
125
|
+
end
|
126
|
+
alias :each_hash_batch_by_sql :each_row_batch_by_sql
|
127
|
+
|
128
|
+
# Public: Yields each batch up to block_size of rows as model instances
|
129
|
+
# to the given block
|
130
|
+
#
|
131
|
+
# As this instantiates a model object, it is slower than each_row_batch_by_sql
|
132
|
+
#
|
133
|
+
# Paramaters: see each_row_by_sql
|
134
|
+
#
|
135
|
+
# Example:
|
136
|
+
# Post.each_instance_batch_by_sql("select * from posts") do |batch|
|
137
|
+
# Post.process_batch(batch)
|
138
|
+
# end
|
139
|
+
# Post.each_instance_batch_by_sql("select * from posts").map do |batch|
|
140
|
+
# Post.transform_batch(batch)
|
141
|
+
# end
|
142
|
+
#
|
143
|
+
# Returns the number of rows yielded to the block
|
144
|
+
def each_instance_batch_by_sql(sql, options={}, &block)
|
145
|
+
options = {:connection => self.connection}.merge(options)
|
146
|
+
cursor = PostgreSQLCursor::Cursor.new(sql, options)
|
147
|
+
return cursor.each_instance_batch(self, &block) if block_given?
|
148
|
+
cursor.iterate_type(self).iterate_batched
|
149
|
+
end
|
150
|
+
|
151
|
+
# Returns an array of the given column names. Use if you need cursors and don't expect
|
78
152
|
# this to comsume too much memory. Values are strings. Like ActiveRecord's pluck.
|
79
153
|
def pluck_rows(*cols)
|
80
154
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -82,7 +156,7 @@ module PostgreSQLCursor
|
|
82
156
|
end
|
83
157
|
alias :pluck_row :pluck_rows
|
84
158
|
|
85
|
-
# Returns
|
159
|
+
# Returns an array of the given column names. Use if you need cursors and don't expect
|
86
160
|
# this to comsume too much memory. Values are instance types. Like ActiveRecord's pluck.
|
87
161
|
def pluck_instances(*cols)
|
88
162
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
################################################################################
|
2
4
|
# PostgreSQLCursor: library class provides postgresql cursor for large result
|
3
5
|
# set processing. Requires ActiveRecord, but can be adapted to other DBI/ORM libraries.
|
@@ -10,6 +12,7 @@
|
|
10
12
|
# while: value - Exits loop when block does not return this value.
|
11
13
|
# until: value - Exits loop when block returns this value.
|
12
14
|
# with_hold: boolean - Allows the query to remain open across commit points.
|
15
|
+
# cursor_name: string - Allows you to name your cursor.
|
13
16
|
#
|
14
17
|
# Exmaples:
|
15
18
|
# PostgreSQLCursor::Cursor.new("select ...").each { |hash| ... }
|
@@ -17,11 +20,11 @@
|
|
17
20
|
# ActiveRecordModel.each_row_by_sql("select ...") { |hash| ... }
|
18
21
|
# ActiveRecordModel.each_instance_by_sql("select ...") { |model| ... }
|
19
22
|
#
|
23
|
+
|
20
24
|
module PostgreSQLCursor
|
21
25
|
class Cursor
|
22
26
|
include Enumerable
|
23
27
|
attr_reader :sql, :options, :connection, :count, :result
|
24
|
-
@@cursor_seq = 0
|
25
28
|
|
26
29
|
# Public: Start a new PostgreSQL cursor query
|
27
30
|
# sql - The SQL statement with interpolated values
|
@@ -38,25 +41,36 @@ module PostgreSQLCursor
|
|
38
41
|
# PostgreSQLCursor::Cursor.new("select ....")
|
39
42
|
#
|
40
43
|
# Returns the cursor object when called with new.
|
41
|
-
def initialize(sql, options={})
|
42
|
-
@sql
|
43
|
-
@options
|
44
|
+
def initialize(sql, options = {})
|
45
|
+
@sql = sql
|
46
|
+
@options = options
|
44
47
|
@connection = @options.fetch(:connection) { ::ActiveRecord::Base.connection }
|
45
|
-
@count
|
46
|
-
@iterate
|
48
|
+
@count = 0
|
49
|
+
@iterate = options[:instances] ? :each_instance : :each_row
|
50
|
+
@batched = false
|
47
51
|
end
|
48
52
|
|
49
|
-
# Specify the type to instantiate, or reset to return a Hash
|
50
|
-
|
51
|
-
|
53
|
+
# Specify the type to instantiate, or reset to return a Hash.
|
54
|
+
#
|
55
|
+
# Explicitly check for type class to prevent calling equality
|
56
|
+
# operator on active record relation, which will load it.
|
57
|
+
def iterate_type(type = nil)
|
58
|
+
if type.nil? || (type.instance_of?(Class) && type == Hash)
|
52
59
|
@iterate = :each_row
|
60
|
+
elsif type.instance_of?(Class) && type == Array
|
61
|
+
@iterate = :each_array
|
53
62
|
else
|
54
63
|
@iterate = :each_instance
|
55
|
-
@type
|
64
|
+
@type = type
|
56
65
|
end
|
57
66
|
self
|
58
67
|
end
|
59
68
|
|
69
|
+
def iterate_batched(batched = true)
|
70
|
+
@batched = batched
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
60
74
|
# Public: Yields each row of the result set to the passed block
|
61
75
|
#
|
62
76
|
# Yields the row to the block. The row is a hash with symbolized keys.
|
@@ -65,24 +79,39 @@ module PostgreSQLCursor
|
|
65
79
|
# Returns the count of rows processed
|
66
80
|
def each(&block)
|
67
81
|
if @iterate == :each_row
|
68
|
-
|
82
|
+
@batched ? each_row_batch(&block) : each_row(&block)
|
83
|
+
elsif @iterate == :each_array
|
84
|
+
@batched ? each_array_batch(&block) : each_array(&block)
|
69
85
|
else
|
70
|
-
|
86
|
+
@batched ? each_instance_batch(@type, &block) : each_instance(@type, &block)
|
71
87
|
end
|
72
88
|
end
|
73
89
|
|
74
90
|
def each_row(&block)
|
75
|
-
|
91
|
+
each_tuple do |row|
|
76
92
|
row = row.symbolize_keys if @options[:symbolize_keys]
|
77
93
|
block.call(row)
|
78
94
|
end
|
79
95
|
end
|
80
96
|
|
81
|
-
def
|
97
|
+
def each_array(&block)
|
98
|
+
old_iterate = @iterate
|
99
|
+
@iterate = :each_array
|
100
|
+
begin
|
101
|
+
rv = each_tuple do |row|
|
102
|
+
block.call(row)
|
103
|
+
end
|
104
|
+
ensure
|
105
|
+
@iterate = old_iterate
|
106
|
+
end
|
107
|
+
rv
|
108
|
+
end
|
109
|
+
|
110
|
+
def each_instance(klass = nil, &block)
|
82
111
|
klass ||= @type
|
83
|
-
|
112
|
+
each_tuple do |row|
|
84
113
|
if ::ActiveRecord::VERSION::MAJOR < 4
|
85
|
-
model = klass.send(:instantiate,row)
|
114
|
+
model = klass.send(:instantiate, row)
|
86
115
|
else
|
87
116
|
@column_types ||= column_types
|
88
117
|
model = klass.send(:instantiate, row, @column_types)
|
@@ -91,6 +120,41 @@ module PostgreSQLCursor
|
|
91
120
|
end
|
92
121
|
end
|
93
122
|
|
123
|
+
def each_row_batch(&block)
|
124
|
+
each_batch do |batch|
|
125
|
+
batch.map!(&:symbolize_keys) if @options[:symbolize_keys]
|
126
|
+
block.call(batch)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def each_array_batch(&block)
|
131
|
+
old_iterate = @iterate
|
132
|
+
@iterate = :each_array
|
133
|
+
begin
|
134
|
+
rv = each_batch do |batch|
|
135
|
+
block.call(batch)
|
136
|
+
end
|
137
|
+
ensure
|
138
|
+
@iterate = old_iterate
|
139
|
+
end
|
140
|
+
rv
|
141
|
+
end
|
142
|
+
|
143
|
+
def each_instance_batch(klass = nil, &block)
|
144
|
+
klass ||= @type
|
145
|
+
each_batch do |batch|
|
146
|
+
models = batch.map do |row|
|
147
|
+
if ::ActiveRecord::VERSION::MAJOR < 4
|
148
|
+
klass.send(:instantiate, row)
|
149
|
+
else
|
150
|
+
@column_types ||= column_types
|
151
|
+
klass.send(:instantiate, row, @column_types)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
block.call(models)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
94
158
|
# Returns an array of columns plucked from the result rows.
|
95
159
|
# Experimental function, as this could still use too much memory
|
96
160
|
# and negate the purpose of this libarary.
|
@@ -99,11 +163,11 @@ module PostgreSQLCursor
|
|
99
163
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
100
164
|
@options.merge!(options)
|
101
165
|
@options[:symbolize_keys] = true
|
102
|
-
|
103
|
-
cols
|
104
|
-
result
|
166
|
+
iterate_type(options[:class]) if options[:class]
|
167
|
+
cols = cols.map { |c| c.to_sym }
|
168
|
+
result = []
|
105
169
|
|
106
|
-
|
170
|
+
each do |row|
|
107
171
|
row = row.symbolize_keys if row.is_a?(Hash)
|
108
172
|
result << cols.map { |c| row[c] }
|
109
173
|
end
|
@@ -112,26 +176,44 @@ module PostgreSQLCursor
|
|
112
176
|
result
|
113
177
|
end
|
114
178
|
|
115
|
-
def each_tuple(&block)
|
116
|
-
has_do_until
|
117
|
-
has_do_while
|
118
|
-
@count
|
179
|
+
def each_tuple(&block) # :nodoc:
|
180
|
+
has_do_until = @options.has_key?(:until)
|
181
|
+
has_do_while = @options.has_key?(:while)
|
182
|
+
@count = 0
|
119
183
|
@column_types = nil
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
184
|
+
with_optional_transaction do
|
185
|
+
open
|
186
|
+
while (row = fetch)
|
187
|
+
break if row.size == 0
|
188
|
+
@count += 1
|
189
|
+
rc = block.call(row)
|
190
|
+
break if has_do_until && rc == @options[:until]
|
191
|
+
break if has_do_while && rc != @options[:while]
|
192
|
+
end
|
193
|
+
rescue => e
|
194
|
+
raise e
|
195
|
+
ensure
|
196
|
+
close if @block && connection.active?
|
197
|
+
end
|
198
|
+
@count
|
199
|
+
end
|
200
|
+
|
201
|
+
def each_batch(&block) # :nodoc:
|
202
|
+
has_do_until = @options.key?(:until)
|
203
|
+
has_do_while = @options.key?(:while)
|
204
|
+
@count = 0
|
205
|
+
@column_types = nil
|
206
|
+
with_optional_transaction do
|
207
|
+
open
|
208
|
+
while (batch = fetch_block)
|
209
|
+
break if batch.empty?
|
210
|
+
@count += 1
|
211
|
+
rc = block.call(batch)
|
212
|
+
break if has_do_until && rc == @options[:until]
|
213
|
+
break if has_do_while && rc != @options[:while]
|
134
214
|
end
|
215
|
+
ensure
|
216
|
+
close if @block && connection.active?
|
135
217
|
end
|
136
218
|
@count
|
137
219
|
end
|
@@ -148,11 +230,15 @@ module PostgreSQLCursor
|
|
148
230
|
fields = @result.fields
|
149
231
|
fields.each_with_index do |fname, i|
|
150
232
|
ftype = @result.ftype i
|
151
|
-
fmod
|
152
|
-
types[fname] = @connection.get_type_map.fetch(ftype, fmod)
|
233
|
+
fmod = @result.fmod i
|
234
|
+
types[fname] = @connection.get_type_map.fetch(ftype, fmod) do |oid, mod|
|
153
235
|
warn "unknown OID: #{fname}(#{oid}) (#{sql})"
|
154
|
-
|
155
|
-
|
236
|
+
if ::ActiveRecord::VERSION::MAJOR <= 4
|
237
|
+
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::OID::Identity.new
|
238
|
+
else
|
239
|
+
::ActiveRecord::Type::Value.new
|
240
|
+
end
|
241
|
+
end
|
156
242
|
end
|
157
243
|
|
158
244
|
@column_types = types
|
@@ -161,41 +247,55 @@ module PostgreSQLCursor
|
|
161
247
|
# Public: Opens (actually, "declares") the cursor. Call this before fetching
|
162
248
|
def open
|
163
249
|
set_cursor_tuple_fraction
|
164
|
-
@cursor =
|
165
|
-
hold = @options[:with_hold] ?
|
166
|
-
@result = @connection.execute("declare
|
250
|
+
@cursor = @options[:cursor_name] || ("cursor_" + SecureRandom.uuid.delete("-"))
|
251
|
+
hold = @options[:with_hold] ? "with hold " : ""
|
252
|
+
@result = @connection.execute("declare #{@cursor} no scroll cursor #{hold}for #{@sql}")
|
167
253
|
@block = []
|
168
254
|
end
|
169
255
|
|
170
256
|
# Public: Returns the next row from the cursor, or empty hash if end of results
|
171
257
|
#
|
172
258
|
# Returns a row as a hash of {'colname'=>value,...}
|
173
|
-
def fetch(options={})
|
259
|
+
def fetch(options = {})
|
174
260
|
open unless @block
|
175
|
-
fetch_block if @block.size==0
|
261
|
+
fetch_block if @block.size == 0
|
176
262
|
row = @block.shift
|
177
263
|
row = row.symbolize_keys if row && options[:symbolize_keys]
|
178
264
|
row
|
179
265
|
end
|
180
266
|
|
181
267
|
# Private: Fetches the next block of rows into @block
|
182
|
-
def fetch_block(block_size=nil)
|
183
|
-
block_size ||= @block_size ||= @options.fetch(:block_size
|
184
|
-
@result = @connection.execute("fetch #{block_size} from
|
185
|
-
|
268
|
+
def fetch_block(block_size = nil)
|
269
|
+
block_size ||= @block_size ||= @options.fetch(:block_size, 1000)
|
270
|
+
@result = @connection.execute("fetch #{block_size} from #{@cursor}")
|
271
|
+
|
272
|
+
@block = if @iterate == :each_array
|
273
|
+
@result.each_row.collect { |row| row }
|
274
|
+
else
|
275
|
+
@result.collect { |row| row }
|
276
|
+
end
|
186
277
|
end
|
187
278
|
|
188
279
|
# Public: Closes the cursor
|
189
280
|
def close
|
190
|
-
@connection.execute("close
|
281
|
+
@connection.execute("close #{@cursor}")
|
282
|
+
end
|
283
|
+
|
284
|
+
# Private: Open transaction unless with_hold option, specified
|
285
|
+
def with_optional_transaction
|
286
|
+
if @options[:with_hold]
|
287
|
+
yield
|
288
|
+
else
|
289
|
+
@connection.transaction { yield }
|
290
|
+
end
|
191
291
|
end
|
192
292
|
|
193
293
|
# Private: Sets the PostgreSQL cursor_tuple_fraction value = 1.0 to assume all rows will be fetched
|
194
294
|
# This is a value between 0.1 and 1.0 (PostgreSQL defaults to 0.1, this library defaults to 1.0)
|
195
295
|
# used to determine the expected fraction (percent) of result rows returned the the caller.
|
196
296
|
# This value determines the access path by the query planner.
|
197
|
-
def set_cursor_tuple_fraction(frac=1.0)
|
198
|
-
@cursor_tuple_fraction ||= @options.fetch(:fraction
|
297
|
+
def set_cursor_tuple_fraction(frac = 1.0)
|
298
|
+
@cursor_tuple_fraction ||= @options.fetch(:fraction, 1.0)
|
199
299
|
return @cursor_tuple_fraction if frac == @cursor_tuple_fraction
|
200
300
|
@cursor_tuple_fraction = frac
|
201
301
|
@result = @connection.execute("set cursor_tuple_fraction to #{frac}")
|
data/lib/postgresql_cursor.rb
CHANGED
@@ -1,12 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'postgresql_cursor/version'
|
2
|
-
require '
|
3
|
-
|
4
|
-
|
5
|
-
require 'postgresql_cursor/
|
4
|
+
require 'active_support'
|
5
|
+
|
6
|
+
ActiveSupport.on_load :active_record do
|
7
|
+
require 'postgresql_cursor/cursor'
|
8
|
+
require 'postgresql_cursor/active_record/relation/cursor_iterators'
|
9
|
+
require 'postgresql_cursor/active_record/sql_cursor'
|
10
|
+
require 'postgresql_cursor/active_record/connection_adapters/postgresql_type_map'
|
6
11
|
|
7
|
-
# ActiveRecord 4.x
|
8
|
-
require 'active_record'
|
9
|
-
|
10
|
-
ActiveRecord::
|
11
|
-
ActiveRecord::
|
12
|
-
|
12
|
+
# ActiveRecord 4.x
|
13
|
+
require 'active_record/connection_adapters/postgresql_adapter'
|
14
|
+
ActiveRecord::Base.extend(PostgreSQLCursor::ActiveRecord::SqlCursor)
|
15
|
+
ActiveRecord::Relation.send(:include, PostgreSQLCursor::ActiveRecord::Relation::CursorIterators)
|
16
|
+
ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.send(:include, PostgreSQLCursor::ActiveRecord::ConnectionAdapters::PostgreSQLTypeMap)
|
17
|
+
end
|
data/postgresql_cursor.gemspec
CHANGED
@@ -1,32 +1,44 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require "postgresql_cursor/version"
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.authors
|
10
|
-
spec.email
|
11
|
-
spec.summary
|
12
|
-
|
13
|
-
|
14
|
-
|
8
|
+
spec.name = "postgresql_cursor"
|
9
|
+
spec.version = PostgresqlCursor::VERSION
|
10
|
+
spec.authors = ["Allen Fair"]
|
11
|
+
spec.email = ["allen.fair@gmail.com"]
|
12
|
+
spec.summary = <<-SUMMARY
|
13
|
+
ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
14
|
+
large result set
|
15
|
+
SUMMARY
|
16
|
+
spec.description = <<-DESCRIPTION
|
17
|
+
PostgreSQL Cursor is an extension to the ActiveRecord PostgreSQLAdapter for
|
18
|
+
very large result sets. It provides a cursor open/fetch/close interface to
|
19
|
+
access data without loading all rows into memory, and instead loads the result
|
20
|
+
rows in 'chunks' (default of 1_000 rows), buffers them, and returns the rows
|
21
|
+
one at a time.
|
22
|
+
DESCRIPTION
|
23
|
+
spec.homepage = "http://github.com/afair/postgresql_cursor"
|
24
|
+
spec.license = "MIT"
|
15
25
|
|
16
|
-
spec.files
|
17
|
-
spec.executables
|
18
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
+
spec.files = `git ls-files -z`.split("\x0")
|
27
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
28
|
spec.require_paths = ["lib"]
|
20
29
|
|
21
|
-
#
|
30
|
+
# Remove this for jruby which should use 'activerecord-jdbcpostgresql-adapter'
|
31
|
+
# spec.add_dependency 'pg'
|
32
|
+
|
22
33
|
spec.add_dependency "activerecord", ">= 3.1.0"
|
23
|
-
#spec.add_dependency
|
24
|
-
#
|
25
|
-
#spec.add_dependency
|
26
|
-
#spec.add_dependency
|
27
|
-
#spec.add_dependency "activerecord", "~> 5.0.0.beta2"
|
34
|
+
# spec.add_dependency 'activerecord', '~> 3.1.0'
|
35
|
+
# spec.add_dependency 'activerecord', '~> 4.1.0'
|
36
|
+
# spec.add_dependency 'activerecord', '~> 5.0.0'
|
37
|
+
# spec.add_dependency 'activerecord', '~> 6.0.0'
|
28
38
|
|
39
|
+
spec.add_development_dependency "irb"
|
40
|
+
spec.add_development_dependency "minitest"
|
29
41
|
spec.add_development_dependency "pg"
|
30
42
|
spec.add_development_dependency "rake"
|
31
|
-
spec.add_development_dependency "
|
43
|
+
spec.add_development_dependency "appraisal"
|
32
44
|
end
|
data/test/helper.rb
CHANGED
@@ -10,6 +10,8 @@ ActiveRecord::Base.establish_connection(adapter: 'postgresql',
|
|
10
10
|
username: ENV['TEST_USER'] || ENV['USER'] || 'postgresql_cursor')
|
11
11
|
|
12
12
|
class Product < ActiveRecord::Base
|
13
|
+
has_many :prices
|
14
|
+
|
13
15
|
# create table records (id serial primary key);
|
14
16
|
def self.generate(max=1_000)
|
15
17
|
max.times do |i|
|
@@ -18,5 +20,9 @@ class Product < ActiveRecord::Base
|
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
23
|
+
class Price < ActiveRecord::Base
|
24
|
+
belongs_to :product
|
25
|
+
end
|
26
|
+
|
21
27
|
Product.destroy_all
|
22
28
|
Product.generate(1000)
|
@@ -3,16 +3,22 @@
|
|
3
3
|
# rake setup
|
4
4
|
# or create the database manually if your environment doesn't permit
|
5
5
|
################################################################################
|
6
|
-
require_relative
|
7
|
-
require
|
8
|
-
require
|
6
|
+
require_relative "helper"
|
7
|
+
require "minitest/autorun"
|
8
|
+
require "minitest/pride"
|
9
9
|
|
10
10
|
class TestPostgresqlCursor < Minitest::Test
|
11
|
-
|
12
11
|
def test_each
|
13
12
|
c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
14
13
|
nn = 0
|
15
|
-
n = c.each { nn += 1}
|
14
|
+
n = c.each { nn += 1 }
|
15
|
+
assert_equal nn, n
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_each_batch
|
19
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
20
|
+
nn = 0
|
21
|
+
n = c.each_batch { |b| nn += 1 }
|
16
22
|
assert_equal nn, n
|
17
23
|
end
|
18
24
|
|
@@ -22,42 +28,137 @@ class TestPostgresqlCursor < Minitest::Test
|
|
22
28
|
end
|
23
29
|
|
24
30
|
def test_each_while_until
|
25
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until:true)
|
26
|
-
n = c.each { |r| r[
|
27
|
-
assert_equal
|
31
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until: true)
|
32
|
+
n = c.each { |r| r["id"].to_i > 100 }
|
33
|
+
assert_equal 101, n
|
28
34
|
|
29
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while:true)
|
30
|
-
n = c.each { |r| r[
|
31
|
-
assert_equal
|
35
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while: true)
|
36
|
+
n = c.each { |r| r["id"].to_i < 100 }
|
37
|
+
assert_equal 100, n
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_each_batch_while_until
|
41
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", until: true, block_size: 50)
|
42
|
+
n = c.each_batch { |b| b.last["id"].to_i > 100 }
|
43
|
+
assert_equal 3, n
|
44
|
+
|
45
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", while: true, block_size: 50)
|
46
|
+
n = c.each_batch { |b| b.last["id"].to_i < 100 }
|
47
|
+
assert_equal 2, n
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_each_array
|
51
|
+
c = PostgreSQLCursor::Cursor.new("select * from products where id = 1")
|
52
|
+
c.each_array do |ary|
|
53
|
+
assert_equal Array, ary.class
|
54
|
+
assert_equal 1, ary[0].to_i
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_each_array_batch
|
59
|
+
c = PostgreSQLCursor::Cursor.new("select * from products where id = 1")
|
60
|
+
c.each_array_batch do |b|
|
61
|
+
assert_equal 1, b.size
|
62
|
+
ary = b.first
|
63
|
+
assert_equal Array, ary.class
|
64
|
+
assert_equal 1, ary[0].to_i
|
65
|
+
end
|
32
66
|
end
|
33
67
|
|
34
68
|
def test_relation
|
35
69
|
nn = 0
|
36
|
-
Product.where("id>0").each_row {|r| nn += 1 }
|
70
|
+
Product.where("id>0").each_row { |r| nn += 1 }
|
37
71
|
assert_equal 1000, nn
|
38
72
|
end
|
39
73
|
|
74
|
+
def test_relation_batch
|
75
|
+
nn = 0
|
76
|
+
row = nil
|
77
|
+
Product.where("id>0").each_row_batch(block_size: 100) { |b|
|
78
|
+
row = b.last
|
79
|
+
nn += 1
|
80
|
+
}
|
81
|
+
assert_equal 10, nn
|
82
|
+
assert_equal Hash, row.class
|
83
|
+
|
84
|
+
nn = 0
|
85
|
+
row = nil
|
86
|
+
Product.where("id>0").each_instance_batch(block_size: 100) { |b|
|
87
|
+
row = b.last
|
88
|
+
nn += 1
|
89
|
+
}
|
90
|
+
assert_equal 10, nn
|
91
|
+
assert_equal Product, row.class
|
92
|
+
end
|
93
|
+
|
40
94
|
def test_activerecord
|
41
95
|
nn = 0
|
42
96
|
row = nil
|
43
|
-
Product.each_row_by_sql("select * from products") {|r|
|
97
|
+
Product.each_row_by_sql("select * from products") { |r|
|
98
|
+
row = r
|
99
|
+
nn += 1
|
100
|
+
}
|
44
101
|
assert_equal 1000, nn
|
45
102
|
assert_equal Hash, row.class
|
46
103
|
|
47
104
|
nn = 0
|
48
|
-
Product.each_instance_by_sql("select * from products") {|r|
|
105
|
+
Product.each_instance_by_sql("select * from products") { |r|
|
106
|
+
row = r
|
107
|
+
nn += 1
|
108
|
+
}
|
49
109
|
assert_equal 1000, nn
|
50
110
|
assert_equal Product, row.class
|
51
111
|
end
|
52
112
|
|
113
|
+
def test_activerecord_batch
|
114
|
+
nn = 0
|
115
|
+
row = nil
|
116
|
+
Product.each_row_batch_by_sql("select * from products", block_size: 100) { |b|
|
117
|
+
row = b.last
|
118
|
+
nn += 1
|
119
|
+
}
|
120
|
+
assert_equal 10, nn
|
121
|
+
assert_equal Hash, row.class
|
122
|
+
|
123
|
+
nn = 0
|
124
|
+
Product.each_instance_batch_by_sql("select * from products", block_size: 100) { |b|
|
125
|
+
row = b.last
|
126
|
+
nn += 1
|
127
|
+
}
|
128
|
+
assert_equal 10, nn
|
129
|
+
assert_equal Product, row.class
|
130
|
+
end
|
131
|
+
|
53
132
|
def test_exception
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
133
|
+
Product.each_row_by_sql("select * from products") do |r|
|
134
|
+
raise "Oops"
|
135
|
+
end
|
136
|
+
rescue => e
|
137
|
+
assert_equal e.message, "Oops"
|
138
|
+
end
|
139
|
+
|
140
|
+
def test_batch_exception
|
141
|
+
Product.each_row_batch_by_sql("select * from products") do |r|
|
142
|
+
raise "Oops"
|
143
|
+
end
|
144
|
+
rescue => e
|
145
|
+
assert_equal e.message, "Oops"
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_exception_in_failed_transaction
|
149
|
+
Product.each_row_by_sql("select * from products") do |r|
|
150
|
+
Product.connection.execute("select kaboom")
|
151
|
+
end
|
152
|
+
rescue => e
|
153
|
+
assert_match(/PG::InFailedSqlTransaction/, e.message)
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_batch_exception_in_failed_transaction
|
157
|
+
Product.each_row_batch_by_sql("select * from products") do |r|
|
158
|
+
Product.connection.execute("select kaboom")
|
60
159
|
end
|
160
|
+
rescue => e
|
161
|
+
assert_match(/PG::InFailedSqlTransaction/, e.message)
|
61
162
|
end
|
62
163
|
|
63
164
|
def test_cursor
|
@@ -71,19 +172,30 @@ class TestPostgresqlCursor < Minitest::Test
|
|
71
172
|
assert_equal 1000, r.size
|
72
173
|
end
|
73
174
|
|
175
|
+
def test_batched_cursor
|
176
|
+
cursor = Product.all.each_row_batch(block_size: 100)
|
177
|
+
assert cursor.respond_to?(:each)
|
178
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
179
|
+
assert_equal 10, b.size
|
180
|
+
cursor = Product.each_row_batch_by_sql("select * from products", block_size: 100)
|
181
|
+
assert cursor.respond_to?(:each)
|
182
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
183
|
+
assert_equal 10, b.size
|
184
|
+
end
|
185
|
+
|
74
186
|
def test_pluck
|
75
187
|
r = Product.pluck_rows(:id)
|
76
188
|
assert_equal 1000, r.size
|
77
189
|
r = Product.all.pluck_instances(:id)
|
78
190
|
assert_equal 1000, r.size
|
79
|
-
assert_equal
|
191
|
+
assert_equal Integer, r.first.class
|
80
192
|
end
|
81
193
|
|
82
194
|
def test_with_hold
|
83
195
|
items = 0
|
84
|
-
Product.where("id < 4")
|
196
|
+
Product.where("id < 4").each_instance(with_hold: true, block_size: 1) do |row|
|
85
197
|
Product.transaction do
|
86
|
-
row.update(data:Time.now.to_f.to_s)
|
198
|
+
row.update(data: Time.now.to_f.to_s)
|
87
199
|
items += 1
|
88
200
|
end
|
89
201
|
end
|
@@ -92,22 +204,25 @@ class TestPostgresqlCursor < Minitest::Test
|
|
92
204
|
|
93
205
|
def test_fetch_symbolize_keys
|
94
206
|
Product.transaction do
|
95
|
-
#cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
207
|
+
# cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
96
208
|
cursor = Product.all.each_row
|
97
209
|
r = cursor.fetch
|
98
210
|
assert r.has_key?("id")
|
99
|
-
r = cursor.fetch(symbolize_keys:true)
|
211
|
+
r = cursor.fetch(symbolize_keys: true)
|
100
212
|
assert r.has_key?(:id)
|
101
213
|
cursor.close
|
102
214
|
end
|
103
215
|
end
|
104
216
|
|
105
217
|
def test_bad_sql
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
218
|
+
ActiveRecord::Base.each_row_by_sql("select * from bad_table") {}
|
219
|
+
raise "Did Not Raise Expected Exception"
|
220
|
+
rescue => e
|
221
|
+
assert_match(/bad_table/, e.message)
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_relation_association_is_not_loaded
|
225
|
+
cursor = Product.first.prices.each_instance
|
226
|
+
refute cursor.instance_variable_get(:@type).loaded?
|
112
227
|
end
|
113
228
|
end
|
data/test-app/Gemfile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# A sample Gemfile
|
2
2
|
source "https://rubygems.org"
|
3
3
|
|
4
|
-
gem 'activerecord', '~>
|
4
|
+
gem 'activerecord', '~> 5.2.0'
|
5
5
|
#gem 'activerecord', '~> 3.2.0'
|
6
6
|
#gem 'activerecord', '~> 4.0.0'
|
7
7
|
#gem 'activerecord', '~> 4.1.0'
|
@@ -12,4 +12,4 @@ gem 'activerecord', '~> 3.1.0'
|
|
12
12
|
#gem 'arel', github: 'rails/arel', branch: 'master'
|
13
13
|
|
14
14
|
gem 'pg'
|
15
|
-
gem 'postgresql_cursor', path:"
|
15
|
+
gem 'postgresql_cursor', path:"../"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgresql_cursor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Allen Fair
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -24,6 +24,34 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 3.1.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: irb
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
27
55
|
- !ruby/object:Gem::Dependency
|
28
56
|
name: pg
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -53,7 +81,7 @@ dependencies:
|
|
53
81
|
- !ruby/object:Gem::Version
|
54
82
|
version: '0'
|
55
83
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
84
|
+
name: appraisal
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
58
86
|
requirements:
|
59
87
|
- - ">="
|
@@ -66,10 +94,12 @@ dependencies:
|
|
66
94
|
- - ">="
|
67
95
|
- !ruby/object:Gem::Version
|
68
96
|
version: '0'
|
69
|
-
description:
|
70
|
-
|
71
|
-
|
72
|
-
|
97
|
+
description: |2
|
98
|
+
PostgreSQL Cursor is an extension to the ActiveRecord PostgreSQLAdapter for
|
99
|
+
very large result sets. It provides a cursor open/fetch/close interface to
|
100
|
+
access data without loading all rows into memory, and instead loads the result
|
101
|
+
rows in 'chunks' (default of 1_000 rows), buffers them, and returns the rows
|
102
|
+
one at a time.
|
73
103
|
email:
|
74
104
|
- allen.fair@gmail.com
|
75
105
|
executables: []
|
@@ -78,11 +108,15 @@ extra_rdoc_files: []
|
|
78
108
|
files:
|
79
109
|
- ".document"
|
80
110
|
- ".gitignore"
|
111
|
+
- ".travis.yml"
|
112
|
+
- Appraisals
|
81
113
|
- Gemfile
|
82
|
-
- Gemfile.lock
|
83
114
|
- LICENSE
|
84
115
|
- README.md
|
85
116
|
- Rakefile
|
117
|
+
- gemfiles/activerecord_4.gemfile
|
118
|
+
- gemfiles/activerecord_5.gemfile
|
119
|
+
- gemfiles/activerecord_6.gemfile
|
86
120
|
- lib/postgresql_cursor.rb
|
87
121
|
- lib/postgresql_cursor/active_record/connection_adapters/postgresql_type_map.rb
|
88
122
|
- lib/postgresql_cursor/active_record/relation/cursor_iterators.rb
|
@@ -100,7 +134,7 @@ homepage: http://github.com/afair/postgresql_cursor
|
|
100
134
|
licenses:
|
101
135
|
- MIT
|
102
136
|
metadata: {}
|
103
|
-
post_install_message:
|
137
|
+
post_install_message:
|
104
138
|
rdoc_options: []
|
105
139
|
require_paths:
|
106
140
|
- lib
|
@@ -115,12 +149,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
115
149
|
- !ruby/object:Gem::Version
|
116
150
|
version: '0'
|
117
151
|
requirements: []
|
118
|
-
|
119
|
-
|
120
|
-
signing_key:
|
152
|
+
rubygems_version: 3.3.7
|
153
|
+
signing_key:
|
121
154
|
specification_version: 4
|
122
155
|
summary: ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
123
156
|
large result set
|
124
|
-
test_files:
|
125
|
-
- test/helper.rb
|
126
|
-
- test/test_postgresql_cursor.rb
|
157
|
+
test_files: []
|
data/Gemfile.lock
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
postgresql_cursor (0.6.0)
|
5
|
-
activerecord (>= 3.1.0)
|
6
|
-
|
7
|
-
GEM
|
8
|
-
remote: https://rubygems.org/
|
9
|
-
specs:
|
10
|
-
activemodel (4.2.5.1)
|
11
|
-
activesupport (= 4.2.5.1)
|
12
|
-
builder (~> 3.1)
|
13
|
-
activerecord (4.2.5.1)
|
14
|
-
activemodel (= 4.2.5.1)
|
15
|
-
activesupport (= 4.2.5.1)
|
16
|
-
arel (~> 6.0)
|
17
|
-
activesupport (4.2.5.1)
|
18
|
-
i18n (~> 0.7)
|
19
|
-
json (~> 1.7, >= 1.7.7)
|
20
|
-
minitest (~> 5.1)
|
21
|
-
thread_safe (~> 0.3, >= 0.3.4)
|
22
|
-
tzinfo (~> 1.1)
|
23
|
-
arel (6.0.3)
|
24
|
-
builder (3.2.2)
|
25
|
-
i18n (0.7.0)
|
26
|
-
json (1.8.3)
|
27
|
-
minitest (5.8.4)
|
28
|
-
pg (0.18.4)
|
29
|
-
rake (10.5.0)
|
30
|
-
thread_safe (0.3.5)
|
31
|
-
tzinfo (1.2.2)
|
32
|
-
thread_safe (~> 0.1)
|
33
|
-
|
34
|
-
PLATFORMS
|
35
|
-
ruby
|
36
|
-
|
37
|
-
DEPENDENCIES
|
38
|
-
minitest
|
39
|
-
pg
|
40
|
-
postgresql_cursor!
|
41
|
-
rake
|
42
|
-
|
43
|
-
BUNDLED WITH
|
44
|
-
1.11.2
|