postgresql_cursor 0.6.0 → 0.6.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +2 -1
- data/.travis.yml +27 -0
- data/Appraisals +12 -0
- data/README.md +45 -28
- data/Rakefile +2 -1
- data/gemfiles/activerecord_4.gemfile +8 -0
- data/gemfiles/activerecord_5.gemfile +7 -0
- data/gemfiles/activerecord_6.gemfile +7 -0
- data/lib/postgresql_cursor/active_record/relation/cursor_iterators.rb +44 -0
- data/lib/postgresql_cursor/active_record/sql_cursor.rb +76 -2
- data/lib/postgresql_cursor/cursor.rb +154 -54
- data/lib/postgresql_cursor/version.rb +3 -1
- data/lib/postgresql_cursor.rb +15 -10
- data/postgresql_cursor.gemspec +33 -21
- data/test/helper.rb +6 -0
- data/test/test_postgresql_cursor.rb +146 -31
- data/test-app/Gemfile +2 -2
- metadata +47 -16
- data/Gemfile.lock +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 8aed2634af7f5f9be7ea31ff6a9b0757eadead1df98b48326e02716453b1a2ff
|
4
|
+
data.tar.gz: a6187a116e1f4e3151d39d834c9b00f665594fd94e80182eccbceb23fbce73e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 04fd5c71657bb4702af5d73efb7ae55c0f3e0e59b625d4331c0c202766faca9f2f3effcfc56bbb26e5aafc9cace30b719c2db7232413d600a3970b39d46393da
|
7
|
+
data.tar.gz: a3dff976f877302d6f0ffa8bfee195eb509c36c18ec3e8c4c81c07ec6bc57b0434d9fd9b63528b39da43ade976c61b47888d3a3451df9c1eedf24f8561e35aa0
|
data/.gitignore
CHANGED
data/.travis.yml
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
language: ruby
|
2
|
+
rvm:
|
3
|
+
- 2.6.5
|
4
|
+
- 2.7.1
|
5
|
+
before_install:
|
6
|
+
- sudo apt-get update
|
7
|
+
- sudo apt-get --yes remove postgresql\*
|
8
|
+
- sudo apt-get install -y postgresql-12 postgresql-client-12
|
9
|
+
- sudo cp /etc/postgresql/{9.6,12}/main/pg_hba.conf
|
10
|
+
- sudo service postgresql restart 12
|
11
|
+
gemfile:
|
12
|
+
- gemfiles/activerecord_4.gemfile
|
13
|
+
- gemfiles/activerecord_5.gemfile
|
14
|
+
- gemfiles/activerecord_6.gemfile
|
15
|
+
matrix:
|
16
|
+
exclude:
|
17
|
+
- rvm: 2.7.1
|
18
|
+
gemfile: gemfiles/activerecord_4.gemfile
|
19
|
+
services:
|
20
|
+
- postgresql
|
21
|
+
before_script:
|
22
|
+
- psql -c 'create database postgresql_cursor_test;' -U postgres
|
23
|
+
- psql -c 'CREATE ROLE travis SUPERUSER LOGIN CREATEDB;' -U postgres
|
24
|
+
- psql -c 'create table products ( id serial primary key, data varchar);' -U postgres -d postgresql_cursor_test
|
25
|
+
- psql -c 'create table prices ( id serial primary key, data varchar, product_id integer);' -U postgres -d postgresql_cursor_test
|
26
|
+
addons:
|
27
|
+
postgresql: '12.3'
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#PostgreSQLCursor for handling large Result Sets
|
1
|
+
# PostgreSQLCursor for handling large Result Sets
|
2
2
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/postgresql_cursor.svg)](http://badge.fury.io/rb/postgresql_cursor)
|
4
4
|
|
@@ -13,16 +13,12 @@ set is exhausted. By fetching a smaller chunk of data, this reduces the
|
|
13
13
|
amount of memory your application uses and prevents the potential crash
|
14
14
|
of running out of memory.
|
15
15
|
|
16
|
-
This extension is not intended to support the "FOR UPDATE / WHERE
|
17
|
-
CURRENT OF" syntax to process and update each row in place. The primary
|
18
|
-
goal is to read a large number of rows using buffering.
|
19
|
-
|
20
16
|
Supports Rails/ActiveRecord v3.1 (v3.2 recommended) higher (including
|
21
17
|
v5.0) and Ruby 1.9 and higher. Not all features work in ActiveRecord v3.1.
|
22
18
|
Support for this gem will only be for officially supported versions of
|
23
19
|
ActiveRecord and Ruby; others can try older versions of the gem.
|
24
20
|
|
25
|
-
##Using Cursors
|
21
|
+
## Using Cursors
|
26
22
|
|
27
23
|
PostgreSQLCursor was developed to take advantage of PostgreSQL's cursors. Cursors allow the program
|
28
24
|
to declare a cursor to run a given query returning "chunks" of rows to the application program while
|
@@ -71,6 +67,7 @@ All these methods take an options hash to control things more:
|
|
71
67
|
This library uses 1.0 (Optimize for 100% of the result set)
|
72
68
|
Do not override this value unless you understand it.
|
73
69
|
with_hold:boolean Keep the cursor "open" even after a commit.
|
70
|
+
cursor_name:string Give your cursor a name.
|
74
71
|
|
75
72
|
Notes:
|
76
73
|
|
@@ -79,7 +76,7 @@ Notes:
|
|
79
76
|
* Aliases each_hash and each_hash_by_sql are provided for each_row and each_row_by_sql
|
80
77
|
if you prefer to express what types are being returned.
|
81
78
|
|
82
|
-
###PostgreSQLCursor is an Enumerable
|
79
|
+
### PostgreSQLCursor is an Enumerable
|
83
80
|
|
84
81
|
If you do not pass in a block, the cursor is returned, which mixes in the Enumerable
|
85
82
|
libary. With that, you can pass it around, or chain in the awesome enumerable things
|
@@ -91,7 +88,7 @@ Product.each_row.map {|r| r["id"].to_i } #=> [1, 2, 3, ...]
|
|
91
88
|
Product.each_instance.map {|r| r.id }.each {|id| p id } #=> [1, 2, 3, ...]
|
92
89
|
Product.each_instance.lazy.inject(0) {|sum,r| sum + r.quantity } #=> 499500
|
93
90
|
```
|
94
|
-
###Hashes vs. Instances
|
91
|
+
### Hashes vs. Instances
|
95
92
|
|
96
93
|
The each_row method returns the Hash of strings for speed (as this allows you to process a lot of rows).
|
97
94
|
Hashes are returned with String values, and you must take care of any type conversion.
|
@@ -103,7 +100,7 @@ If you find you need the types cast for your attributes, consider using each_ins
|
|
103
100
|
insead. ActiveRecord's read casting algorithm will only cast the values you need and
|
104
101
|
has become more efficient over time.
|
105
102
|
|
106
|
-
###Select and Pluck
|
103
|
+
### Select and Pluck
|
107
104
|
|
108
105
|
To limit the columns returned to just those you need, use `.select(:id, :name)`
|
109
106
|
query method.
|
@@ -128,7 +125,7 @@ Product.pluck_rows(:id) #=> ["1", "2", ...]
|
|
128
125
|
Product.pluck_instances(:id, :quantity) #=> [[1, 503], [2, 932], ...]
|
129
126
|
```
|
130
127
|
|
131
|
-
###Associations and Eager Loading
|
128
|
+
### Associations and Eager Loading
|
132
129
|
|
133
130
|
ActiveRecord performs some magic when eager-loading associated row. It
|
134
131
|
will usually not join the tables, and prefers to load the data in
|
@@ -138,24 +135,50 @@ This library hooks onto the `to_sql` feature of the query builder. As a
|
|
138
135
|
result, it can't do the join if ActiveRecord decided not to join, nor
|
139
136
|
can it construct the association objects eagerly.
|
140
137
|
|
141
|
-
##
|
138
|
+
## Locking and Updating Each Row (FOR UPDATE Queries)
|
139
|
+
|
140
|
+
When you use the AREL `lock` method, a "FOR UPDATE" clause is added to
|
141
|
+
the query. This causes the block of rows returned from each FETCH
|
142
|
+
operation (see the `block_size` option) to be locked for you to update.
|
143
|
+
The lock is released on those rows once the block is exhausted and the
|
144
|
+
next FETCH or CLOSE statement is executed.
|
145
|
+
|
146
|
+
This example will run through a large table and potentially update each
|
147
|
+
row, locking only a set of rows at a time to allow concurrent use.
|
148
|
+
|
149
|
+
```ruby
|
150
|
+
Product.lock.each_instance(block_size:100) do |p|
|
151
|
+
p.update(price: p.price * 1.05)
|
152
|
+
end
|
153
|
+
```
|
154
|
+
|
155
|
+
Also, pay attention to the `block_size` you request. Locking large
|
156
|
+
blocks of rows for an extended time can cause deadlocks or other
|
157
|
+
performance issues in your application. On a busy table, or if the
|
158
|
+
processing of each row consumes a lot of time or resources, try a
|
159
|
+
`block_size` <= 10.
|
160
|
+
|
161
|
+
See the [PostgreSQL Select Documentation](https://www.postgresql.org/docs/current/static/sql-select.html)
|
162
|
+
for more information and limitations when using "FOR UPDATE" locking.
|
163
|
+
|
164
|
+
## Background: Why PostgreSQL Cursors?
|
142
165
|
|
143
166
|
ActiveRecord is designed and optimized for web performance. In a web transaction, only a "page" of
|
144
167
|
around 20 rows is returned to the user. When you do this
|
145
168
|
|
146
169
|
```ruby
|
147
|
-
Product.
|
170
|
+
Product.where("id>0").each { |product| product.process }
|
148
171
|
```
|
149
172
|
|
150
173
|
The database returns all matching result set rows to ActiveRecord, which instantiates each row with
|
151
174
|
the data returned. This function returns an array of all these rows to the caller.
|
152
175
|
|
153
|
-
|
176
|
+
Asynchronous, Background, or Offline processing may require processing a large amount of data.
|
154
177
|
When there is a very large number of rows, this requires a lot more memory to hold the data. Ruby
|
155
178
|
does not return that memory after processing the array, and the causes your process to "bloat". If you
|
156
179
|
don't have enough memory, it will cause an exception.
|
157
180
|
|
158
|
-
###ActiveRecord.find_each and find_in_batches
|
181
|
+
### ActiveRecord.find_each and find_in_batches
|
159
182
|
|
160
183
|
To solve this problem, ActiveRecord gives us two alternative methods that work in "chunks" of your data:
|
161
184
|
|
@@ -179,7 +202,7 @@ There are drawbacks with these methods:
|
|
179
202
|
### How it works
|
180
203
|
|
181
204
|
Under the covers, the library calls the PostgreSQL cursor operations
|
182
|
-
with the
|
205
|
+
with the pseudo-code:
|
183
206
|
|
184
207
|
SET cursor_tuple_fraction TO 1.0;
|
185
208
|
DECLARE cursor_1 CURSOR WITH HOLD FOR select * from widgets;
|
@@ -189,17 +212,11 @@ with the psuedo-code:
|
|
189
212
|
until rows.size < 100;
|
190
213
|
CLOSE cursor_1;
|
191
214
|
|
192
|
-
##Meta
|
193
|
-
###Author
|
194
|
-
Allen Fair, [@allenfair](https://twitter.com/allenfair),
|
195
|
-
|
196
|
-
Thanks to:
|
197
|
-
|
198
|
-
* Iulian Dogariu, http://github.com/iulianu (Fixes)
|
199
|
-
* Julian Mehnle, julian@mehnle.net (Suggestions)
|
200
|
-
* ...And all the other contributers!
|
215
|
+
## Meta
|
216
|
+
### Author
|
217
|
+
Allen Fair, [@allenfair](https://twitter.com/allenfair), [github://afair](https://github.com/afair)
|
201
218
|
|
202
|
-
###Note on Patches/Pull Requests
|
219
|
+
### Note on Patches/Pull Requests
|
203
220
|
|
204
221
|
* Fork the project.
|
205
222
|
* Make your feature addition or bug fix.
|
@@ -209,11 +226,11 @@ Thanks to:
|
|
209
226
|
(if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
210
227
|
* Send me a pull request. Bonus points for topic branches.
|
211
228
|
|
212
|
-
###Code of Conduct
|
229
|
+
### Code of Conduct
|
213
230
|
|
214
231
|
This project adheres to the [Open Code of Conduct](http://todogroup.org/opencodeofconduct/#postgresql_cursor/2016@allenfair.com).
|
215
232
|
By participating, you are expected to honor this code.
|
216
233
|
|
217
|
-
###Copyright
|
234
|
+
### Copyright
|
218
235
|
|
219
|
-
Copyright (c) 2010-
|
236
|
+
Copyright (c) 2010-2017 Allen Fair. See (MIT) LICENSE for details.
|
data/Rakefile
CHANGED
@@ -11,7 +11,7 @@ end
|
|
11
11
|
|
12
12
|
desc "Open and IRB Console with the gem and test-app loaded"
|
13
13
|
task :console do
|
14
|
-
sh "bundle exec irb -Ilib -I . -r postgresql_cursor -r test-app/app"
|
14
|
+
sh "bundle exec irb -Ilib -I . -r pg -r postgresql_cursor -r test-app/app"
|
15
15
|
#require 'irb'
|
16
16
|
#ARGV.clear
|
17
17
|
#IRB.start
|
@@ -21,4 +21,5 @@ desc "Setup testing database and table"
|
|
21
21
|
task :setup do
|
22
22
|
sh %q(createdb postgresql_cursor_test)
|
23
23
|
sh %Q<echo "create table products ( id serial primary key, data varchar);" | psql postgresql_cursor_test>
|
24
|
+
sh %Q<echo "create table prices ( id serial primary key, data varchar, product_id integer);" | psql postgresql_cursor_test>
|
24
25
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Defines extension to ActiveRecord/AREL to use this library
|
2
4
|
module PostgreSQLCursor
|
3
5
|
module ActiveRecord
|
@@ -12,6 +14,7 @@ module PostgreSQLCursor
|
|
12
14
|
# block_size: 1..n - The number of rows to fetch per db block fetch
|
13
15
|
# while: value - Exits loop when block does not return this value.
|
14
16
|
# until: value - Exits loop when block returns this value.
|
17
|
+
# cursor_name: string - Allows you to name your cursor.
|
15
18
|
#
|
16
19
|
# Example:
|
17
20
|
# Post.where(user_id:123).each_row { |hash| Post.process(hash) }
|
@@ -42,6 +45,47 @@ module PostgreSQLCursor
|
|
42
45
|
cursor.iterate_type(self)
|
43
46
|
end
|
44
47
|
|
48
|
+
# Public: Executes the query, yielding each batch of up to block_size
|
49
|
+
# rows where each row is a hash to the given block.
|
50
|
+
#
|
51
|
+
# Parameters: same as each_row
|
52
|
+
#
|
53
|
+
# Example:
|
54
|
+
# Post.where(user_id:123).each_row_batch do |batch|
|
55
|
+
# Post.process_batch(batch)
|
56
|
+
# end
|
57
|
+
# Post.each_row_batch.map { |batch| Post.transform_batch(batch) }
|
58
|
+
#
|
59
|
+
# Returns the number of rows yielded to the block
|
60
|
+
def each_row_batch(options={}, &block)
|
61
|
+
options = {:connection => self.connection}.merge(options)
|
62
|
+
cursor = PostgreSQLCursor::Cursor.new(to_unprepared_sql, options)
|
63
|
+
return cursor.each_row_batch(&block) if block_given?
|
64
|
+
cursor.iterate_batched
|
65
|
+
end
|
66
|
+
alias :each_hash_batch :each_row_batch
|
67
|
+
|
68
|
+
# Public: Like each_row, but yields an array of instantiated model
|
69
|
+
# objects to the block
|
70
|
+
#
|
71
|
+
# Parameters: same as each_row
|
72
|
+
#
|
73
|
+
# Example:
|
74
|
+
# Post.where(user_id:123).each_instance_batch do |batch|
|
75
|
+
# Post.process_batch(batch)
|
76
|
+
# end
|
77
|
+
# Post.where(user_id:123).each_instance_batch.map do |batch|
|
78
|
+
# Post.transform_batch(batch)
|
79
|
+
# end
|
80
|
+
#
|
81
|
+
# Returns the number of rows yielded to the block
|
82
|
+
def each_instance_batch(options={}, &block)
|
83
|
+
options = {:connection => self.connection}.merge(options)
|
84
|
+
cursor = PostgreSQLCursor::Cursor.new(to_unprepared_sql, options)
|
85
|
+
return cursor.each_instance_batch(self, &block) if block_given?
|
86
|
+
cursor.iterate_type(self).iterate_batched
|
87
|
+
end
|
88
|
+
|
45
89
|
# Plucks the column names from the rows, and return them in an array
|
46
90
|
def pluck_rows(*cols)
|
47
91
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -74,7 +74,81 @@ module PostgreSQLCursor
|
|
74
74
|
cursor.iterate_type(self)
|
75
75
|
end
|
76
76
|
|
77
|
-
#
|
77
|
+
# Public: Executes the query, yielding an array of up to block_size rows
|
78
|
+
# where each row is a hash to the given block.
|
79
|
+
#
|
80
|
+
# Parameters: same as each_row
|
81
|
+
#
|
82
|
+
# Example:
|
83
|
+
# Post.each_row_batch { |batch| Post.process_batch(batch) }
|
84
|
+
#
|
85
|
+
# Returns the number of rows yielded to the block
|
86
|
+
def each_row_batch(options={}, &block)
|
87
|
+
options = {:connection => self.connection}.merge(options)
|
88
|
+
all.each_row_batch(options, &block)
|
89
|
+
end
|
90
|
+
alias :each_hash_batch :each_row_batch
|
91
|
+
|
92
|
+
# Public: Like each_row_batch, but yields an array of instantiated model
|
93
|
+
# objects to the block
|
94
|
+
#
|
95
|
+
# Parameters: same as each_row
|
96
|
+
#
|
97
|
+
# Example:
|
98
|
+
# Post.each_instance_batch { |batch| Post.process_batch(batch) }
|
99
|
+
#
|
100
|
+
# Returns the number of rows yielded to the block
|
101
|
+
def each_instance_batch(options={}, &block)
|
102
|
+
options = {:connection => self.connection}.merge(options)
|
103
|
+
all.each_instance_batch(options, &block)
|
104
|
+
end
|
105
|
+
|
106
|
+
# Public: Yields each batch of up to block_size rows as an array of rows
|
107
|
+
# where each row as a hash to the given block
|
108
|
+
#
|
109
|
+
# Parameters: see each_row_by_sql
|
110
|
+
#
|
111
|
+
# Example:
|
112
|
+
# Post.each_row_batch_by_sql("select * from posts") do |batch|
|
113
|
+
# Post.process_batch(batch)
|
114
|
+
# end
|
115
|
+
# Post.each_row_batch_by_sql("select * from posts").map do |batch|
|
116
|
+
# Post.transform_batch(batch)
|
117
|
+
# end
|
118
|
+
#
|
119
|
+
# Returns the number of rows yielded to the block
|
120
|
+
def each_row_batch_by_sql(sql, options={}, &block)
|
121
|
+
options = {:connection => self.connection}.merge(options)
|
122
|
+
cursor = PostgreSQLCursor::Cursor.new(sql, options)
|
123
|
+
return cursor.each_row_batch(&block) if block_given?
|
124
|
+
cursor.iterate_batched
|
125
|
+
end
|
126
|
+
alias :each_hash_batch_by_sql :each_row_batch_by_sql
|
127
|
+
|
128
|
+
# Public: Yields each batch up to block_size of rows as model instances
|
129
|
+
# to the given block
|
130
|
+
#
|
131
|
+
# As this instantiates a model object, it is slower than each_row_batch_by_sql
|
132
|
+
#
|
133
|
+
# Paramaters: see each_row_by_sql
|
134
|
+
#
|
135
|
+
# Example:
|
136
|
+
# Post.each_instance_batch_by_sql("select * from posts") do |batch|
|
137
|
+
# Post.process_batch(batch)
|
138
|
+
# end
|
139
|
+
# Post.each_instance_batch_by_sql("select * from posts").map do |batch|
|
140
|
+
# Post.transform_batch(batch)
|
141
|
+
# end
|
142
|
+
#
|
143
|
+
# Returns the number of rows yielded to the block
|
144
|
+
def each_instance_batch_by_sql(sql, options={}, &block)
|
145
|
+
options = {:connection => self.connection}.merge(options)
|
146
|
+
cursor = PostgreSQLCursor::Cursor.new(sql, options)
|
147
|
+
return cursor.each_instance_batch(self, &block) if block_given?
|
148
|
+
cursor.iterate_type(self).iterate_batched
|
149
|
+
end
|
150
|
+
|
151
|
+
# Returns an array of the given column names. Use if you need cursors and don't expect
|
78
152
|
# this to comsume too much memory. Values are strings. Like ActiveRecord's pluck.
|
79
153
|
def pluck_rows(*cols)
|
80
154
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -82,7 +156,7 @@ module PostgreSQLCursor
|
|
82
156
|
end
|
83
157
|
alias :pluck_row :pluck_rows
|
84
158
|
|
85
|
-
# Returns
|
159
|
+
# Returns an array of the given column names. Use if you need cursors and don't expect
|
86
160
|
# this to comsume too much memory. Values are instance types. Like ActiveRecord's pluck.
|
87
161
|
def pluck_instances(*cols)
|
88
162
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
################################################################################
|
2
4
|
# PostgreSQLCursor: library class provides postgresql cursor for large result
|
3
5
|
# set processing. Requires ActiveRecord, but can be adapted to other DBI/ORM libraries.
|
@@ -10,6 +12,7 @@
|
|
10
12
|
# while: value - Exits loop when block does not return this value.
|
11
13
|
# until: value - Exits loop when block returns this value.
|
12
14
|
# with_hold: boolean - Allows the query to remain open across commit points.
|
15
|
+
# cursor_name: string - Allows you to name your cursor.
|
13
16
|
#
|
14
17
|
# Exmaples:
|
15
18
|
# PostgreSQLCursor::Cursor.new("select ...").each { |hash| ... }
|
@@ -17,11 +20,11 @@
|
|
17
20
|
# ActiveRecordModel.each_row_by_sql("select ...") { |hash| ... }
|
18
21
|
# ActiveRecordModel.each_instance_by_sql("select ...") { |model| ... }
|
19
22
|
#
|
23
|
+
|
20
24
|
module PostgreSQLCursor
|
21
25
|
class Cursor
|
22
26
|
include Enumerable
|
23
27
|
attr_reader :sql, :options, :connection, :count, :result
|
24
|
-
@@cursor_seq = 0
|
25
28
|
|
26
29
|
# Public: Start a new PostgreSQL cursor query
|
27
30
|
# sql - The SQL statement with interpolated values
|
@@ -38,25 +41,36 @@ module PostgreSQLCursor
|
|
38
41
|
# PostgreSQLCursor::Cursor.new("select ....")
|
39
42
|
#
|
40
43
|
# Returns the cursor object when called with new.
|
41
|
-
def initialize(sql, options={})
|
42
|
-
@sql
|
43
|
-
@options
|
44
|
+
def initialize(sql, options = {})
|
45
|
+
@sql = sql
|
46
|
+
@options = options
|
44
47
|
@connection = @options.fetch(:connection) { ::ActiveRecord::Base.connection }
|
45
|
-
@count
|
46
|
-
@iterate
|
48
|
+
@count = 0
|
49
|
+
@iterate = options[:instances] ? :each_instance : :each_row
|
50
|
+
@batched = false
|
47
51
|
end
|
48
52
|
|
49
|
-
# Specify the type to instantiate, or reset to return a Hash
|
50
|
-
|
51
|
-
|
53
|
+
# Specify the type to instantiate, or reset to return a Hash.
|
54
|
+
#
|
55
|
+
# Explicitly check for type class to prevent calling equality
|
56
|
+
# operator on active record relation, which will load it.
|
57
|
+
def iterate_type(type = nil)
|
58
|
+
if type.nil? || (type.instance_of?(Class) && type == Hash)
|
52
59
|
@iterate = :each_row
|
60
|
+
elsif type.instance_of?(Class) && type == Array
|
61
|
+
@iterate = :each_array
|
53
62
|
else
|
54
63
|
@iterate = :each_instance
|
55
|
-
@type
|
64
|
+
@type = type
|
56
65
|
end
|
57
66
|
self
|
58
67
|
end
|
59
68
|
|
69
|
+
def iterate_batched(batched = true)
|
70
|
+
@batched = batched
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
60
74
|
# Public: Yields each row of the result set to the passed block
|
61
75
|
#
|
62
76
|
# Yields the row to the block. The row is a hash with symbolized keys.
|
@@ -65,24 +79,39 @@ module PostgreSQLCursor
|
|
65
79
|
# Returns the count of rows processed
|
66
80
|
def each(&block)
|
67
81
|
if @iterate == :each_row
|
68
|
-
|
82
|
+
@batched ? each_row_batch(&block) : each_row(&block)
|
83
|
+
elsif @iterate == :each_array
|
84
|
+
@batched ? each_array_batch(&block) : each_array(&block)
|
69
85
|
else
|
70
|
-
|
86
|
+
@batched ? each_instance_batch(@type, &block) : each_instance(@type, &block)
|
71
87
|
end
|
72
88
|
end
|
73
89
|
|
74
90
|
def each_row(&block)
|
75
|
-
|
91
|
+
each_tuple do |row|
|
76
92
|
row = row.symbolize_keys if @options[:symbolize_keys]
|
77
93
|
block.call(row)
|
78
94
|
end
|
79
95
|
end
|
80
96
|
|
81
|
-
def
|
97
|
+
def each_array(&block)
|
98
|
+
old_iterate = @iterate
|
99
|
+
@iterate = :each_array
|
100
|
+
begin
|
101
|
+
rv = each_tuple do |row|
|
102
|
+
block.call(row)
|
103
|
+
end
|
104
|
+
ensure
|
105
|
+
@iterate = old_iterate
|
106
|
+
end
|
107
|
+
rv
|
108
|
+
end
|
109
|
+
|
110
|
+
def each_instance(klass = nil, &block)
|
82
111
|
klass ||= @type
|
83
|
-
|
112
|
+
each_tuple do |row|
|
84
113
|
if ::ActiveRecord::VERSION::MAJOR < 4
|
85
|
-
model = klass.send(:instantiate,row)
|
114
|
+
model = klass.send(:instantiate, row)
|
86
115
|
else
|
87
116
|
@column_types ||= column_types
|
88
117
|
model = klass.send(:instantiate, row, @column_types)
|
@@ -91,6 +120,41 @@ module PostgreSQLCursor
|
|
91
120
|
end
|
92
121
|
end
|
93
122
|
|
123
|
+
def each_row_batch(&block)
|
124
|
+
each_batch do |batch|
|
125
|
+
batch.map!(&:symbolize_keys) if @options[:symbolize_keys]
|
126
|
+
block.call(batch)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def each_array_batch(&block)
|
131
|
+
old_iterate = @iterate
|
132
|
+
@iterate = :each_array
|
133
|
+
begin
|
134
|
+
rv = each_batch do |batch|
|
135
|
+
block.call(batch)
|
136
|
+
end
|
137
|
+
ensure
|
138
|
+
@iterate = old_iterate
|
139
|
+
end
|
140
|
+
rv
|
141
|
+
end
|
142
|
+
|
143
|
+
def each_instance_batch(klass = nil, &block)
|
144
|
+
klass ||= @type
|
145
|
+
each_batch do |batch|
|
146
|
+
models = batch.map do |row|
|
147
|
+
if ::ActiveRecord::VERSION::MAJOR < 4
|
148
|
+
klass.send(:instantiate, row)
|
149
|
+
else
|
150
|
+
@column_types ||= column_types
|
151
|
+
klass.send(:instantiate, row, @column_types)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
block.call(models)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
94
158
|
# Returns an array of columns plucked from the result rows.
|
95
159
|
# Experimental function, as this could still use too much memory
|
96
160
|
# and negate the purpose of this libarary.
|
@@ -99,11 +163,11 @@ module PostgreSQLCursor
|
|
99
163
|
options = cols.last.is_a?(Hash) ? cols.pop : {}
|
100
164
|
@options.merge!(options)
|
101
165
|
@options[:symbolize_keys] = true
|
102
|
-
|
103
|
-
cols
|
104
|
-
result
|
166
|
+
iterate_type(options[:class]) if options[:class]
|
167
|
+
cols = cols.map { |c| c.to_sym }
|
168
|
+
result = []
|
105
169
|
|
106
|
-
|
170
|
+
each do |row|
|
107
171
|
row = row.symbolize_keys if row.is_a?(Hash)
|
108
172
|
result << cols.map { |c| row[c] }
|
109
173
|
end
|
@@ -112,26 +176,44 @@ module PostgreSQLCursor
|
|
112
176
|
result
|
113
177
|
end
|
114
178
|
|
115
|
-
def each_tuple(&block)
|
116
|
-
has_do_until
|
117
|
-
has_do_while
|
118
|
-
@count
|
179
|
+
def each_tuple(&block) # :nodoc:
|
180
|
+
has_do_until = @options.has_key?(:until)
|
181
|
+
has_do_while = @options.has_key?(:while)
|
182
|
+
@count = 0
|
119
183
|
@column_types = nil
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
184
|
+
with_optional_transaction do
|
185
|
+
open
|
186
|
+
while (row = fetch)
|
187
|
+
break if row.size == 0
|
188
|
+
@count += 1
|
189
|
+
rc = block.call(row)
|
190
|
+
break if has_do_until && rc == @options[:until]
|
191
|
+
break if has_do_while && rc != @options[:while]
|
192
|
+
end
|
193
|
+
rescue => e
|
194
|
+
raise e
|
195
|
+
ensure
|
196
|
+
close if @block && connection.active?
|
197
|
+
end
|
198
|
+
@count
|
199
|
+
end
|
200
|
+
|
201
|
+
def each_batch(&block) # :nodoc:
|
202
|
+
has_do_until = @options.key?(:until)
|
203
|
+
has_do_while = @options.key?(:while)
|
204
|
+
@count = 0
|
205
|
+
@column_types = nil
|
206
|
+
with_optional_transaction do
|
207
|
+
open
|
208
|
+
while (batch = fetch_block)
|
209
|
+
break if batch.empty?
|
210
|
+
@count += 1
|
211
|
+
rc = block.call(batch)
|
212
|
+
break if has_do_until && rc == @options[:until]
|
213
|
+
break if has_do_while && rc != @options[:while]
|
134
214
|
end
|
215
|
+
ensure
|
216
|
+
close if @block && connection.active?
|
135
217
|
end
|
136
218
|
@count
|
137
219
|
end
|
@@ -148,11 +230,15 @@ module PostgreSQLCursor
|
|
148
230
|
fields = @result.fields
|
149
231
|
fields.each_with_index do |fname, i|
|
150
232
|
ftype = @result.ftype i
|
151
|
-
fmod
|
152
|
-
types[fname] = @connection.get_type_map.fetch(ftype, fmod)
|
233
|
+
fmod = @result.fmod i
|
234
|
+
types[fname] = @connection.get_type_map.fetch(ftype, fmod) do |oid, mod|
|
153
235
|
warn "unknown OID: #{fname}(#{oid}) (#{sql})"
|
154
|
-
|
155
|
-
|
236
|
+
if ::ActiveRecord::VERSION::MAJOR <= 4
|
237
|
+
::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter::OID::Identity.new
|
238
|
+
else
|
239
|
+
::ActiveRecord::Type::Value.new
|
240
|
+
end
|
241
|
+
end
|
156
242
|
end
|
157
243
|
|
158
244
|
@column_types = types
|
@@ -161,41 +247,55 @@ module PostgreSQLCursor
|
|
161
247
|
# Public: Opens (actually, "declares") the cursor. Call this before fetching
|
162
248
|
def open
|
163
249
|
set_cursor_tuple_fraction
|
164
|
-
@cursor =
|
165
|
-
hold = @options[:with_hold] ?
|
166
|
-
@result = @connection.execute("declare
|
250
|
+
@cursor = @options[:cursor_name] || ("cursor_" + SecureRandom.uuid.delete("-"))
|
251
|
+
hold = @options[:with_hold] ? "with hold " : ""
|
252
|
+
@result = @connection.execute("declare #{@cursor} no scroll cursor #{hold}for #{@sql}")
|
167
253
|
@block = []
|
168
254
|
end
|
169
255
|
|
170
256
|
# Public: Returns the next row from the cursor, or empty hash if end of results
|
171
257
|
#
|
172
258
|
# Returns a row as a hash of {'colname'=>value,...}
|
173
|
-
def fetch(options={})
|
259
|
+
def fetch(options = {})
|
174
260
|
open unless @block
|
175
|
-
fetch_block if @block.size==0
|
261
|
+
fetch_block if @block.size == 0
|
176
262
|
row = @block.shift
|
177
263
|
row = row.symbolize_keys if row && options[:symbolize_keys]
|
178
264
|
row
|
179
265
|
end
|
180
266
|
|
181
267
|
# Private: Fetches the next block of rows into @block
|
182
|
-
def fetch_block(block_size=nil)
|
183
|
-
block_size ||= @block_size ||= @options.fetch(:block_size
|
184
|
-
@result = @connection.execute("fetch #{block_size} from
|
185
|
-
|
268
|
+
def fetch_block(block_size = nil)
|
269
|
+
block_size ||= @block_size ||= @options.fetch(:block_size, 1000)
|
270
|
+
@result = @connection.execute("fetch #{block_size} from #{@cursor}")
|
271
|
+
|
272
|
+
@block = if @iterate == :each_array
|
273
|
+
@result.each_row.collect { |row| row }
|
274
|
+
else
|
275
|
+
@result.collect { |row| row }
|
276
|
+
end
|
186
277
|
end
|
187
278
|
|
188
279
|
# Public: Closes the cursor
|
189
280
|
def close
|
190
|
-
@connection.execute("close
|
281
|
+
@connection.execute("close #{@cursor}")
|
282
|
+
end
|
283
|
+
|
284
|
+
# Private: Open transaction unless with_hold option, specified
|
285
|
+
def with_optional_transaction
|
286
|
+
if @options[:with_hold]
|
287
|
+
yield
|
288
|
+
else
|
289
|
+
@connection.transaction { yield }
|
290
|
+
end
|
191
291
|
end
|
192
292
|
|
193
293
|
# Private: Sets the PostgreSQL cursor_tuple_fraction value = 1.0 to assume all rows will be fetched
|
194
294
|
# This is a value between 0.1 and 1.0 (PostgreSQL defaults to 0.1, this library defaults to 1.0)
|
195
295
|
# used to determine the expected fraction (percent) of result rows returned the the caller.
|
196
296
|
# This value determines the access path by the query planner.
|
197
|
-
def set_cursor_tuple_fraction(frac=1.0)
|
198
|
-
@cursor_tuple_fraction ||= @options.fetch(:fraction
|
297
|
+
def set_cursor_tuple_fraction(frac = 1.0)
|
298
|
+
@cursor_tuple_fraction ||= @options.fetch(:fraction, 1.0)
|
199
299
|
return @cursor_tuple_fraction if frac == @cursor_tuple_fraction
|
200
300
|
@cursor_tuple_fraction = frac
|
201
301
|
@result = @connection.execute("set cursor_tuple_fraction to #{frac}")
|
data/lib/postgresql_cursor.rb
CHANGED
@@ -1,12 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'postgresql_cursor/version'
|
2
|
-
require '
|
3
|
-
|
4
|
-
|
5
|
-
require 'postgresql_cursor/
|
4
|
+
require 'active_support'
|
5
|
+
|
6
|
+
ActiveSupport.on_load :active_record do
|
7
|
+
require 'postgresql_cursor/cursor'
|
8
|
+
require 'postgresql_cursor/active_record/relation/cursor_iterators'
|
9
|
+
require 'postgresql_cursor/active_record/sql_cursor'
|
10
|
+
require 'postgresql_cursor/active_record/connection_adapters/postgresql_type_map'
|
6
11
|
|
7
|
-
# ActiveRecord 4.x
|
8
|
-
require 'active_record'
|
9
|
-
|
10
|
-
ActiveRecord::
|
11
|
-
ActiveRecord::
|
12
|
-
|
12
|
+
# ActiveRecord 4.x
|
13
|
+
require 'active_record/connection_adapters/postgresql_adapter'
|
14
|
+
ActiveRecord::Base.extend(PostgreSQLCursor::ActiveRecord::SqlCursor)
|
15
|
+
ActiveRecord::Relation.send(:include, PostgreSQLCursor::ActiveRecord::Relation::CursorIterators)
|
16
|
+
ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.send(:include, PostgreSQLCursor::ActiveRecord::ConnectionAdapters::PostgreSQLTypeMap)
|
17
|
+
end
|
data/postgresql_cursor.gemspec
CHANGED
@@ -1,32 +1,44 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require "postgresql_cursor/version"
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
|
-
spec.name
|
8
|
-
spec.version
|
9
|
-
spec.authors
|
10
|
-
spec.email
|
11
|
-
spec.summary
|
12
|
-
|
13
|
-
|
14
|
-
|
8
|
+
spec.name = "postgresql_cursor"
|
9
|
+
spec.version = PostgresqlCursor::VERSION
|
10
|
+
spec.authors = ["Allen Fair"]
|
11
|
+
spec.email = ["allen.fair@gmail.com"]
|
12
|
+
spec.summary = <<-SUMMARY
|
13
|
+
ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
14
|
+
large result set
|
15
|
+
SUMMARY
|
16
|
+
spec.description = <<-DESCRIPTION
|
17
|
+
PostgreSQL Cursor is an extension to the ActiveRecord PostgreSQLAdapter for
|
18
|
+
very large result sets. It provides a cursor open/fetch/close interface to
|
19
|
+
access data without loading all rows into memory, and instead loads the result
|
20
|
+
rows in 'chunks' (default of 1_000 rows), buffers them, and returns the rows
|
21
|
+
one at a time.
|
22
|
+
DESCRIPTION
|
23
|
+
spec.homepage = "http://github.com/afair/postgresql_cursor"
|
24
|
+
spec.license = "MIT"
|
15
25
|
|
16
|
-
spec.files
|
17
|
-
spec.executables
|
18
|
-
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
26
|
+
spec.files = `git ls-files -z`.split("\x0")
|
27
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
28
|
spec.require_paths = ["lib"]
|
20
29
|
|
21
|
-
#
|
30
|
+
# Remove this for jruby which should use 'activerecord-jdbcpostgresql-adapter'
|
31
|
+
# spec.add_dependency 'pg'
|
32
|
+
|
22
33
|
spec.add_dependency "activerecord", ">= 3.1.0"
|
23
|
-
#spec.add_dependency
|
24
|
-
#
|
25
|
-
#spec.add_dependency
|
26
|
-
#spec.add_dependency
|
27
|
-
#spec.add_dependency "activerecord", "~> 5.0.0.beta2"
|
34
|
+
# spec.add_dependency 'activerecord', '~> 3.1.0'
|
35
|
+
# spec.add_dependency 'activerecord', '~> 4.1.0'
|
36
|
+
# spec.add_dependency 'activerecord', '~> 5.0.0'
|
37
|
+
# spec.add_dependency 'activerecord', '~> 6.0.0'
|
28
38
|
|
39
|
+
spec.add_development_dependency "irb"
|
40
|
+
spec.add_development_dependency "minitest"
|
29
41
|
spec.add_development_dependency "pg"
|
30
42
|
spec.add_development_dependency "rake"
|
31
|
-
spec.add_development_dependency "
|
43
|
+
spec.add_development_dependency "appraisal"
|
32
44
|
end
|
data/test/helper.rb
CHANGED
@@ -10,6 +10,8 @@ ActiveRecord::Base.establish_connection(adapter: 'postgresql',
|
|
10
10
|
username: ENV['TEST_USER'] || ENV['USER'] || 'postgresql_cursor')
|
11
11
|
|
12
12
|
class Product < ActiveRecord::Base
|
13
|
+
has_many :prices
|
14
|
+
|
13
15
|
# create table records (id serial primary key);
|
14
16
|
def self.generate(max=1_000)
|
15
17
|
max.times do |i|
|
@@ -18,5 +20,9 @@ class Product < ActiveRecord::Base
|
|
18
20
|
end
|
19
21
|
end
|
20
22
|
|
23
|
+
class Price < ActiveRecord::Base
|
24
|
+
belongs_to :product
|
25
|
+
end
|
26
|
+
|
21
27
|
Product.destroy_all
|
22
28
|
Product.generate(1000)
|
@@ -3,16 +3,22 @@
|
|
3
3
|
# rake setup
|
4
4
|
# or create the database manually if your environment doesn't permit
|
5
5
|
################################################################################
|
6
|
-
require_relative
|
7
|
-
require
|
8
|
-
require
|
6
|
+
require_relative "helper"
|
7
|
+
require "minitest/autorun"
|
8
|
+
require "minitest/pride"
|
9
9
|
|
10
10
|
class TestPostgresqlCursor < Minitest::Test
|
11
|
-
|
12
11
|
def test_each
|
13
12
|
c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
14
13
|
nn = 0
|
15
|
-
n = c.each { nn += 1}
|
14
|
+
n = c.each { nn += 1 }
|
15
|
+
assert_equal nn, n
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_each_batch
|
19
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
20
|
+
nn = 0
|
21
|
+
n = c.each_batch { |b| nn += 1 }
|
16
22
|
assert_equal nn, n
|
17
23
|
end
|
18
24
|
|
@@ -22,42 +28,137 @@ class TestPostgresqlCursor < Minitest::Test
|
|
22
28
|
end
|
23
29
|
|
24
30
|
def test_each_while_until
|
25
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until:true)
|
26
|
-
n = c.each { |r| r[
|
27
|
-
assert_equal
|
31
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until: true)
|
32
|
+
n = c.each { |r| r["id"].to_i > 100 }
|
33
|
+
assert_equal 101, n
|
28
34
|
|
29
|
-
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while:true)
|
30
|
-
n = c.each { |r| r[
|
31
|
-
assert_equal
|
35
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while: true)
|
36
|
+
n = c.each { |r| r["id"].to_i < 100 }
|
37
|
+
assert_equal 100, n
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_each_batch_while_until
|
41
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", until: true, block_size: 50)
|
42
|
+
n = c.each_batch { |b| b.last["id"].to_i > 100 }
|
43
|
+
assert_equal 3, n
|
44
|
+
|
45
|
+
c = PostgreSQLCursor::Cursor.new("select * from products order by id asc", while: true, block_size: 50)
|
46
|
+
n = c.each_batch { |b| b.last["id"].to_i < 100 }
|
47
|
+
assert_equal 2, n
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_each_array
|
51
|
+
c = PostgreSQLCursor::Cursor.new("select * from products where id = 1")
|
52
|
+
c.each_array do |ary|
|
53
|
+
assert_equal Array, ary.class
|
54
|
+
assert_equal 1, ary[0].to_i
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def test_each_array_batch
|
59
|
+
c = PostgreSQLCursor::Cursor.new("select * from products where id = 1")
|
60
|
+
c.each_array_batch do |b|
|
61
|
+
assert_equal 1, b.size
|
62
|
+
ary = b.first
|
63
|
+
assert_equal Array, ary.class
|
64
|
+
assert_equal 1, ary[0].to_i
|
65
|
+
end
|
32
66
|
end
|
33
67
|
|
34
68
|
def test_relation
|
35
69
|
nn = 0
|
36
|
-
Product.where("id>0").each_row {|r| nn += 1 }
|
70
|
+
Product.where("id>0").each_row { |r| nn += 1 }
|
37
71
|
assert_equal 1000, nn
|
38
72
|
end
|
39
73
|
|
74
|
+
def test_relation_batch
|
75
|
+
nn = 0
|
76
|
+
row = nil
|
77
|
+
Product.where("id>0").each_row_batch(block_size: 100) { |b|
|
78
|
+
row = b.last
|
79
|
+
nn += 1
|
80
|
+
}
|
81
|
+
assert_equal 10, nn
|
82
|
+
assert_equal Hash, row.class
|
83
|
+
|
84
|
+
nn = 0
|
85
|
+
row = nil
|
86
|
+
Product.where("id>0").each_instance_batch(block_size: 100) { |b|
|
87
|
+
row = b.last
|
88
|
+
nn += 1
|
89
|
+
}
|
90
|
+
assert_equal 10, nn
|
91
|
+
assert_equal Product, row.class
|
92
|
+
end
|
93
|
+
|
40
94
|
def test_activerecord
|
41
95
|
nn = 0
|
42
96
|
row = nil
|
43
|
-
Product.each_row_by_sql("select * from products") {|r|
|
97
|
+
Product.each_row_by_sql("select * from products") { |r|
|
98
|
+
row = r
|
99
|
+
nn += 1
|
100
|
+
}
|
44
101
|
assert_equal 1000, nn
|
45
102
|
assert_equal Hash, row.class
|
46
103
|
|
47
104
|
nn = 0
|
48
|
-
Product.each_instance_by_sql("select * from products") {|r|
|
105
|
+
Product.each_instance_by_sql("select * from products") { |r|
|
106
|
+
row = r
|
107
|
+
nn += 1
|
108
|
+
}
|
49
109
|
assert_equal 1000, nn
|
50
110
|
assert_equal Product, row.class
|
51
111
|
end
|
52
112
|
|
113
|
+
def test_activerecord_batch
|
114
|
+
nn = 0
|
115
|
+
row = nil
|
116
|
+
Product.each_row_batch_by_sql("select * from products", block_size: 100) { |b|
|
117
|
+
row = b.last
|
118
|
+
nn += 1
|
119
|
+
}
|
120
|
+
assert_equal 10, nn
|
121
|
+
assert_equal Hash, row.class
|
122
|
+
|
123
|
+
nn = 0
|
124
|
+
Product.each_instance_batch_by_sql("select * from products", block_size: 100) { |b|
|
125
|
+
row = b.last
|
126
|
+
nn += 1
|
127
|
+
}
|
128
|
+
assert_equal 10, nn
|
129
|
+
assert_equal Product, row.class
|
130
|
+
end
|
131
|
+
|
53
132
|
def test_exception
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
133
|
+
Product.each_row_by_sql("select * from products") do |r|
|
134
|
+
raise "Oops"
|
135
|
+
end
|
136
|
+
rescue => e
|
137
|
+
assert_equal e.message, "Oops"
|
138
|
+
end
|
139
|
+
|
140
|
+
def test_batch_exception
|
141
|
+
Product.each_row_batch_by_sql("select * from products") do |r|
|
142
|
+
raise "Oops"
|
143
|
+
end
|
144
|
+
rescue => e
|
145
|
+
assert_equal e.message, "Oops"
|
146
|
+
end
|
147
|
+
|
148
|
+
def test_exception_in_failed_transaction
|
149
|
+
Product.each_row_by_sql("select * from products") do |r|
|
150
|
+
Product.connection.execute("select kaboom")
|
151
|
+
end
|
152
|
+
rescue => e
|
153
|
+
assert_match(/PG::InFailedSqlTransaction/, e.message)
|
154
|
+
end
|
155
|
+
|
156
|
+
def test_batch_exception_in_failed_transaction
|
157
|
+
Product.each_row_batch_by_sql("select * from products") do |r|
|
158
|
+
Product.connection.execute("select kaboom")
|
60
159
|
end
|
160
|
+
rescue => e
|
161
|
+
assert_match(/PG::InFailedSqlTransaction/, e.message)
|
61
162
|
end
|
62
163
|
|
63
164
|
def test_cursor
|
@@ -71,19 +172,30 @@ class TestPostgresqlCursor < Minitest::Test
|
|
71
172
|
assert_equal 1000, r.size
|
72
173
|
end
|
73
174
|
|
175
|
+
def test_batched_cursor
|
176
|
+
cursor = Product.all.each_row_batch(block_size: 100)
|
177
|
+
assert cursor.respond_to?(:each)
|
178
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
179
|
+
assert_equal 10, b.size
|
180
|
+
cursor = Product.each_row_batch_by_sql("select * from products", block_size: 100)
|
181
|
+
assert cursor.respond_to?(:each)
|
182
|
+
b = cursor.map { |batch| batch.map { |r| r["id"] } }
|
183
|
+
assert_equal 10, b.size
|
184
|
+
end
|
185
|
+
|
74
186
|
def test_pluck
|
75
187
|
r = Product.pluck_rows(:id)
|
76
188
|
assert_equal 1000, r.size
|
77
189
|
r = Product.all.pluck_instances(:id)
|
78
190
|
assert_equal 1000, r.size
|
79
|
-
assert_equal
|
191
|
+
assert_equal Integer, r.first.class
|
80
192
|
end
|
81
193
|
|
82
194
|
def test_with_hold
|
83
195
|
items = 0
|
84
|
-
Product.where("id < 4")
|
196
|
+
Product.where("id < 4").each_instance(with_hold: true, block_size: 1) do |row|
|
85
197
|
Product.transaction do
|
86
|
-
row.update(data:Time.now.to_f.to_s)
|
198
|
+
row.update(data: Time.now.to_f.to_s)
|
87
199
|
items += 1
|
88
200
|
end
|
89
201
|
end
|
@@ -92,22 +204,25 @@ class TestPostgresqlCursor < Minitest::Test
|
|
92
204
|
|
93
205
|
def test_fetch_symbolize_keys
|
94
206
|
Product.transaction do
|
95
|
-
#cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
207
|
+
# cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
|
96
208
|
cursor = Product.all.each_row
|
97
209
|
r = cursor.fetch
|
98
210
|
assert r.has_key?("id")
|
99
|
-
r = cursor.fetch(symbolize_keys:true)
|
211
|
+
r = cursor.fetch(symbolize_keys: true)
|
100
212
|
assert r.has_key?(:id)
|
101
213
|
cursor.close
|
102
214
|
end
|
103
215
|
end
|
104
216
|
|
105
217
|
def test_bad_sql
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
218
|
+
ActiveRecord::Base.each_row_by_sql("select * from bad_table") {}
|
219
|
+
raise "Did Not Raise Expected Exception"
|
220
|
+
rescue => e
|
221
|
+
assert_match(/bad_table/, e.message)
|
222
|
+
end
|
223
|
+
|
224
|
+
def test_relation_association_is_not_loaded
|
225
|
+
cursor = Product.first.prices.each_instance
|
226
|
+
refute cursor.instance_variable_get(:@type).loaded?
|
112
227
|
end
|
113
228
|
end
|
data/test-app/Gemfile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# A sample Gemfile
|
2
2
|
source "https://rubygems.org"
|
3
3
|
|
4
|
-
gem 'activerecord', '~>
|
4
|
+
gem 'activerecord', '~> 5.2.0'
|
5
5
|
#gem 'activerecord', '~> 3.2.0'
|
6
6
|
#gem 'activerecord', '~> 4.0.0'
|
7
7
|
#gem 'activerecord', '~> 4.1.0'
|
@@ -12,4 +12,4 @@ gem 'activerecord', '~> 3.1.0'
|
|
12
12
|
#gem 'arel', github: 'rails/arel', branch: 'master'
|
13
13
|
|
14
14
|
gem 'pg'
|
15
|
-
gem 'postgresql_cursor', path:"
|
15
|
+
gem 'postgresql_cursor', path:"../"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: postgresql_cursor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Allen Fair
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -24,6 +24,34 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 3.1.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: irb
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: minitest
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
27
55
|
- !ruby/object:Gem::Dependency
|
28
56
|
name: pg
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -53,7 +81,7 @@ dependencies:
|
|
53
81
|
- !ruby/object:Gem::Version
|
54
82
|
version: '0'
|
55
83
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
84
|
+
name: appraisal
|
57
85
|
requirement: !ruby/object:Gem::Requirement
|
58
86
|
requirements:
|
59
87
|
- - ">="
|
@@ -66,10 +94,12 @@ dependencies:
|
|
66
94
|
- - ">="
|
67
95
|
- !ruby/object:Gem::Version
|
68
96
|
version: '0'
|
69
|
-
description:
|
70
|
-
|
71
|
-
|
72
|
-
|
97
|
+
description: |2
|
98
|
+
PostgreSQL Cursor is an extension to the ActiveRecord PostgreSQLAdapter for
|
99
|
+
very large result sets. It provides a cursor open/fetch/close interface to
|
100
|
+
access data without loading all rows into memory, and instead loads the result
|
101
|
+
rows in 'chunks' (default of 1_000 rows), buffers them, and returns the rows
|
102
|
+
one at a time.
|
73
103
|
email:
|
74
104
|
- allen.fair@gmail.com
|
75
105
|
executables: []
|
@@ -78,11 +108,15 @@ extra_rdoc_files: []
|
|
78
108
|
files:
|
79
109
|
- ".document"
|
80
110
|
- ".gitignore"
|
111
|
+
- ".travis.yml"
|
112
|
+
- Appraisals
|
81
113
|
- Gemfile
|
82
|
-
- Gemfile.lock
|
83
114
|
- LICENSE
|
84
115
|
- README.md
|
85
116
|
- Rakefile
|
117
|
+
- gemfiles/activerecord_4.gemfile
|
118
|
+
- gemfiles/activerecord_5.gemfile
|
119
|
+
- gemfiles/activerecord_6.gemfile
|
86
120
|
- lib/postgresql_cursor.rb
|
87
121
|
- lib/postgresql_cursor/active_record/connection_adapters/postgresql_type_map.rb
|
88
122
|
- lib/postgresql_cursor/active_record/relation/cursor_iterators.rb
|
@@ -100,7 +134,7 @@ homepage: http://github.com/afair/postgresql_cursor
|
|
100
134
|
licenses:
|
101
135
|
- MIT
|
102
136
|
metadata: {}
|
103
|
-
post_install_message:
|
137
|
+
post_install_message:
|
104
138
|
rdoc_options: []
|
105
139
|
require_paths:
|
106
140
|
- lib
|
@@ -115,12 +149,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
115
149
|
- !ruby/object:Gem::Version
|
116
150
|
version: '0'
|
117
151
|
requirements: []
|
118
|
-
|
119
|
-
|
120
|
-
signing_key:
|
152
|
+
rubygems_version: 3.3.7
|
153
|
+
signing_key:
|
121
154
|
specification_version: 4
|
122
155
|
summary: ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
|
123
156
|
large result set
|
124
|
-
test_files:
|
125
|
-
- test/helper.rb
|
126
|
-
- test/test_postgresql_cursor.rb
|
157
|
+
test_files: []
|
data/Gemfile.lock
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
PATH
|
2
|
-
remote: .
|
3
|
-
specs:
|
4
|
-
postgresql_cursor (0.6.0)
|
5
|
-
activerecord (>= 3.1.0)
|
6
|
-
|
7
|
-
GEM
|
8
|
-
remote: https://rubygems.org/
|
9
|
-
specs:
|
10
|
-
activemodel (4.2.5.1)
|
11
|
-
activesupport (= 4.2.5.1)
|
12
|
-
builder (~> 3.1)
|
13
|
-
activerecord (4.2.5.1)
|
14
|
-
activemodel (= 4.2.5.1)
|
15
|
-
activesupport (= 4.2.5.1)
|
16
|
-
arel (~> 6.0)
|
17
|
-
activesupport (4.2.5.1)
|
18
|
-
i18n (~> 0.7)
|
19
|
-
json (~> 1.7, >= 1.7.7)
|
20
|
-
minitest (~> 5.1)
|
21
|
-
thread_safe (~> 0.3, >= 0.3.4)
|
22
|
-
tzinfo (~> 1.1)
|
23
|
-
arel (6.0.3)
|
24
|
-
builder (3.2.2)
|
25
|
-
i18n (0.7.0)
|
26
|
-
json (1.8.3)
|
27
|
-
minitest (5.8.4)
|
28
|
-
pg (0.18.4)
|
29
|
-
rake (10.5.0)
|
30
|
-
thread_safe (0.3.5)
|
31
|
-
tzinfo (1.2.2)
|
32
|
-
thread_safe (~> 0.1)
|
33
|
-
|
34
|
-
PLATFORMS
|
35
|
-
ruby
|
36
|
-
|
37
|
-
DEPENDENCIES
|
38
|
-
minitest
|
39
|
-
pg
|
40
|
-
postgresql_cursor!
|
41
|
-
rake
|
42
|
-
|
43
|
-
BUNDLED WITH
|
44
|
-
1.11.2
|