postgresql_cursor 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01a81f4aa5a5a18d0b26ad2bc2ae9d3b7d02dcb5
-  data.tar.gz: f8b3ca7158110a45ba7f28797a4d0c970f11bdb5
+  metadata.gz: 8bd16d1710bfc6796d8410b99d01b17c085118df
+  data.tar.gz: 4ad5f707f0c96ddbf85fb57ed2935d7c6e2e1da6
 SHA512:
-  metadata.gz: 204072400a16ff6934d82d03b95b3bca16ee047bd2676e00fd799de199e6fb5e6e89dc0935cb3b58fc36279e0957ebe63394c0cfac6e5d7051a434aa0d78045d
-  data.tar.gz: 1454bea0121abc656677f69150581ee04884717eed3b53bae7097cc555ec23312f5b3469cf74628f21fe16b9df1ab2b07999c678ea7cfa615ee68a8aa35dd749
+  metadata.gz: 69547256fbc79ab776b41a05729ead0eb05bc0c6cc94fb2da60a19922376e4ddbd3fa686f76f3ed0bd2e98e35e16a072828fdae0f29f687fc4e14ff0364bd8be
+  data.tar.gz: 90e5cda34d77577766143c53b26300696791298ac2c9f20ba09d204e9be443ccafa3c0bbbf70619679e829abaf2734ef271f59991352739dfcab54c63e6050f1

data/.gitignore

@@ -22,3 +22,4 @@ rdoc
 pkg
 
 ## PROJECT::SPECIFIC
+Gemfile.lock

data/Gemfile.lock

@@ -1,33 +1,33 @@
 PATH
   remote: .
   specs:
-    postgresql_cursor (0.5.1)
-      activerecord (>= 3.2.0)
+    postgresql_cursor (0.6.0)
+      activerecord (>= 3.1.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activemodel (4.1.5)
-      activesupport (= 4.1.5)
+    activemodel (4.2.5.1)
+      activesupport (= 4.2.5.1)
       builder (~> 3.1)
-    activerecord (4.1.5)
-      activemodel (= 4.1.5)
-      activesupport (= 4.1.5)
-      arel (~> 5.0.0)
-    activesupport (4.1.5)
-      i18n (~> 0.6, >= 0.6.9)
+    activerecord (4.2.5.1)
+      activemodel (= 4.2.5.1)
+      activesupport (= 4.2.5.1)
+      arel (~> 6.0)
+    activesupport (4.2.5.1)
+      i18n (~> 0.7)
       json (~> 1.7, >= 1.7.7)
       minitest (~> 5.1)
-      thread_safe (~> 0.1)
+      thread_safe (~> 0.3, >= 0.3.4)
       tzinfo (~> 1.1)
-    arel (5.0.1.20140414130214)
+    arel (6.0.3)
     builder (3.2.2)
-    i18n (0.6.11)
-    json (1.8.1)
-    minitest (5.3.3)
-    pg (0.17.1)
-    rake (10.3.1)
-    thread_safe (0.3.4)
+    i18n (0.7.0)
+    json (1.8.3)
+    minitest (5.8.4)
+    pg (0.18.4)
+    rake (10.5.0)
+    thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
 
@@ -39,3 +39,6 @@ DEPENDENCIES
   pg
   postgresql_cursor!
   rake
+
+BUNDLED WITH
+   1.11.2

data/README.md

@@ -13,10 +13,16 @@ set is exhausted. By fetching a smaller chunk of data, this reduces the
 amount of memory your application uses and prevents the potential crash
 of running out of memory.
 
-Version 0.5.0 has been refactored to install more smoothly into ActiveRecord.
-It supports Rails and ActiveRecord 3.2.x and up.
+This extension is not intended to support the "FOR UPDATE / WHERE
+CURRENT OF" syntax to process and update each row in place. The primary
+goal is to read a large number of rows using buffering.
 
-##Use Cursors
+Supports Rails/ActiveRecord v3.1 (v3.2 recommended) higher (including
+v5.0) and Ruby 1.9 and higher. Not all features work in ActiveRecord v3.1.
+Support for this gem will only be for officially supported versions of
+ActiveRecord and Ruby; others can try older versions of the gem.
+
+##Using Cursors
 
 PostgreSQLCursor was developed to take advantage of PostgreSQL's cursors. Cursors allow the program
 to declare a cursor to run a given query returning "chunks" of rows to the application program while
@@ -43,6 +49,17 @@ Product.each_row_by_sql("select * from products") { |hash| Product.process(hash)
 Product.each_instance_by_sql("select * from products") { |product| product.process }
 ```
 
+Cursors must be run in a transaction if you need to fetch each row yourself
+
+```ruby
+Product.transaction do
+  cursor = Product.all.each_row
+  row = cursor.fetch                       #=> {"id"=>"1"}
+  row = cursor.fetch(symbolize_keys:true)  #=> {:id =>"2"}
+  cursor.close
+end
+```
+
 All these methods take an options hash to control things more:
 
     block_size:n      The number of rows to fetch from the database each time (default 1000)
@@ -53,6 +70,7 @@ All these methods take an options hash to control things more:
                       PostgreSQL uses 0.1 (optimize for 10% of result set)
                       This library uses 1.0 (Optimize for 100% of the result set)
                       Do not override this value unless you understand it.
+    with_hold:boolean Keep the cursor "open" even after a commit.
 
 Notes:
 
@@ -158,10 +176,22 @@ There are drawbacks with these methods:
 * The query is rerun for each chunk (1000 rows), starting at the next id sequence.
 * You cannot use overly complex queries as that will be rerun and incur more overhead.
 
+### How it works
+
+Under the covers, the library calls the PostgreSQL cursor operations
+with the psuedo-code:
+
+    SET cursor_tuple_fraction TO 1.0;
+    DECLARE cursor_1 CURSOR WITH HOLD FOR select * from widgets;
+    loop
+      rows = FETCH 100 FROM cursor_1;
+      rows.each {|row| yield row}
+    until rows.size < 100;
+    CLOSE cursor_1;
 
 ##Meta
 ###Author
-Allen Fair, [@allenfair](https://twitter/com/allenfair), http://github.com/afair
+Allen Fair, [@allenfair](https://twitter.com/allenfair), http://github.com/afair
 
 Thanks to:
 
@@ -179,6 +209,11 @@ Thanks to:
   (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
 * Send me a pull request. Bonus points for topic branches.
 
+###Code of Conduct
+
+This project adheres to the [Open Code of Conduct](http://todogroup.org/opencodeofconduct/#postgresql_cursor/2016@allenfair.com).
+By participating, you are expected to honor this code.
+
 ###Copyright
 
 Copyright (c) 2010-2014 Allen Fair. See (MIT) LICENSE for details.

data/Rakefile

@@ -20,5 +20,5 @@ end
 desc "Setup testing database and table"
 task :setup do
   sh %q(createdb postgresql_cursor_test)
-  sh %Q<echo "create table products ( id serial primary key);" | psql postgresql_cursor_test>
+  sh %Q<echo "create table products ( id serial primary key, data varchar);" | psql postgresql_cursor_test>
 end

data/lib/postgresql_cursor/active_record/sql_cursor.rb

@@ -9,6 +9,7 @@ module PostgreSQLCursor
       #   block_size: 1..n      - The number of rows to fetch per db block fetch
       #   while: value          - Exits loop when block does not return this value.
       #   until: value          - Exits loop when block returns this value.
+      #   with_hold: boolean    - Allows the query to remain open across commit points.
       #
       # Example:
       #   Post.each_row { |hash| Post.process(hash) }
@@ -41,6 +42,7 @@ module PostgreSQLCursor
       #   block_size: 1..n      - The number of rows to fetch per db block fetch
       #   while: value          - Exits loop when block does not return this value.
       #   until: value          - Exits loop when block returns this value.
+      #   with_hold: boolean    - Allows the query to remain open across commit points.
       #
       # Example:
       #   Post.each_row_by_sql("select * from posts") { |hash| Post.process(hash) }

data/lib/postgresql_cursor/cursor.rb

@@ -9,6 +9,7 @@
 #   block_size: 1..n      - The number of rows to fetch per db block fetch
 #   while: value          - Exits loop when block does not return this value.
 #   until: value          - Exits loop when block returns this value.
+#   with_hold: boolean    - Allows the query to remain open across commit points.
 #
 # Exmaples:
 #   PostgreSQLCursor::Cursor.new("select ...").each { |hash| ... }
@@ -30,6 +31,7 @@ module PostgreSQLCursor
     #   fraction: 0.1..1.0    - The cursor_tuple_fraction (default 1.0)
     #   block_size: 1..n      - The number of rows to fetch per db block fetch
     #                           Defaults to 1000
+    #   with_hold       - Allows the query to remain open across commit points.
     #
     # Examples
     #
@@ -128,7 +130,7 @@ module PostgreSQLCursor
         rescue Exception => e
           raise e
         ensure
-          close
+          close if @block
         end
       end
       @count
@@ -160,16 +162,20 @@ module PostgreSQLCursor
     def open
       set_cursor_tuple_fraction
       @cursor = @@cursor_seq += 1
-      @result = @connection.execute("declare cursor_#{@cursor} cursor for #{@sql}")
+      hold = @options[:with_hold] ? 'with hold ' : ''
+      @result = @connection.execute("declare cursor_#{@cursor} cursor #{hold}for #{@sql}")
       @block = []
     end
 
     # Public: Returns the next row from the cursor, or empty hash if end of results
     #
     # Returns a row as a hash of {'colname'=>value,...}
-    def fetch
+    def fetch(options={})
+      open unless @block
       fetch_block if @block.size==0
-      @block.shift
+      row = @block.shift
+      row = row.symbolize_keys if row && options[:symbolize_keys]
+      row
     end
 
     # Private: Fetches the next block of rows into @block

data/lib/postgresql_cursor/version.rb

@@ -1,3 +1,3 @@
 module PostgresqlCursor
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/postgresql_cursor.gemspec

@@ -18,8 +18,13 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
- #spec.add_dependency "pg" # Remove this for jruby, which should specify 'activerecord-jdbcpostgresql-adapter'
-  spec.add_dependency "activerecord", ">= 3.2.0"
+  #spec.add_dependency "pg" # Remove this for jruby, which should specify 'activerecord-jdbcpostgresql-adapter'
+  spec.add_dependency "activerecord", ">= 3.1.0"
+  #spec.add_dependency "activerecord", "~> 3.1.0"
+  # Tests don't run on 4.0.0 since AR/AS have an older version of minitest as a run-time dependency(!) than our tests support
+  #spec.add_dependency "activerecord", "~> 4.0.0";# spec.add_dependency "minitest", "~> 4.2.0"
+  #spec.add_dependency "activerecord", "~> 4.1.0"
+  #spec.add_dependency "activerecord", "~> 5.0.0.beta2"
 
   spec.add_development_dependency "pg"
   spec.add_development_dependency "rake"

data/test-app/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/test-app/Gemfile

@@ -1,7 +1,8 @@
 # A sample Gemfile
 source "https://rubygems.org"
 
-gem 'activerecord', '~> 3.2.0'
+gem 'activerecord', '~> 3.1.0'
+#gem 'activerecord', '~> 3.2.0'
 #gem 'activerecord', '~> 4.0.0'
 #gem 'activerecord', '~> 4.1.0'
 #gem 'activerecord', '4.1.2.rc1'

data/test/helper.rb

@@ -8,7 +8,7 @@ require 'postgresql_cursor'
 ActiveRecord::Base.establish_connection(adapter: 'postgresql',
   database: ENV['TEST_DATABASE'] || 'postgresql_cursor_test',
   username: ENV['TEST_USER']     || ENV['USER'] || 'postgresql_cursor')
-   
+
 class Product < ActiveRecord::Base
   # create table records (id serial primary key);
   def self.generate(max=1_000)

data/test/test_postgresql_cursor.rb

@@ -78,5 +78,36 @@ class TestPostgresqlCursor < Minitest::Test
     assert_equal 1000, r.size
     assert_equal Fixnum, r.first.class
   end
-  
+
+  def test_with_hold
+    items = 0
+    Product.where("id < 4") .each_instance(with_hold: true, block_size:1) do |row|
+      Product.transaction do
+        row.update(data:Time.now.to_f.to_s)
+        items += 1
+      end
+    end
+    assert_equal 3, items
+  end
+
+  def test_fetch_symbolize_keys
+    Product.transaction do
+      #cursor = PostgreSQLCursor::Cursor.new("select * from products order by 1")
+      cursor = Product.all.each_row
+      r = cursor.fetch
+      assert r.has_key?("id")
+      r = cursor.fetch(symbolize_keys:true)
+      assert r.has_key?(:id)
+      cursor.close
+    end
+  end
+
+  def test_bad_sql
+    begin
+      ActiveRecord::Base.each_row_by_sql('select * from bad_table') { }
+      raise "Did Not Raise Expected Exception"
+    rescue Exception => e
+      assert_match(/bad_table/, e.message)
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgresql_cursor
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Allen Fair
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-11 00:00:00.000000000 Z
+date: 2016-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: 3.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.2.0
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +83,6 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- VERSION
 - lib/postgresql_cursor.rb
 - lib/postgresql_cursor/active_record/connection_adapters/postgresql_type_map.rb
 - lib/postgresql_cursor/active_record/relation/cursor_iterators.rb
@@ -91,8 +90,8 @@ files:
 - lib/postgresql_cursor/cursor.rb
 - lib/postgresql_cursor/version.rb
 - postgresql_cursor.gemspec
+- test-app/.gitignore
 - test-app/Gemfile
-- test-app/Gemfile.lock
 - test-app/app.rb
 - test-app/run.sh
 - test/helper.rb
@@ -117,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: ActiveRecord PostgreSQL Adapter extension for using a cursor to return a

data/VERSION

@@ -1 +0,0 @@
-0.4.3

data/test-app/Gemfile.lock

@@ -1,34 +0,0 @@
-PATH
-  remote: /Users/allen/src/postgresql_cursor
-  specs:
-    postgresql_cursor (0.5.0)
-      activerecord (>= 3.2.0)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    activemodel (3.2.18)
-      activesupport (= 3.2.18)
-      builder (~> 3.0.0)
-    activerecord (3.2.18)
-      activemodel (= 3.2.18)
-      activesupport (= 3.2.18)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activesupport (3.2.18)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
-    arel (3.0.3)
-    builder (3.0.4)
-    i18n (0.6.9)
-    multi_json (1.10.1)
-    pg (0.17.1)
-    tzinfo (0.3.39)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  activerecord (~> 3.2.0)
-  pg
-  postgresql_cursor!