postgresql_cursor 0.4.3 → 0.5.0

data/test-app/Gemfile

@@ -0,0 +1,14 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+gem 'activerecord', '~> 3.2.0'
+#gem 'activerecord', '~> 4.0.0'
+#gem 'activerecord', '~> 4.1.0'
+#gem 'activerecord', '4.1.2.rc1'
+
+# For testing against Edge Rails
+#gem 'activerecord', github: 'rails/rails', branch: 'master'
+#gem 'arel', github: 'rails/arel', branch: 'master'
+
+gem 'pg'
+gem 'postgresql_cursor', path:"#{ENV['HOME']}/src/postgresql_cursor"

data/test-app/Gemfile.lock

@@ -0,0 +1,34 @@
+PATH
+  remote: /Users/allen/src/postgresql_cursor
+  specs:
+    postgresql_cursor (0.5.0)
+      activerecord (>= 3.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (3.2.18)
+      activesupport (= 3.2.18)
+      builder (~> 3.0.0)
+    activerecord (3.2.18)
+      activemodel (= 3.2.18)
+      activesupport (= 3.2.18)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activesupport (3.2.18)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
+    arel (3.0.3)
+    builder (3.0.4)
+    i18n (0.6.9)
+    multi_json (1.10.1)
+    pg (0.17.1)
+    tzinfo (0.3.39)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (~> 3.2.0)
+  pg
+  postgresql_cursor!

data/test-app/app.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+################################################################################
+# To run this "app", do a "rake setup" first.
+# To work with this app, run the "rake console" task, which loads this file.
+################################################################################
+require 'rubygems'
+require 'bundler/setup'
+require 'pg'
+require 'active_record'
+require 'postgresql_cursor'
+
+ActiveRecord::Base.establish_connection( adapter: 'postgresql',
+  database: ENV['TEST_DATABASE'] || 'postgresql_cursor_test',
+  username: ENV['TEST_USER']     || ENV['USER'] || 'postgresql_cursor')
+
+class Product < ActiveRecord::Base
+  def self.generate(max=1_000)
+    Product.destroy_all
+    max.times do |i|
+      connection.execute("insert into products values (#{i})")
+    end
+  end
+
+  def tests
+    Product.where("id>0").each_row(block_size:100) { |r| p r["id"] } # Hash
+    Product.where("id>0").each_instance(block_size:100) { |r| p r.id } # Instance
+  end
+end
+
+#Product.generate

data/test-app/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+#bundle install
+if [ "$1" = "irb" ]; then
+  bundle exec irb -Ilib -r postgresql_cursor
+elif [ "$1" = "setup" ]; then
+  createdb postgresql_cursor_test
+  echo "create table products ( id serial);" | psql postgresql_cursor_test
+else
+  bundle exec ruby app.rb
+fi

data/test/helper.rb

@@ -1,25 +1,22 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'rubygems'
 require 'minitest'
 require 'active_record'
-
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'postgresql_cursor'
 
-ActiveRecord::Base.establish_connection :database=>'allen_test', :adapter=>'postgresql', :username=>'allen'
-class Model < ActiveRecord::Base
-  #set_table_name "records"
-  self.table_name = "records"
-
+ActiveRecord::Base.establish_connection(adapter: 'postgresql',
+  database: ENV['TEST_DATABASE'] || 'postgresql_cursor_test',
+  username: ENV['TEST_USER']     || ENV['USER'] || 'postgresql_cursor')
+   
+class Product < ActiveRecord::Base
   # create table records (id serial primary key);
-  def self.generate(max=1_000_000)
-    max.times do
-      connection.execute("insert into records values (nextval('records_id_seq'::regclass))")
+  def self.generate(max=1_000)
+    max.times do |i|
+      connection.execute("insert into products values (#{i+1})")
     end
   end
 end
 
-Model.generate(1000) if Model.count == 0
-
-class MiniTest::Unit::TestCase
-end
+Product.destroy_all
+Product.generate(1000)

data/test/test_postgresql_cursor.rb

@@ -1,57 +1,82 @@
-require 'helper'
-# database: allen_test
-# create table records ( id serial);
+################################################################################
+# Before running test, set up the test db & table with:
+#     rake setup
+# or create the database manually if your environment doesn't permit
+################################################################################
+require_relative 'helper'
+require 'minitest/autorun'
+require 'minitest/pride'
 
 class TestPostgresqlCursor < Minitest::Test
 
   def test_each
-    c = PostgreSQLCursor.new("select * from records order by 1")
+    c = PostgreSQLCursor::Cursor.new("select * from products order by 1")
     nn = 0
     n = c.each { nn += 1}
     assert_equal nn, n
   end
 
   def test_enumerables
-    assert_equal true, PostgreSQLCursor.new("select * from records order by 1").any?
-    assert_equal false, PostgreSQLCursor.new("select * from records where id<0").any?
+    assert_equal true, PostgreSQLCursor::Cursor.new("select * from products order by 1").any?
+    assert_equal false, PostgreSQLCursor::Cursor.new("select * from products where id<0").any?
   end
 
   def test_each_while_until
-    c = PostgreSQLCursor.new("select * from records order by 1", until:true)
+    c = PostgreSQLCursor::Cursor.new("select * from products order by 1", until:true)
     n = c.each { |r| r[:id].to_i > 100 }
-    assert_equal 101, n
+    assert_equal 1000, n
 
-    c = PostgreSQLCursor.new("select * from records order by 1", while:true)
+    c = PostgreSQLCursor::Cursor.new("select * from products order by 1", while:true)
     n = c.each { |r| r[:id].to_i < 100 }
-    assert_equal 100, n
+    assert_equal 1000, n
   end
 
   def test_relation
     nn = 0
-    Model.where("id>0").each_row {|r| nn += 1 }
+    Product.where("id>0").each_row {|r| nn += 1 }
     assert_equal 1000, nn
   end
 
   def test_activerecord
     nn = 0
-    Model.each_row_by_sql("select * from records") {|r| nn += 1 }
+    row = nil
+    Product.each_row_by_sql("select * from products") {|r| row = r; nn += 1 }
     assert_equal 1000, nn
+    assert_equal Hash, row.class
 
     nn = 0
-    row = nil
-    Model.each_instance_by_sql("select * from records") {|r| row = r; nn += 1 }
+    Product.each_instance_by_sql("select * from products") {|r| row = r; nn += 1 }
     assert_equal 1000, nn
-    assert_equal Model, row.class
+    assert_equal Product, row.class
   end
 
   def test_exception
     begin
-      Model.each_row_by_sql("select * from records") do |r|
+      Product.each_row_by_sql("select * from products") do |r|
         raise "Oops"
       end
     rescue Exception => e
       assert_equal e.message, 'Oops'
     end
   end
+
+  def test_cursor
+    cursor = Product.all.each_row
+    assert cursor.respond_to?(:each)
+    r = cursor.map { |row| row["id"] }
+    assert_equal 1000, r.size
+    cursor = Product.each_row_by_sql("select * from products")
+    assert cursor.respond_to?(:each)
+    r = cursor.map { |row| row["id"] }
+    assert_equal 1000, r.size
+  end
+
+  def test_pluck
+    r = Product.pluck_rows(:id)
+    assert_equal 1000, r.size
+    r = Product.all.pluck_instances(:id)
+    assert_equal 1000, r.size
+    assert_equal Fixnum, r.first.class
+  end
   
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: postgresql_cursor
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Allen Fair
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-06 00:00:00.000000000 Z
+date: 2014-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -16,9 +16,51 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.2.0
   type: :runtime
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -27,25 +69,37 @@ dependencies:
 description: PostgreSQL Cursor is an extension to the ActiveRecord PostgreSQLAdapter
   for very large result sets. It provides a cursor open/fetch/close interface to access
   data without loading all rows into memory, and instead loads the result rows in
-  "chunks" (default of 10_000 rows), buffers them, and returns the rows one at a time.
-email: allen.fair@gmail.com
+  "chunks" (default of 1_000 rows), buffers them, and returns the rows one at a time.
+email:
+- allen.fair@gmail.com
 executables: []
 extensions: []
-extra_rdoc_files:
-- LICENSE
-- README.rdoc
+extra_rdoc_files: []
 files:
 - ".document"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
 - LICENSE
-- README.rdoc
+- README.md
 - Rakefile
 - VERSION
 - lib/postgresql_cursor.rb
+- lib/postgresql_cursor/active_record/connection_adapters/postgresql_type_map.rb
+- lib/postgresql_cursor/active_record/relation/cursor_iterators.rb
+- lib/postgresql_cursor/active_record/sql_cursor.rb
+- lib/postgresql_cursor/cursor.rb
+- lib/postgresql_cursor/version.rb
 - postgresql_cursor.gemspec
+- test-app/Gemfile
+- test-app/Gemfile.lock
+- test-app/app.rb
+- test-app/run.sh
 - test/helper.rb
 - test/test_postgresql_cursor.rb
 homepage: http://github.com/afair/postgresql_cursor
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -63,9 +117,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.1
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: ActiveRecord PostgreSQL Adapter extension for using a cursor to return a
   large result set
-test_files: []
+test_files:
+- test/helper.rb
+- test/test_postgresql_cursor.rb

data/README.rdoc

@@ -1,97 +0,0 @@
-= PostgreSQLCursor for handling large Result Sets
-
-{<img src="https://badge.fury.io/rb/postgresql_cursor.svg" alt="Gem Version" />}[http://badge.fury.io/rb/postgresql_cursor]
-
-PostgreSQLCursor extends ActiveRecord to allow for efficient processing of queries
-returning a large number of rows.
-
-Note: Rails/ActiveRecord 4.x support has been reworked in the v0.5 Branch. For AR < 4.x, use the gem 0.4.x series.
-
-== Why use this?
-
-ActiveRecord is designed and optimized for web performance. In a web transaction, only a "page" of
-around 20 rows is returned to the user. When you do this
-
-  Model.find(:all, :conditions=>["id>0"]
-
-The database returns all matching result set rows to ActiveRecord, which instantiates each row with
-the data returned. This function returns an array of all these rows to the caller.
-
-Asyncronous, Background, or Offline processing may require processing a large amount of data.
-When there is a very large number of rows, this requires a lot more memory to hold the data. Ruby
-does not return that memory after processing the array, and the causes your process to "bloat". If you
-don't have enough memory, it will cause an exception.
-
-== Enter find_each
-
-To solve this problem, ActiveRecord gives us two alternative methods that work in "chunks" of your data:
-
-  Model.where("id>0").find_each { |model| model.process! }
-
-  Model.where("id>0").find_in_batches do |batch|
-    batch.each { |model| model.process! }
-  end
-
-Optionally, you can specify a :batch_size option as the size of the "chunk", and defaults to 1000.
-
-There are drawbacks with these methods:
-
-* You cannot specify the order, it will be ordered by the primary key (usually id)
-* The primary key must be numeric
-* The query is rerun for each chunk (1000 rows), starting at the next id sequence.
-* You cannot use overly complex queries as that will be rerun and incur more overhead.
-
-== PostgreSQLCursor FTW!
-
-PostgreSQLCursor was developed to take advantage of PostgreSQL's cursors. Cursors allow the program
-to declare a cursor to run a given query returning "chunks" of rows to the application program while
-retaining the position of the full result set in the database. This overcomes all the disadvantages
-of using find_each and find_in_batches.
-
-Also, with PostgreSQL, you have on option to have raw hashes of the row returned instead of the
-instantiated models. An informal benchmark showed that returning instances is a factor of 4 times
-slower than returning hashes. If you are can work with the data in this form, you will find better
-performance.
-
-With PostgreSQL, you can work with cursors as follows:
-
-  Model.where("id>0").each_row { |hash| Model.process(hash) }
-
-  Model.where("id>0").each_instance { |model| model.process! }
-  Model.where("id>0").each_instance(block_size:100000) { |model| model.process! }
-
-  Model.each_row_by_sql("select * from models") { |hash| Model.process(hash) }
-
-  Model.each_instance_by_sql("select * from models") { |model| model.process }
-
-All these methods take an options hash to control things more:
-
-  block_size:n      The number of rows to fetch from the database each time (default 1000)
-  while:value       Continue looping as long as the block returns this value
-  until:value       Continue looping until the block returns this value
-  connection:conn   Use this connection instead of the current model connection
-  fraction:float    A value to set for the cursor_tuple_fraction variable.
-                    PostgreSQL uses 0.1 (optimize for 10% of result set)
-                    This library uses 1.0 (Optimize for 100% of the result set)
-                    Do not override this value unless you understand it.
-
-==Authors
-Allen Fair, allen.fair@gmail.com, http://github.com/afair
-
-Thank you to:
-* Iulian Dogariu, http://github.com/iulianu (Fixes)
-* Julian Mehnle, julian@mehnle.net (Suggestions)
-
-== Note on Patches/Pull Requests
-
-* Fork the project.
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a
-  future version unintentionally.
-* Commit, do not mess with rakefile, version, or history.
-  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
-
-== Copyright
-
-Copyright (c) 2010 Allen Fair. See LICENSE for details.