activerecord-postgresql-cursors 0.0.1

data/MIT-LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 2011 2167961 Ontario Inc., Zoocasa <code@zoocasa.com>
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+

data/README.rdoc

@@ -0,0 +1,52 @@
+== ActiveRecord PostgreSQL Cursors
+
+This extension allows you to loop through record sets using cursors in an
+Enumerable fashion. This allows you to cut down memory usage by only pulling
+in individual records on each loop rather than pulling everything into
+memory all at once.
+
+To use a cursor, just change the first parameter to an
+ActiveRecord::Base.find to :cursor instead of :first or :all or
+whatever or use the ActiveRecord::Base.cursor method directly.
+
+	MyModel.find(:cursor, :conditions => 'some_column = true').each do |r|
+		puts r.inspect
+	end
+
+	MyModel.find(:cursor).collect { |r| r.foo / PI }.avg
+
+	MyModel.cursor.each do |r|
+		puts r.inspect
+	end
+
+All ActiveRecord::Base.find options are available and should work as-is.
+As a bonus, the PostgreSQLCursor object returned includes Enumerable,
+so you can iterate to your heart's content.
+
+This extension should work in both Rails 2.3 as well as Rails 3.
+
+At the moment, this is a non-scrollable cursor -- it will only fetch
+forward. Also note that these cursors are non-updateable/insensitive to
+updates to the underlying data. You can write to the records themselves,
+but changes outside of the cursor's transaction will not affect the
+data being retrieved from the point of the cursor's creation, or rather
+more specifically from the time you begin iterating.
+
+The cursor itself is wrapped in a transaction as is required by PostgreSQL
+and the cursor name is automatically generated using random numbers or
+a name supplied during cursor creation. On raised SQL exceptions, the
+transaction is ABORTed and the cursor CLOSEd.
+
+Associations are handled, so you can use :include in your find options. Of
+course, this requires some nonsense when moving the cursor around, but it
+works all the same. In some cases, pre-loading and eager loading of
+associations and whatnot creates an initial query that will grab the initial
+IDs of the model being fetched and then create the actual cursor query out
+of those large joins that ActiveRecord sometimes generates. This larger
+query will be the query that's wrapped in the transaction and in a cursor,
+while the first query is just used to build the larger joined query. This
+allows for a brief window between the point that the cursor query is
+created and the time it is executed. In these cases, it may be wise to wrap
+your use or cursors in your own transaction to ensure that changes made to
+the underlying data don't interfere with your cursor's visbility.
+

data/Rakefile

@@ -0,0 +1,42 @@
+
+# -*- ruby -*-
+
+require 'rubygems'
+require 'rubygems/package_task'
+require 'rake/testtask'
+require 'rdoc/task'
+
+$:.push 'lib'
+
+version = File.read('VERSION') rescue ''
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "activerecord-postgresql-cursors"
+    gem.summary = "Provides some support for PostgreSQL cursors in ActiveRecord."
+    gem.description = gem.summary
+    gem.email = "code@zoocasa.com"
+    gem.homepage = "http://github.com/zoocasa/activerecord-postgresql-cursors"
+    gem.authors =    [ "J Smith" ]
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+desc 'Test PostgreSQL extensions'
+Rake::TestTask.new(:test) do |t|
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+desc 'Build docs'
+Rake::RDocTask.new do |t|
+  require 'rdoc'
+  t.title = "ActiveRecord PostgreSQL Cursors #{version}"
+  t.main = 'README.rdoc'
+  t.rdoc_dir = 'doc'
+  t.rdoc_files.include('README.rdoc', 'MIT-LICENSE', 'lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/activerecord-postgresql-cursors.gemspec

@@ -0,0 +1,45 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{activerecord-postgresql-cursors}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{J Smith}]
+  s.date = %q{2011-07-11}
+  s.description = %q{Provides some support for PostgreSQL cursors in ActiveRecord.}
+  s.email = %q{code@zoocasa.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    "MIT-LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "activerecord-postgresql-cursors.gemspec",
+    "init.rb",
+    "lib/activerecord-postgresql-cursors.rb",
+    "lib/postgresql_cursors_2.rb",
+    "lib/postgresql_cursors_3.rb",
+    "test/cursor_test.rb",
+    "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/zoocasa/activerecord-postgresql-cursors}
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.5}
+  s.summary = %q{Provides some support for PostgreSQL cursors in ActiveRecord.}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/init.rb

@@ -0,0 +1,2 @@
+
+require File.join(File.dirname(__FILE__), 'lib', 'postgresql_cursors')

data/lib/activerecord-postgresql-cursors.rb

@@ -0,0 +1,109 @@
+
+module ActiveRecord
+  # Exception raised when database cursors aren't supported, which they
+  # absolutely should be in our app.
+  class CursorsNotSupported < ActiveRecordError; end
+
+  module Associations
+    module ClassMethods
+      class JoinDependency
+        # Extra method we can use to clear out a couple of things in
+        # JoinDependency so we can use some of the methods for our
+        # cursors code.
+        def clear_with_cursor
+          @reflections            = []
+          @base_records_hash      = {}
+          @base_records_in_order  = []
+        end
+      end
+    end
+  end
+
+  # PostgreSQLCursor is an Enumerable class so you can use each, map,
+  # any? and all of those nice Enumerable methods.
+  #
+  # At the moment, cursors aren't scrollable and are fetch forward-only
+  # and read-only.
+  #
+  # This class isn't really meant to be used outside of the
+  # ActiveRecord::Base#find method.
+  class PostgreSQLCursor
+    include Enumerable
+
+    attr_accessor :cursor_name
+
+    def initialize(model, cursor_name, query, join_dependency = nil)
+      @model = model
+      @cursor_name = if cursor_name
+        @model.connection.quote_table_name(cursor_name.gsub(/"/, '\"'))
+      end
+      @query = query
+      @join_dependency = join_dependency
+    end
+
+    def inspect
+      %{#<ActiveRecord::PostgreSQLCursor cursor_name: "#{cursor_name}", query: "#{@query}">}
+    end
+
+    # Calls block once for each record in the cursor, passing that
+    # record as a parameter.
+    def each
+      @model.transaction do
+        begin
+          declare_cursor
+          if @join_dependency
+            rows = Array.new
+            last_id = nil
+            while row = fetch_forward
+              current_id = row[@join_dependency.join_base.aliased_primary_key]
+              last_id ||= current_id
+              if last_id == current_id
+                rows << row
+                last_id = current_id
+              else
+                yield @join_dependency.instantiate(rows).first
+                @join_dependency.clear_with_cursor
+                rows = [ row ]
+              end
+              last_id = current_id
+            end
+
+            if !rows.empty?
+              yield @join_dependency.instantiate(rows).first
+            end
+          else
+            while row = fetch_forward
+              yield row
+            end
+          end
+        ensure
+          close_cursor
+        end
+      end
+      nil
+    end
+
+    private
+      def cursor_name
+        @cursor_name ||= "cursor_#{(rand * 1000000).ceil}"
+      end
+
+      def fetch_forward #:nodoc:
+        @model.find_by_sql(%{FETCH FORWARD FROM #{cursor_name}}).first
+      end
+
+      def declare_cursor #:nodoc:
+        @model.connection.execute(%{DECLARE #{cursor_name} CURSOR FOR #{@query}})
+      end
+
+      def close_cursor #:nodoc:
+        @model.connection.execute(%{CLOSE #{cursor_name}})
+      end
+  end
+end
+
+if ActiveRecord::VERSION::MAJOR >= 3
+  require File.join(File.dirname(__FILE__), 'postgresql_cursors_3')
+else
+  require File.join(File.dirname(__FILE__), 'postgresql_cursors_2')
+end

data/lib/postgresql_cursors_2.rb

@@ -0,0 +1,75 @@
+
+module ActiveRecord
+  class Base
+    class << self
+      # Override ActiveRecord::Base#find to allow for cursors in
+      # PostgreSQL. To use a cursor, set the first argument of
+      # find to :cursor. A PostgreSQLCursor object will be returned,
+      # which can then be used as an Enumerable to loop through the
+      # results.
+      #
+      # By default, cursor names are generated automatically using
+      # "cursor_#{rand}", where rand is a big ol' random number that
+      # is pretty unlikely to clash if you're using nested cursors.
+      # Alternatively, you can supply a specific cursor name by
+      # supplying a :cursor_name option.
+      def find_with_cursors *args
+        if args.first.to_s == 'cursor'
+          options = args.extract_options!
+          cursor_name = options.delete(:cursor_name)
+          validate_find_options(options)
+          set_readonly_option!(options)
+          find_cursor(cursor_name, options)
+        else
+          find_without_cursors(*args)
+        end
+      end
+      alias_method_chain :find, :cursors
+
+      def cursor(*args)
+        find(:cursor, *args)
+      end
+    end
+
+    private
+      # Find method for using cursors. This works just like the regular
+      # ActiveRecord::Base#find_every method, except it returns a
+      # PostgreSQLCursor object that can be used to loop through records.
+      def self.find_cursor(cursor_name, options)
+        unless connection.is_a? ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+          raise CursorsNotSupported, "#{connection.class} doesn't support cursors"
+        end
+
+        catch :invalid_query do
+          if options[:include]
+            join_dependency = ActiveRecord::Associations::ClassMethods::JoinDependency.new(self, merge_includes(scope(:find, :include), options[:include]), options[:joins])
+            return ActiveRecord::PostgreSQLCursor.new(
+              self,
+              cursor_name,
+              construct_finder_sql_with_included_associations(
+                options,
+                join_dependency
+              ),
+              join_dependency
+            )
+          else
+            return ActiveRecord::PostgreSQLCursor.new(
+              self,
+              cursor_name,
+              construct_finder_sql(
+                options
+              )
+            )
+          end
+        end
+        nil
+      end
+  end
+
+  class PostgreSQLCursor
+    def initialize_with_rails_2(model, cursor_name, query, join_dependency = nil)
+      initialize_without_rails_2(model, cursor_name, query, join_dependency)
+    end
+    alias_method_chain :initialize, :rails_2
+  end
+end

data/lib/postgresql_cursors_3.rb

@@ -0,0 +1,75 @@
+
+module ActiveRecord
+  module CursorExtensions
+    extend ActiveSupport::Concern
+
+    included do
+      alias_method_chain :find, :cursors
+    end
+
+    # Override ActiveRecord::Base#find to allow for cursors in
+    # PostgreSQL. To use a cursor, set the first argument of
+    # find to :cursor. A PostgreSQLCursor object will be returned,
+    # which can then be used as an Enumerable to loop through the
+    # results.
+    #
+    # By default, cursor names are generated automatically using
+    # "cursor_#{rand}", where rand is a big ol' random number that
+    # is pretty unlikely to clash if you're using nested cursors.
+    # Alternatively, you can supply a specific cursor name by
+    # supplying a :cursor_name option.
+    def find_with_cursors(*args)
+      if args.first.to_s == 'cursor'
+        options = args.extract_options!
+        cursor_name = options.delete(:cursor_name)
+        find_cursor(cursor_name, options)
+      else
+        find_without_cursors(*args)
+      end
+    end
+
+    def cursor(*args)
+      find_with_cursors('cursor', *args)
+    end
+
+    private
+      # Find method for using cursors. This works just like the regular
+      # ActiveRecord::Base#find_every method, except it returns a
+      # PostgreSQLCursor object that can be used to loop through records.
+      def find_cursor(cursor_name, options)
+        unless connection.is_a? ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+          raise CursorsNotSupported, "#{connection.class} doesn't support cursors"
+        end
+
+        relation = apply_finder_options(options)
+        including = (relation.eager_load_values + relation.includes_values).uniq
+
+        if including.present?
+          join_dependency = ActiveRecord::Associations::ClassMethods::JoinDependency.new(@klass, including, nil)
+          join_relation = relation.construct_relation_for_association_find(join_dependency)
+
+          ActiveRecord::PostgreSQLCursor.new(self, cursor_name, join_relation, join_dependency)
+        else
+          ActiveRecord::PostgreSQLCursor.new(self, cursor_name, relation)
+        end
+      end
+  end
+
+  class PostgreSQLCursor
+    def initialize_with_rails_3(model, cursor_name, relation, join_dependency = nil)
+      @relation = relation
+      initialize_without_rails_3(model, cursor_name, relation.to_sql, join_dependency)
+    end
+    alias_method_chain :initialize, :rails_3
+  end
+end
+
+class ActiveRecord::Relation
+  include ActiveRecord::CursorExtensions
+end
+
+class ActiveRecord::Base
+  class << self
+    delegate :cursor, :to => :scoped
+  end
+end

data/test/cursor_test.rb

@@ -0,0 +1,97 @@
+
+$: << File.dirname(__FILE__)
+require 'test_helper'
+
+class PostgreSQLCursorTests < Test::Unit::TestCase
+  include PostgreSQLCursorTestHelper
+
+  def test_find_cursor
+    cursor = Foo.find(:cursor, :order => 'id')
+
+    assert(cursor.is_a?(ActiveRecord::PostgreSQLCursor))
+
+    assert_equal(%w{ one two three four five }, cursor.collect(&:name))
+  end
+
+  def test_cursor_scoped
+    cursor = Foo.cursor(:order => 'id')
+
+    assert(cursor.is_a?(ActiveRecord::PostgreSQLCursor))
+
+    assert_equal(%w{ one two three four five }, cursor.collect(&:name))
+  end
+
+  def test_cursor_while_updating
+    cursor = Foo.cursor(:order => 'id')
+
+    cursor.each do |row|
+      row.name = "#{row.name}_updated"
+      assert(row.save)
+    end
+
+    assert_equal(%w{ one_updated two_updated three_updated four_updated five_updated }, cursor.collect(&:name))
+  end
+
+  def test_with_associations
+    cursor = Foo.cursor(:order => 'id')
+
+    cursor.each do |row|
+      assert(row.is_a?(Foo))
+      row.bars.each do |bar|
+        assert(bar.is_a?(Bar))
+      end
+    end
+  end
+
+  def test_with_associations_eager_loading
+    cursor = Foo.cursor(:order => 'foos.id', :include => :bars)
+
+    cursor.each do |row|
+      assert(row.is_a?(Foo))
+      row.bars.each do |bar|
+        assert(bar.is_a?(Bar))
+      end
+    end
+  end
+
+  def test_nested_cursors
+    cursor = Foo.cursor(:order => 'foos.id')
+
+    cursor.each do |row|
+      bars_cursor = row.bars.cursor
+      assert(bars_cursor.is_a?(ActiveRecord::PostgreSQLCursor))
+
+      bars_cursor.each do |bar|
+        assert(bar.is_a?(Bar))
+      end
+    end
+  end
+
+  if ActiveRecord::VERSION::MAJOR >= 3
+    def test_as_relation
+      cursor = Foo.order('foos.id').where('foos.id >= 3').cursor
+      assert_equal(3, cursor.to_a.length)
+
+      cursor.each do |row|
+        assert(row.is_a?(Foo))
+        assert_equal(2, row.bars.length)
+        row.bars.each do |bar|
+          assert(bar.is_a?(Bar))
+        end
+      end
+    end
+
+    def test_as_relation_with_associations
+      cursor = Foo.includes(:bars).order('foos.id').where('foos.id >= 3').cursor
+      assert_equal(3, cursor.to_a.length)
+
+      cursor.each do |row|
+        assert(row.is_a?(Foo))
+        assert_equal(2, row.bars.length)
+        row.bars.each do |bar|
+          assert(bar.is_a?(Bar))
+        end
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,68 @@
+
+ACTIVERECORD_GEM_VERSION = ENV['ACTIVERECORD_GEM_VERSION'] || '~> 3.0.3'
+
+require 'rubygems'
+gem 'activerecord', ACTIVERECORD_GEM_VERSION
+
+require 'active_support'
+require 'active_support/core_ext/module/aliasing'
+require 'active_record'
+require 'test/unit'
+require 'logger'
+require File.join(File.dirname(__FILE__), *%w{ .. lib activerecord-postgresql-cursors })
+
+puts "Testing against ActiveRecord #{Gem.loaded_specs['activerecord'].version.to_s}"
+
+ActiveRecord::Base.logger = Logger.new("debug.log")
+ActiveRecord::Base.configurations = {
+  'arunit' => {
+    :adapter => 'postgresql',
+    :database => 'postgresql_cursors_unit_tests',
+    :min_messages => 'warning',
+    :schema_search_path => 'public'
+  }
+}
+
+ActiveRecord::Base.establish_connection 'arunit'
+ARBC = ActiveRecord::Base.connection
+
+if !ARBC.table_exists?('foos')
+  ActiveRecord::Migration.create_table(:foos) do |t|
+    t.text :name
+  end
+end
+
+if !ARBC.table_exists?('bars')
+  ActiveRecord::Migration.create_table(:bars) do |t|
+    t.text :name
+    t.integer :foo_id
+  end
+end
+
+
+class Bar < ActiveRecord::Base
+  belongs_to :foo
+end
+
+class Foo < ActiveRecord::Base
+  has_many :bars
+end
+
+module PostgreSQLCursorTestHelper
+  def setup
+    Foo.delete_all
+    Bar.delete_all
+    ARBC.execute(%{select setval('foos_id_seq', 1, false)})
+    ARBC.execute(%{select setval('bars_id_seq', 1, false)})
+
+    %w{ six seven eight nine ten eleven twelve thirteen fourteen fifteen }.each do |name|
+      Bar.create(:name => name)
+    end
+
+    %w{ one two three four five }.each_with_index do |name, i|
+      foo = Foo.new(:name => name)
+      foo.bar_ids = [ i + 1, i + 6 ]
+      foo.save
+    end
+  end
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification 
+name: activerecord-postgresql-cursors
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- J Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-11 00:00:00 Z
+dependencies: []
+
+description: Provides some support for PostgreSQL cursors in ActiveRecord.
+email: code@zoocasa.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- MIT-LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- activerecord-postgresql-cursors.gemspec
+- init.rb
+- lib/activerecord-postgresql-cursors.rb
+- lib/postgresql_cursors_2.rb
+- lib/postgresql_cursors_3.rb
+- test/cursor_test.rb
+- test/test_helper.rb
+homepage: http://github.com/zoocasa/activerecord-postgresql-cursors
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: Provides some support for PostgreSQL cursors in ActiveRecord.
+test_files: []
+