columns_on_demand 2.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+columns-on-demand-*.gem
+Gemfile.lock
+test/columns_on_demand_test.db

data/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+
+# Declare your gem's dependencies in transaction_isolation_level.gemspec.
+# Bundler will treat runtime dependencies like base dependencies, and
+# development dependencies will be added by default to the :development group.
+gemspec
+
+# Declare any dependencies that are still in development here instead of in
+# your gemspec. These might include edge Rails or gems from your path or
+# Git. Remember to move these dependencies to your gemspec before releasing
+# your gem to rubygems.org.
+
+# To use debugger
+# gem 'ruby-debug19', :require => 'ruby-debug'
+# gem 'ruby-debug'

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008-2009 Will Bryant, Sekuda Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,45 @@
+ColumnsOnDemand
+===============
+
+Lazily loads large columns on demand.
+
+By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
+of specific columns to load on demand can be given.
+
+This is useful to reduce the memory taken by Rails when loading a number of records
+that have large columns if those particular columns are actually not required most
+of the time.  In this situation it can also greatly reduce the database query time
+because loading large BLOB/TEXT columns generally means seeking to other database
+pages since they are not stored wholly in the record's page itself.
+
+Although this plugin is mainly used for BLOB and TEXT columns, it will actually
+work on all types - and is just as useful for large string fields etc.
+
+
+Compatibility
+=============
+
+Supports mysql, mysql2, postgresql, and sqlite3.
+
+Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8.  Note that
+3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
+
+
+Example
+=======
+
+# Example.all will exclude the file_data and processing_log columns from the
+# SELECT query, and example.file_data and example.processing_log will load & cache
+# that individual column value for the record instance.
+class Example
+  columns_on_demand :file_data, :processing_log
+end
+
+# Scans the 'examples' table columns and registers all TEXT (:text) and BLOB (:binary)
+# columns for loading on demand.
+class Example
+  columns_on_demand
+end
+
+
+Copyright (c) 2008-2012 Will Bryant, Sekuda Ltd, released under the MIT license

data/Rakefile

@@ -0,0 +1,16 @@
+#!/usr/bin/env rake
+
+require 'bundler/gem_tasks'
+require 'rake'
+require 'rake/testtask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the columns_on_demand plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/*_test.rb'
+  t.verbose = true
+end

data/columns_on_demand.gemspec

@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/columns_on_demand/version', __FILE__)
+
+spec = Gem::Specification.new do |gem|
+  gem.name         = 'columns_on_demand'
+  gem.version      = ColumnsOnDemand::VERSION
+  gem.summary      = "Lazily loads large columns on demand."
+  gem.description  = <<-EOF
+Lazily loads large columns on demand.
+
+By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
+of specific columns to load on demand can be given.
+
+This is useful to reduce the memory taken by Rails when loading a number of records
+that have large columns if those particular columns are actually not required most
+of the time.  In this situation it can also greatly reduce the database query time
+because loading large BLOB/TEXT columns generally means seeking to other database
+pages since they are not stored wholly in the record's page itself.
+
+Although this plugin is mainly used for BLOB and TEXT columns, it will actually
+work on all types - and is just as useful for large string fields etc.
+
+
+Compatibility
+=============
+
+Supports mysql, mysql2, postgresql, and sqlite3.
+
+Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8.  Note that
+3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
+EOF
+  gem.has_rdoc     = false
+  gem.author       = "Will Bryant"
+  gem.email        = "will.bryant@gmail.com"
+  gem.homepage     = "http://github.com/willbryant/columns_on_demand"
+  
+  gem.executables  = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files        = `git ls-files`.split("\n")
+  gem.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.require_path = "lib"
+  
+  gem.add_dependency "activerecord"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "mysql"
+  gem.add_development_dependency "mysql2"
+  gem.add_development_dependency "pg"
+  gem.add_development_dependency "sqlite3"
+end

data/init.rb

@@ -0,0 +1,2 @@
+require 'columns_on_demand'
+ActiveRecord::Base.send(:extend, ColumnsOnDemand::BaseMethods)

data/lib/columns_on_demand.rb

@@ -0,0 +1,143 @@
+module ColumnsOnDemand
+  module BaseMethods
+    def columns_on_demand(*columns_to_load_on_demand)
+      class_attribute :columns_to_load_on_demand, :instance_writer => false
+      self.columns_to_load_on_demand = columns_to_load_on_demand.empty? ? blob_and_text_columns : columns_to_load_on_demand.collect(&:to_s)
+
+      extend ClassMethods
+      include InstanceMethods
+
+      if ActiveRecord::VERSION::MAJOR > 2
+        relation # set up @relation
+        class << @relation # but always modify @relation, not the temporary returned by .relation if there's a where(type condition)
+          def build_select_with_columns_on_demand(arel, selects)
+            unless selects.empty?
+              build_select_without_columns_on_demand(arel, selects)
+            else
+              arel.project(Arel::SqlLiteral.new(@klass.default_select(true)))
+            end
+          end
+          alias_method_chain :build_select, :columns_on_demand
+        end
+      end
+      
+      class <<self
+        unless ActiveRecord.const_defined?(:AttributeMethods) &&
+               ActiveRecord::AttributeMethods::const_defined?(:Serialization) &&
+               ActiveRecord::AttributeMethods::Serialization::Attribute
+          alias_method_chain :define_read_method_for_serialized_attribute, :columns_on_demand
+        end
+        alias_method_chain :reset_column_information,        :columns_on_demand
+      end
+      alias_method_chain   :attributes,                      :columns_on_demand
+      alias_method_chain   :attribute_names,                 :columns_on_demand
+      alias_method_chain   :read_attribute,                  :columns_on_demand
+      alias_method_chain   :read_attribute_before_type_cast, :columns_on_demand
+      alias_method_chain   :missing_attribute,               :columns_on_demand
+      alias_method_chain   :reload,                          :columns_on_demand
+    end
+    
+    def reset_column_information_with_columns_on_demand
+      @columns_to_select = nil
+      reset_column_information_without_columns_on_demand
+    end
+    
+    def define_read_method_for_serialized_attribute_with_columns_on_demand(attr_name)
+      define_read_method_for_serialized_attribute_without_columns_on_demand(attr_name)
+      scope = method_defined?(:generated_attribute_methods) ? generated_attribute_methods : self
+      scope.module_eval("def #{attr_name}_with_columns_on_demand; ensure_loaded('#{attr_name}'); #{attr_name}_without_columns_on_demand; end; alias_method_chain :#{attr_name}, :columns_on_demand", __FILE__, __LINE__)      
+    end
+
+    def blob_and_text_columns
+      columns.inject([]) do |blob_and_text_columns, column|
+        blob_and_text_columns << column.name if column.type == :binary || column.type == :text
+        blob_and_text_columns
+      end
+    end
+  end
+  
+  module ClassMethods
+    # this is the method API as called by ActiveRecord 2.x.  we also call it ourselves above in our ActiveRecord 3 extensions.
+    def default_select(qualified)
+      @columns_to_select ||= (columns.collect(&:name) - columns_to_load_on_demand).collect {|attr_name| connection.quote_column_name(attr_name)}
+      if qualified
+        quoted_table_name + '.' + @columns_to_select.join(", #{quoted_table_name}.")
+      else
+        @columns_to_select.join(", ")
+      end
+    end
+  end
+  
+  module InstanceMethods
+    def columns_loaded
+      @columns_loaded ||= Set.new
+    end
+
+    def column_loaded?(attr_name)
+      !columns_to_load_on_demand.include?(attr_name) || !@attributes[attr_name].nil? || new_record? || columns_loaded.include?(attr_name)
+    end
+
+    def attributes_with_columns_on_demand
+      load_attributes(*columns_to_load_on_demand.reject {|attr_name| column_loaded?(attr_name)})
+      attributes_without_columns_on_demand
+    end
+
+    def attribute_names_with_columns_on_demand
+      (attribute_names_without_columns_on_demand + columns_to_load_on_demand).uniq.sort
+    end
+    
+    def load_attributes(*attr_names)
+      return if attr_names.blank?
+      values = connection.select_rows(
+        "SELECT #{attr_names.collect {|attr_name| connection.quote_column_name(attr_name)}.join(", ")}" +
+        "  FROM #{self.class.quoted_table_name}" +
+        " WHERE #{connection.quote_column_name(self.class.primary_key)} = #{quote_value(id, self.class.columns_hash[self.class.primary_key])}")
+      row = values.first || raise(ActiveRecord::RecordNotFound, "Couldn't find #{self.class.name} with ID=#{id}")
+      attr_names.each_with_index do |attr_name, i|
+        columns_loaded << attr_name
+        @attributes[attr_name] = row[i]
+
+        if coder = self.class.serialized_attributes[attr_name]
+          if ActiveRecord.const_defined?(:AttributeMethods) &&
+             ActiveRecord::AttributeMethods::const_defined?(:Serialization) &&
+             ActiveRecord::AttributeMethods::Serialization::Attribute
+            # in 3.2 @attributes has a special Attribute struct to help cache both serialized and unserialized forms
+            @attributes[attr_name] = ActiveRecord::AttributeMethods::Serialization::Attribute.new(coder, @attributes[attr_name], :serialized)
+          elsif ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR == 1
+            # in 2.3 an 3.0, @attributes has the serialized form; from 3.1 it has the deserialized form
+            @attributes[attr_name] = coder.load @attributes[attr_name]
+          end
+        end
+      end
+    end
+    
+    def ensure_loaded(attr_name)
+      load_attributes(attr_name.to_s) unless column_loaded?(attr_name.to_s)
+    end
+    
+    def read_attribute_with_columns_on_demand(attr_name)
+      ensure_loaded(attr_name)
+      read_attribute_without_columns_on_demand(attr_name)
+    end
+
+    def read_attribute_before_type_cast_with_columns_on_demand(attr_name)
+      ensure_loaded(attr_name)
+      read_attribute_before_type_cast_without_columns_on_demand(attr_name)
+    end
+
+    def missing_attribute_with_columns_on_demand(attr_name, *args)
+      if columns_to_load_on_demand.include?(attr_name)
+        load_attributes(attr_name)
+      else
+        missing_attribute_without_columns_on_demand(attr_name, *args)
+      end
+    end
+    
+    def reload_with_columns_on_demand(*args)
+      reload_without_columns_on_demand(*args).tap do
+        columns_loaded.clear
+        columns_to_load_on_demand.each {|attr_name| @attributes.delete(attr_name)}
+      end
+    end
+  end
+end

data/lib/columns_on_demand/version.rb

@@ -0,0 +1,3 @@
+module ColumnsOnDemand
+  VERSION = '2.0.0'
+end

data/test/activerecord_count_queries.rb

@@ -0,0 +1,70 @@
+if ActiveRecord::VERSION::MAJOR < 3 || (ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR < 1)
+  # proudly stolen from ActiveRecord's test suite, with addition of BEGIN and COMMIT
+  ActiveRecord::Base.connection.class.class_eval do
+    IGNORED_SQL = [/^PRAGMA/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /SHOW FIELDS/, /^BEGIN$/, /^COMMIT$/]
+
+    def execute_with_query_record(sql, name = nil, &block)
+      $queries_executed ||= []
+      $queries_executed << sql unless IGNORED_SQL.any? { |r| sql =~ r }
+      execute_without_query_record(sql, name, &block)
+    end
+
+    alias_method_chain :execute, :query_record
+  end
+elsif ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR < 2
+  # this is from 3.1's test suite.  ugly.
+  class ActiveRecord::SQLCounter
+    cattr_accessor :ignored_sql
+    self.ignored_sql = [/^PRAGMA (?!(table_info))/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /^SHOW max_identifier_length/, /^BEGIN/, /^COMMIT/]
+
+    # FIXME: this needs to be refactored so specific database can add their own
+    # ignored SQL.  This ignored SQL is for Oracle.
+    ignored_sql.concat [/^select .*nextval/i, /^SAVEPOINT/, /^ROLLBACK TO/, /^\s*select .* from all_triggers/im]
+
+    def initialize
+      $queries_executed = []
+    end
+
+    def call(name, start, finish, message_id, values)
+      sql = values[:sql]
+
+      # FIXME: this seems bad. we should probably have a better way to indicate
+      # the query was cached
+      unless ['CACHE', 'SCHEMA'].include?(values[:name]) # we have altered this from the original, to exclude SCHEMA as well
+        # debugger if sql =~ /^PRAGMA table_info/ && Kernel.caller.any? {|i| i.include?('test_it_creates_named_class_methods_if_a_')}
+        $queries_executed << sql unless self.class.ignored_sql.any? { |r| sql =~ r }
+      end
+    end
+  end
+  ActiveSupport::Notifications.subscribe('sql.active_record', ActiveRecord::SQLCounter.new)
+else
+  # this is from 3.2's test suite.  ugly.
+  class ActiveRecord::SQLCounter
+    cattr_accessor :ignored_sql
+    self.ignored_sql = [/^PRAGMA (?!(table_info))/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /^SHOW max_identifier_length/, /^BEGIN/, /^COMMIT/]
+
+    # FIXME: this needs to be refactored so specific database can add their own
+    # ignored SQL.  This ignored SQL is for Oracle.
+    ignored_sql.concat [/^select .*nextval/i, /^SAVEPOINT/, /^ROLLBACK TO/, /^\s*select .* from all_triggers/im]
+
+    cattr_accessor :log
+    self.log = []
+
+    attr_reader :ignore
+
+    def initialize(ignore = self.class.ignored_sql)
+      @ignore   = ignore
+    end
+
+    def call(name, start, finish, message_id, values)
+      sql = values[:sql]
+
+      # FIXME: this seems bad. we should probably have a better way to indicate
+      # the query was cached
+      return if 'CACHE' == values[:name] || ignore.any? { |x| x =~ sql }
+      self.class.log << sql
+    end
+  end
+
+  ActiveSupport::Notifications.subscribe('sql.active_record', ActiveRecord::SQLCounter.new)
+end

data/test/columns_on_demand_test.rb

@@ -0,0 +1,273 @@
+require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'schema'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'activerecord_count_queries'))
+
+class Explicit < ActiveRecord::Base
+  columns_on_demand :file_data, :processing_log, :original_filename
+end
+
+class Implicit < ActiveRecord::Base
+  columns_on_demand
+end
+
+class Parent < ActiveRecord::Base
+  columns_on_demand
+  
+  has_many :children
+end
+
+class Child < ActiveRecord::Base
+  columns_on_demand
+  
+  belongs_to :parent
+end
+
+class ColumnsOnDemandTest < ActiveRecord::TestCase
+  def assert_not_loaded(record, attr_name)
+    assert !record.column_loaded?(attr_name.to_s)
+  end
+  
+  def assert_loaded(record, attr_name)
+    assert record.column_loaded?(attr_name.to_s)
+  end
+  
+  fixtures :all
+  self.use_transactional_fixtures = true
+  
+  test "it lists explicitly given columns for loading on demand" do
+    assert_equal ["file_data", "processing_log", "original_filename"], Explicit.columns_to_load_on_demand
+  end
+
+  test "it lists all :binary and :text columns for loading on demand if none are explicitly given" do
+    assert_equal ["file_data", "processing_log", "results"], Implicit.columns_to_load_on_demand
+  end
+  
+  test "it selects all the other columns for loading eagerly" do
+    assert_match /\W*id\W*, \W*results\W*, \W*processed_at\W*/, Explicit.default_select(false)
+    assert_match /\W*explicits\W*.results/, Explicit.default_select(true)
+    
+    assert_match /\W*id\W*, \W*original_filename\W*, \W*processed_at\W*/, Implicit.default_select(false)
+    assert_match /\W*implicits\W*.original_filename/, Implicit.default_select(true)
+  end
+  
+  test "it doesn't load the columns_to_load_on_demand straight away when finding the records" do
+    record = Implicit.find(:first)
+    assert_not_equal nil, record
+    assert_not_loaded record, "file_data"
+    assert_not_loaded record, "processing_log"
+
+    record = Implicit.find(:all).first
+    assert_not_equal nil, record
+    assert_not_loaded record, "file_data"
+    assert_not_loaded record, "processing_log"
+  end
+  
+  test "it loads the columns when accessed as an attribute" do
+    record = Implicit.find(:first)
+    assert_equal "This is the file data!", record.file_data
+    assert_equal "Processed 0 entries OK", record.results
+    assert_equal record.results.object_id, record.results.object_id # should not have to re-find
+
+    record = Implicit.find(:all).first
+    assert_not_equal nil, record.file_data
+  end
+
+  test "it loads the columns only once even if nil" do
+    record = Implicit.find(:first)
+    assert_not_loaded record, "file_data"
+    assert_equal "This is the file data!", record.file_data
+    assert_loaded record, "file_data"
+    Implicit.update_all(:file_data => nil)
+
+    record = Implicit.find(:first)
+    assert_not_loaded record, "file_data"
+    assert_equal nil, record.file_data
+    assert_loaded record, "file_data"
+    assert_no_queries do
+      assert_equal nil, record.file_data
+    end
+  end
+  
+  test "it loads the column when accessed using read_attribute" do
+    record = Implicit.find(:first)
+    assert_equal "This is the file data!", record.read_attribute(:file_data)
+    assert_equal "This is the file data!", record.read_attribute("file_data")
+    assert_equal "Processed 0 entries OK", record.read_attribute("results")
+    assert_equal record.read_attribute(:results).object_id, record.read_attribute("results").object_id # should not have to re-find
+  end
+  
+  test "it loads the column when accessed using read_attribute_before_type_cast" do
+    record = Implicit.find(:first)
+    assert_equal "This is the file data!", record.read_attribute_before_type_cast("file_data")
+    assert_equal "Processed 0 entries OK", record.read_attribute_before_type_cast("results")
+    # read_attribute_before_type_cast doesn't tolerate symbol arguments as read_attribute does
+  end
+  
+  test "it loads the column when generating #attributes" do
+    attributes = Implicit.find(:first).attributes
+    assert_equal "This is the file data!", attributes["file_data"]
+  end
+
+  test "loads all the columns in one query when generating #attributes" do
+    record = Implicit.find(:first)
+    assert_queries(1) do
+      attributes = record.attributes
+      assert_equal "This is the file data!", attributes["file_data"]
+      assert !attributes["processing_log"].blank?
+    end
+  end
+  
+  test "it loads the column when generating #to_json" do
+    ActiveRecord::Base.include_root_in_json = true
+    json = Implicit.find(:first).to_json
+    assert_equal "This is the file data!", ActiveSupport::JSON.decode(json)["implicit"]["file_data"]
+  end
+  
+  test "it loads the column for #clone" do
+    record = Implicit.find(:first).clone
+    assert_equal "This is the file data!", record.file_data
+
+    record = Implicit.find(:first).clone.tap(&:save!)
+    assert_equal "This is the file data!", Implicit.find(record.id).file_data
+  end
+  
+  test "it clears the column on reload, and can load it again" do
+    record = Implicit.find(:first)
+    old_object_id = record.file_data.object_id
+    Implicit.update_all(:file_data => "New file data")
+
+    record.reload
+
+    assert_not_loaded record, "file_data"
+    assert_equal "New file data", record.file_data
+  end
+  
+  test "it doesn't override custom :select finds" do
+    record = Implicit.find(:first, :select => "id, file_data")
+    klass = ActiveRecord.const_defined?(:MissingAttributeError) ? ActiveRecord::MissingAttributeError : ActiveModel::MissingAttributeError
+    assert_raise klass do
+      record.processed_at # explicitly not loaded, overriding default
+    end
+    assert_equal "This is the file data!", record.instance_variable_get("@attributes")["file_data"] # already loaded, overriding default
+  end
+  
+  test "it raises normal ActiveRecord::RecordNotFound if the record is deleted before the column load" do
+    record = Implicit.find(:first)
+    Implicit.delete_all
+    
+    assert_raise ActiveRecord::RecordNotFound do
+      record.file_data
+    end
+  end
+  
+  test "it doesn't raise on column access if the record is deleted after the column load" do
+    record = Implicit.find(:first)
+    record.file_data
+    Implicit.delete_all
+    
+    assert_equal "This is the file data!", record.file_data # check it doesn't raise
+  end
+  
+  test "it updates the select strings when columns are changed and the column information is reset" do
+    ActiveRecord::Schema.define(:version => 1) do
+      create_table :dummies, :force => true do |t|
+        t.string   :some_field
+        t.binary   :big_field
+      end
+    end
+
+    class Dummy < ActiveRecord::Base
+      columns_on_demand
+    end
+
+    assert_match /\W*id\W*, \W*some_field\W*/, Dummy.default_select(false)
+
+    ActiveRecord::Schema.define(:version => 2) do
+      create_table :dummies, :force => true do |t|
+        t.string   :some_field
+        t.binary   :big_field
+        t.string   :another_field
+      end
+    end
+
+    assert_match /\W*id\W*, \W*some_field\W*/, Dummy.default_select(false)
+    Dummy.reset_column_information
+    assert_match /\W*id\W*, \W*some_field\W*, \W*another_field\W*/, Dummy.default_select(false)
+  end
+  
+  test "it handles STI models" do
+    ActiveRecord::Schema.define(:version => 1) do
+      create_table :stis, :force => true do |t|
+        t.string   :type
+        t.string   :some_field
+        t.binary   :big_field
+      end
+    end
+
+    class Sti < ActiveRecord::Base
+      columns_on_demand
+    end
+    
+    class StiChild < Sti
+      columns_on_demand :some_field
+    end
+
+    assert_match /\W*id\W*, \W*type\W*, \W*some_field\W*/, Sti.default_select(false)
+    assert_match /\W*id\W*, \W*type\W*, \W*big_field\W*/,  StiChild.default_select(false)
+  end
+  
+  test "it works on child records loaded from associations" do
+    parent = parents(:some_parent)
+    child = parent.children.find(:first)
+    assert_not_loaded child, "test_data"
+    assert_equal "Some test data", child.test_data
+  end
+  
+  test "it works on parent records loaded from associations" do
+    child = children(:a_child_of_some_parent)
+    parent = child.parent
+    assert_not_loaded parent, "info"
+    assert_equal "Here's some info.", parent.info
+  end
+  
+  test "it doesn't break validates_presence_of" do
+    class ValidatedImplicit < ActiveRecord::Base
+      set_table_name "implicits"
+      columns_on_demand
+      validates_presence_of :original_filename, :file_data, :results
+    end
+    
+    assert !ValidatedImplicit.new(:original_filename => "test.txt").valid?
+    instance = ValidatedImplicit.create!(:original_filename => "test.txt", :file_data => "test file data", :results => "test results")
+    instance.update_attributes!({}) # file_data and results are already loaded
+    new_instance = ValidatedImplicit.find(instance.id)
+    new_instance.update_attributes!({}) # file_data and results aren't loaded yet, but will be loaded to validate
+  end
+  
+  test "it works with serialized columns" do
+    class Serializing < ActiveRecord::Base
+      columns_on_demand
+      serialize :data
+    end
+    
+    data = {:foo => '1', :bar => '2', :baz => '3'}
+    original_record = Serializing.create!(:data => data)
+    assert_equal data, original_record.data
+    
+    record = Serializing.find(:first)
+    assert_not_loaded record, "data"
+    assert_equal data, record.data
+    assert_equal false, record.data_changed?
+    assert_equal false, record.changed?
+    assert_equal data, record.data
+    
+    record.data = "replacement"
+    assert_equal true, record.data_changed?
+    assert_equal true, record.changed?
+    record.save!
+    
+    record = Serializing.find(:first)
+    assert_not_loaded record, "data"
+    assert_equal "replacement", record.data
+  end
+end

data/test/database.yml

@@ -0,0 +1,14 @@
+mysql:
+  adapter: mysql
+  database: columns_on_demand_test
+  username: root
+mysql2:
+  adapter: mysql2
+  database: columns_on_demand_test
+  username: root
+postgresql:
+  adapter: postgresql
+  database: columns_on_demand_test
+sqlite3:
+  adapter: sqlite3
+  database: /tmp/columns_on_demand_test.db

data/test/fixtures/children.yml

@@ -0,0 +1,3 @@
+a_child_of_some_parent:
+  parent: some_parent
+  test_data: Some test data

data/test/fixtures/implicits.yml

@@ -0,0 +1,10 @@
+first_file:
+  original_filename: somefile.txt
+  file_data: This is the file data!
+  processing_log: "Processing somefile.txt\n
+    Processing header\n
+    Processed header\n
+    Processing footer\n
+    Processed footer\n"
+  results: Processed 0 entries OK
+  processed_at: 2008-12-23 21:38:10

data/test/fixtures/parents.yml

@@ -0,0 +1,2 @@
+some_parent:
+  info: Here's some info.

data/test/schema.rb

@@ -0,0 +1,30 @@
+ActiveRecord::Schema.define(:version => 0) do
+  create_table :explicits, :force => true do |t|
+    t.string   :original_filename, :null => false
+    t.binary   :file_data
+    t.text     :processing_log
+    t.text     :results
+    t.datetime :processed_at
+  end
+
+  create_table :implicits, :force => true do |t|
+    t.string   :original_filename, :null => false
+    t.binary   :file_data
+    t.text     :processing_log
+    t.text     :results
+    t.datetime :processed_at
+  end
+  
+  create_table :parents, :force => true do |t|
+    t.text     :info
+  end
+  
+  create_table :children, :force => true do |t|
+    t.integer  :parent_id, :null => false
+    t.text     :test_data
+  end
+  
+  create_table :serializings, :force => true do |t|
+    t.binary   :data
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,33 @@
+RAILS_ROOT = File.expand_path("../../..")
+if File.exist?("#{RAILS_ROOT}/config/boot.rb")
+  require "#{RAILS_ROOT}/config/boot.rb"
+else
+  require 'rubygems'
+end
+
+puts "Rails: #{ENV['RAILS_VERSION'] || 'default'}"
+gem 'activesupport', ENV['RAILS_VERSION']
+gem 'activerecord',  ENV['RAILS_VERSION']
+
+require 'test/unit'
+require 'active_support'
+require 'active_support/test_case'
+require 'active_record'
+require 'active_record/fixtures'
+
+begin
+  require 'ruby-debug'
+  Debugger.start
+rescue LoadError
+  # ruby-debug not installed, no debugging for you
+end
+
+ActiveRecord::Base.configurations = YAML::load(IO.read(File.join(File.dirname(__FILE__), "database.yml")))
+configuration = ActiveRecord::Base.configurations[ENV['RAILS_ENV']]
+raise "use RAILS_ENV=#{ActiveRecord::Base.configurations.keys.sort.join '/'} to test this plugin" unless configuration
+ActiveRecord::Base.establish_connection configuration
+
+ActiveSupport::TestCase.send(:include, ActiveRecord::TestFixtures) if ActiveRecord.const_defined?('TestFixtures')
+ActiveSupport::TestCase.fixture_path = File.join(File.dirname(__FILE__), "fixtures")
+
+require File.expand_path(File.join(File.dirname(__FILE__), '../init')) # load columns_on_demand

metadata

@@ -0,0 +1,194 @@
+--- !ruby/object:Gem::Specification 
+name: columns_on_demand
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: 
+  segments: 
+  - 2
+  - 0
+  - 0
+  version: 2.0.0
+platform: ruby
+authors: 
+- Will Bryant
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-08-18 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activerecord
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: mysql
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: mysql2
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+description: |
+  Lazily loads large columns on demand.
+  
+  By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
+  of specific columns to load on demand can be given.
+  
+  This is useful to reduce the memory taken by Rails when loading a number of records
+  that have large columns if those particular columns are actually not required most
+  of the time.  In this situation it can also greatly reduce the database query time
+  because loading large BLOB/TEXT columns generally means seeking to other database
+  pages since they are not stored wholly in the record's page itself.
+  
+  Although this plugin is mainly used for BLOB and TEXT columns, it will actually
+  work on all types - and is just as useful for large string fields etc.
+  
+  
+  Compatibility
+  =============
+  
+  Supports mysql, mysql2, postgresql, and sqlite3.
+  
+  Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8.  Note that
+  3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
+
+email: will.bryant@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- MIT-LICENSE
+- README
+- Rakefile
+- columns_on_demand.gemspec
+- init.rb
+- lib/columns_on_demand.rb
+- lib/columns_on_demand/version.rb
+- test/activerecord_count_queries.rb
+- test/columns_on_demand_test.rb
+- test/database.yml
+- test/fixtures/children.yml
+- test/fixtures/implicits.yml
+- test/fixtures/parents.yml
+- test/schema.rb
+- test/test_helper.rb
+homepage: http://github.com/willbryant/columns_on_demand
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Lazily loads large columns on demand.
+test_files: 
+- test/activerecord_count_queries.rb
+- test/columns_on_demand_test.rb
+- test/database.yml
+- test/fixtures/children.yml
+- test/fixtures/implicits.yml
+- test/fixtures/parents.yml
+- test/schema.rb
+- test/test_helper.rb