columns_on_demand 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/Gemfile +15 -0
- data/MIT-LICENSE +20 -0
- data/README +45 -0
- data/Rakefile +16 -0
- data/columns_on_demand.gemspec +48 -0
- data/init.rb +2 -0
- data/lib/columns_on_demand.rb +143 -0
- data/lib/columns_on_demand/version.rb +3 -0
- data/test/activerecord_count_queries.rb +70 -0
- data/test/columns_on_demand_test.rb +273 -0
- data/test/database.yml +14 -0
- data/test/fixtures/children.yml +3 -0
- data/test/fixtures/implicits.yml +10 -0
- data/test/fixtures/parents.yml +2 -0
- data/test/schema.rb +30 -0
- data/test/test_helper.rb +33 -0
- metadata +194 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# Declare your gem's dependencies in transaction_isolation_level.gemspec.
|
4
|
+
# Bundler will treat runtime dependencies like base dependencies, and
|
5
|
+
# development dependencies will be added by default to the :development group.
|
6
|
+
gemspec
|
7
|
+
|
8
|
+
# Declare any dependencies that are still in development here instead of in
|
9
|
+
# your gemspec. These might include edge Rails or gems from your path or
|
10
|
+
# Git. Remember to move these dependencies to your gemspec before releasing
|
11
|
+
# your gem to rubygems.org.
|
12
|
+
|
13
|
+
# To use debugger
|
14
|
+
# gem 'ruby-debug19', :require => 'ruby-debug'
|
15
|
+
# gem 'ruby-debug'
|
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2008-2009 Will Bryant, Sekuda Ltd
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
ColumnsOnDemand
|
2
|
+
===============
|
3
|
+
|
4
|
+
Lazily loads large columns on demand.
|
5
|
+
|
6
|
+
By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
|
7
|
+
of specific columns to load on demand can be given.
|
8
|
+
|
9
|
+
This is useful to reduce the memory taken by Rails when loading a number of records
|
10
|
+
that have large columns if those particular columns are actually not required most
|
11
|
+
of the time. In this situation it can also greatly reduce the database query time
|
12
|
+
because loading large BLOB/TEXT columns generally means seeking to other database
|
13
|
+
pages since they are not stored wholly in the record's page itself.
|
14
|
+
|
15
|
+
Although this plugin is mainly used for BLOB and TEXT columns, it will actually
|
16
|
+
work on all types - and is just as useful for large string fields etc.
|
17
|
+
|
18
|
+
|
19
|
+
Compatibility
|
20
|
+
=============
|
21
|
+
|
22
|
+
Supports mysql, mysql2, postgresql, and sqlite3.
|
23
|
+
|
24
|
+
Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8. Note that
|
25
|
+
3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
|
26
|
+
|
27
|
+
|
28
|
+
Example
|
29
|
+
=======
|
30
|
+
|
31
|
+
# Example.all will exclude the file_data and processing_log columns from the
|
32
|
+
# SELECT query, and example.file_data and example.processing_log will load & cache
|
33
|
+
# that individual column value for the record instance.
|
34
|
+
class Example
|
35
|
+
columns_on_demand :file_data, :processing_log
|
36
|
+
end
|
37
|
+
|
38
|
+
# Scans the 'examples' table columns and registers all TEXT (:text) and BLOB (:binary)
|
39
|
+
# columns for loading on demand.
|
40
|
+
class Example
|
41
|
+
columns_on_demand
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
Copyright (c) 2008-2012 Will Bryant, Sekuda Ltd, released under the MIT license
|
data/Rakefile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler/gem_tasks'
|
4
|
+
require 'rake'
|
5
|
+
require 'rake/testtask'
|
6
|
+
|
7
|
+
desc 'Default: run unit tests.'
|
8
|
+
task :default => :test
|
9
|
+
|
10
|
+
desc 'Test the columns_on_demand plugin.'
|
11
|
+
Rake::TestTask.new(:test) do |t|
|
12
|
+
t.libs << 'lib'
|
13
|
+
t.libs << 'test'
|
14
|
+
t.pattern = 'test/*_test.rb'
|
15
|
+
t.verbose = true
|
16
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/columns_on_demand/version', __FILE__)
|
3
|
+
|
4
|
+
spec = Gem::Specification.new do |gem|
|
5
|
+
gem.name = 'columns_on_demand'
|
6
|
+
gem.version = ColumnsOnDemand::VERSION
|
7
|
+
gem.summary = "Lazily loads large columns on demand."
|
8
|
+
gem.description = <<-EOF
|
9
|
+
Lazily loads large columns on demand.
|
10
|
+
|
11
|
+
By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
|
12
|
+
of specific columns to load on demand can be given.
|
13
|
+
|
14
|
+
This is useful to reduce the memory taken by Rails when loading a number of records
|
15
|
+
that have large columns if those particular columns are actually not required most
|
16
|
+
of the time. In this situation it can also greatly reduce the database query time
|
17
|
+
because loading large BLOB/TEXT columns generally means seeking to other database
|
18
|
+
pages since they are not stored wholly in the record's page itself.
|
19
|
+
|
20
|
+
Although this plugin is mainly used for BLOB and TEXT columns, it will actually
|
21
|
+
work on all types - and is just as useful for large string fields etc.
|
22
|
+
|
23
|
+
|
24
|
+
Compatibility
|
25
|
+
=============
|
26
|
+
|
27
|
+
Supports mysql, mysql2, postgresql, and sqlite3.
|
28
|
+
|
29
|
+
Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8. Note that
|
30
|
+
3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
|
31
|
+
EOF
|
32
|
+
gem.has_rdoc = false
|
33
|
+
gem.author = "Will Bryant"
|
34
|
+
gem.email = "will.bryant@gmail.com"
|
35
|
+
gem.homepage = "http://github.com/willbryant/columns_on_demand"
|
36
|
+
|
37
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
38
|
+
gem.files = `git ls-files`.split("\n")
|
39
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
40
|
+
gem.require_path = "lib"
|
41
|
+
|
42
|
+
gem.add_dependency "activerecord"
|
43
|
+
gem.add_development_dependency "rake"
|
44
|
+
gem.add_development_dependency "mysql"
|
45
|
+
gem.add_development_dependency "mysql2"
|
46
|
+
gem.add_development_dependency "pg"
|
47
|
+
gem.add_development_dependency "sqlite3"
|
48
|
+
end
|
data/init.rb
ADDED
@@ -0,0 +1,143 @@
|
|
1
|
+
module ColumnsOnDemand
|
2
|
+
module BaseMethods
|
3
|
+
def columns_on_demand(*columns_to_load_on_demand)
|
4
|
+
class_attribute :columns_to_load_on_demand, :instance_writer => false
|
5
|
+
self.columns_to_load_on_demand = columns_to_load_on_demand.empty? ? blob_and_text_columns : columns_to_load_on_demand.collect(&:to_s)
|
6
|
+
|
7
|
+
extend ClassMethods
|
8
|
+
include InstanceMethods
|
9
|
+
|
10
|
+
if ActiveRecord::VERSION::MAJOR > 2
|
11
|
+
relation # set up @relation
|
12
|
+
class << @relation # but always modify @relation, not the temporary returned by .relation if there's a where(type condition)
|
13
|
+
def build_select_with_columns_on_demand(arel, selects)
|
14
|
+
unless selects.empty?
|
15
|
+
build_select_without_columns_on_demand(arel, selects)
|
16
|
+
else
|
17
|
+
arel.project(Arel::SqlLiteral.new(@klass.default_select(true)))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
alias_method_chain :build_select, :columns_on_demand
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class <<self
|
25
|
+
unless ActiveRecord.const_defined?(:AttributeMethods) &&
|
26
|
+
ActiveRecord::AttributeMethods::const_defined?(:Serialization) &&
|
27
|
+
ActiveRecord::AttributeMethods::Serialization::Attribute
|
28
|
+
alias_method_chain :define_read_method_for_serialized_attribute, :columns_on_demand
|
29
|
+
end
|
30
|
+
alias_method_chain :reset_column_information, :columns_on_demand
|
31
|
+
end
|
32
|
+
alias_method_chain :attributes, :columns_on_demand
|
33
|
+
alias_method_chain :attribute_names, :columns_on_demand
|
34
|
+
alias_method_chain :read_attribute, :columns_on_demand
|
35
|
+
alias_method_chain :read_attribute_before_type_cast, :columns_on_demand
|
36
|
+
alias_method_chain :missing_attribute, :columns_on_demand
|
37
|
+
alias_method_chain :reload, :columns_on_demand
|
38
|
+
end
|
39
|
+
|
40
|
+
def reset_column_information_with_columns_on_demand
|
41
|
+
@columns_to_select = nil
|
42
|
+
reset_column_information_without_columns_on_demand
|
43
|
+
end
|
44
|
+
|
45
|
+
def define_read_method_for_serialized_attribute_with_columns_on_demand(attr_name)
|
46
|
+
define_read_method_for_serialized_attribute_without_columns_on_demand(attr_name)
|
47
|
+
scope = method_defined?(:generated_attribute_methods) ? generated_attribute_methods : self
|
48
|
+
scope.module_eval("def #{attr_name}_with_columns_on_demand; ensure_loaded('#{attr_name}'); #{attr_name}_without_columns_on_demand; end; alias_method_chain :#{attr_name}, :columns_on_demand", __FILE__, __LINE__)
|
49
|
+
end
|
50
|
+
|
51
|
+
def blob_and_text_columns
|
52
|
+
columns.inject([]) do |blob_and_text_columns, column|
|
53
|
+
blob_and_text_columns << column.name if column.type == :binary || column.type == :text
|
54
|
+
blob_and_text_columns
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
module ClassMethods
|
60
|
+
# this is the method API as called by ActiveRecord 2.x. we also call it ourselves above in our ActiveRecord 3 extensions.
|
61
|
+
def default_select(qualified)
|
62
|
+
@columns_to_select ||= (columns.collect(&:name) - columns_to_load_on_demand).collect {|attr_name| connection.quote_column_name(attr_name)}
|
63
|
+
if qualified
|
64
|
+
quoted_table_name + '.' + @columns_to_select.join(", #{quoted_table_name}.")
|
65
|
+
else
|
66
|
+
@columns_to_select.join(", ")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
module InstanceMethods
|
72
|
+
def columns_loaded
|
73
|
+
@columns_loaded ||= Set.new
|
74
|
+
end
|
75
|
+
|
76
|
+
def column_loaded?(attr_name)
|
77
|
+
!columns_to_load_on_demand.include?(attr_name) || !@attributes[attr_name].nil? || new_record? || columns_loaded.include?(attr_name)
|
78
|
+
end
|
79
|
+
|
80
|
+
def attributes_with_columns_on_demand
|
81
|
+
load_attributes(*columns_to_load_on_demand.reject {|attr_name| column_loaded?(attr_name)})
|
82
|
+
attributes_without_columns_on_demand
|
83
|
+
end
|
84
|
+
|
85
|
+
def attribute_names_with_columns_on_demand
|
86
|
+
(attribute_names_without_columns_on_demand + columns_to_load_on_demand).uniq.sort
|
87
|
+
end
|
88
|
+
|
89
|
+
def load_attributes(*attr_names)
|
90
|
+
return if attr_names.blank?
|
91
|
+
values = connection.select_rows(
|
92
|
+
"SELECT #{attr_names.collect {|attr_name| connection.quote_column_name(attr_name)}.join(", ")}" +
|
93
|
+
" FROM #{self.class.quoted_table_name}" +
|
94
|
+
" WHERE #{connection.quote_column_name(self.class.primary_key)} = #{quote_value(id, self.class.columns_hash[self.class.primary_key])}")
|
95
|
+
row = values.first || raise(ActiveRecord::RecordNotFound, "Couldn't find #{self.class.name} with ID=#{id}")
|
96
|
+
attr_names.each_with_index do |attr_name, i|
|
97
|
+
columns_loaded << attr_name
|
98
|
+
@attributes[attr_name] = row[i]
|
99
|
+
|
100
|
+
if coder = self.class.serialized_attributes[attr_name]
|
101
|
+
if ActiveRecord.const_defined?(:AttributeMethods) &&
|
102
|
+
ActiveRecord::AttributeMethods::const_defined?(:Serialization) &&
|
103
|
+
ActiveRecord::AttributeMethods::Serialization::Attribute
|
104
|
+
# in 3.2 @attributes has a special Attribute struct to help cache both serialized and unserialized forms
|
105
|
+
@attributes[attr_name] = ActiveRecord::AttributeMethods::Serialization::Attribute.new(coder, @attributes[attr_name], :serialized)
|
106
|
+
elsif ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR == 1
|
107
|
+
# in 2.3 an 3.0, @attributes has the serialized form; from 3.1 it has the deserialized form
|
108
|
+
@attributes[attr_name] = coder.load @attributes[attr_name]
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def ensure_loaded(attr_name)
|
115
|
+
load_attributes(attr_name.to_s) unless column_loaded?(attr_name.to_s)
|
116
|
+
end
|
117
|
+
|
118
|
+
def read_attribute_with_columns_on_demand(attr_name)
|
119
|
+
ensure_loaded(attr_name)
|
120
|
+
read_attribute_without_columns_on_demand(attr_name)
|
121
|
+
end
|
122
|
+
|
123
|
+
def read_attribute_before_type_cast_with_columns_on_demand(attr_name)
|
124
|
+
ensure_loaded(attr_name)
|
125
|
+
read_attribute_before_type_cast_without_columns_on_demand(attr_name)
|
126
|
+
end
|
127
|
+
|
128
|
+
def missing_attribute_with_columns_on_demand(attr_name, *args)
|
129
|
+
if columns_to_load_on_demand.include?(attr_name)
|
130
|
+
load_attributes(attr_name)
|
131
|
+
else
|
132
|
+
missing_attribute_without_columns_on_demand(attr_name, *args)
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
def reload_with_columns_on_demand(*args)
|
137
|
+
reload_without_columns_on_demand(*args).tap do
|
138
|
+
columns_loaded.clear
|
139
|
+
columns_to_load_on_demand.each {|attr_name| @attributes.delete(attr_name)}
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
if ActiveRecord::VERSION::MAJOR < 3 || (ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR < 1)
|
2
|
+
# proudly stolen from ActiveRecord's test suite, with addition of BEGIN and COMMIT
|
3
|
+
ActiveRecord::Base.connection.class.class_eval do
|
4
|
+
IGNORED_SQL = [/^PRAGMA/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /SHOW FIELDS/, /^BEGIN$/, /^COMMIT$/]
|
5
|
+
|
6
|
+
def execute_with_query_record(sql, name = nil, &block)
|
7
|
+
$queries_executed ||= []
|
8
|
+
$queries_executed << sql unless IGNORED_SQL.any? { |r| sql =~ r }
|
9
|
+
execute_without_query_record(sql, name, &block)
|
10
|
+
end
|
11
|
+
|
12
|
+
alias_method_chain :execute, :query_record
|
13
|
+
end
|
14
|
+
elsif ActiveRecord::VERSION::MAJOR == 3 && ActiveRecord::VERSION::MINOR < 2
|
15
|
+
# this is from 3.1's test suite. ugly.
|
16
|
+
class ActiveRecord::SQLCounter
|
17
|
+
cattr_accessor :ignored_sql
|
18
|
+
self.ignored_sql = [/^PRAGMA (?!(table_info))/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /^SHOW max_identifier_length/, /^BEGIN/, /^COMMIT/]
|
19
|
+
|
20
|
+
# FIXME: this needs to be refactored so specific database can add their own
|
21
|
+
# ignored SQL. This ignored SQL is for Oracle.
|
22
|
+
ignored_sql.concat [/^select .*nextval/i, /^SAVEPOINT/, /^ROLLBACK TO/, /^\s*select .* from all_triggers/im]
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
$queries_executed = []
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(name, start, finish, message_id, values)
|
29
|
+
sql = values[:sql]
|
30
|
+
|
31
|
+
# FIXME: this seems bad. we should probably have a better way to indicate
|
32
|
+
# the query was cached
|
33
|
+
unless ['CACHE', 'SCHEMA'].include?(values[:name]) # we have altered this from the original, to exclude SCHEMA as well
|
34
|
+
# debugger if sql =~ /^PRAGMA table_info/ && Kernel.caller.any? {|i| i.include?('test_it_creates_named_class_methods_if_a_')}
|
35
|
+
$queries_executed << sql unless self.class.ignored_sql.any? { |r| sql =~ r }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
ActiveSupport::Notifications.subscribe('sql.active_record', ActiveRecord::SQLCounter.new)
|
40
|
+
else
|
41
|
+
# this is from 3.2's test suite. ugly.
|
42
|
+
class ActiveRecord::SQLCounter
|
43
|
+
cattr_accessor :ignored_sql
|
44
|
+
self.ignored_sql = [/^PRAGMA (?!(table_info))/, /^SELECT currval/, /^SELECT CAST/, /^SELECT @@IDENTITY/, /^SELECT @@ROWCOUNT/, /^SAVEPOINT/, /^ROLLBACK TO SAVEPOINT/, /^RELEASE SAVEPOINT/, /^SHOW max_identifier_length/, /^BEGIN/, /^COMMIT/]
|
45
|
+
|
46
|
+
# FIXME: this needs to be refactored so specific database can add their own
|
47
|
+
# ignored SQL. This ignored SQL is for Oracle.
|
48
|
+
ignored_sql.concat [/^select .*nextval/i, /^SAVEPOINT/, /^ROLLBACK TO/, /^\s*select .* from all_triggers/im]
|
49
|
+
|
50
|
+
cattr_accessor :log
|
51
|
+
self.log = []
|
52
|
+
|
53
|
+
attr_reader :ignore
|
54
|
+
|
55
|
+
def initialize(ignore = self.class.ignored_sql)
|
56
|
+
@ignore = ignore
|
57
|
+
end
|
58
|
+
|
59
|
+
def call(name, start, finish, message_id, values)
|
60
|
+
sql = values[:sql]
|
61
|
+
|
62
|
+
# FIXME: this seems bad. we should probably have a better way to indicate
|
63
|
+
# the query was cached
|
64
|
+
return if 'CACHE' == values[:name] || ignore.any? { |x| x =~ sql }
|
65
|
+
self.class.log << sql
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
ActiveSupport::Notifications.subscribe('sql.active_record', ActiveRecord::SQLCounter.new)
|
70
|
+
end
|
@@ -0,0 +1,273 @@
|
|
1
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper'))
|
2
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'schema'))
|
3
|
+
require File.expand_path(File.join(File.dirname(__FILE__), 'activerecord_count_queries'))
|
4
|
+
|
5
|
+
class Explicit < ActiveRecord::Base
|
6
|
+
columns_on_demand :file_data, :processing_log, :original_filename
|
7
|
+
end
|
8
|
+
|
9
|
+
class Implicit < ActiveRecord::Base
|
10
|
+
columns_on_demand
|
11
|
+
end
|
12
|
+
|
13
|
+
class Parent < ActiveRecord::Base
|
14
|
+
columns_on_demand
|
15
|
+
|
16
|
+
has_many :children
|
17
|
+
end
|
18
|
+
|
19
|
+
class Child < ActiveRecord::Base
|
20
|
+
columns_on_demand
|
21
|
+
|
22
|
+
belongs_to :parent
|
23
|
+
end
|
24
|
+
|
25
|
+
class ColumnsOnDemandTest < ActiveRecord::TestCase
|
26
|
+
def assert_not_loaded(record, attr_name)
|
27
|
+
assert !record.column_loaded?(attr_name.to_s)
|
28
|
+
end
|
29
|
+
|
30
|
+
def assert_loaded(record, attr_name)
|
31
|
+
assert record.column_loaded?(attr_name.to_s)
|
32
|
+
end
|
33
|
+
|
34
|
+
fixtures :all
|
35
|
+
self.use_transactional_fixtures = true
|
36
|
+
|
37
|
+
test "it lists explicitly given columns for loading on demand" do
|
38
|
+
assert_equal ["file_data", "processing_log", "original_filename"], Explicit.columns_to_load_on_demand
|
39
|
+
end
|
40
|
+
|
41
|
+
test "it lists all :binary and :text columns for loading on demand if none are explicitly given" do
|
42
|
+
assert_equal ["file_data", "processing_log", "results"], Implicit.columns_to_load_on_demand
|
43
|
+
end
|
44
|
+
|
45
|
+
test "it selects all the other columns for loading eagerly" do
|
46
|
+
assert_match /\W*id\W*, \W*results\W*, \W*processed_at\W*/, Explicit.default_select(false)
|
47
|
+
assert_match /\W*explicits\W*.results/, Explicit.default_select(true)
|
48
|
+
|
49
|
+
assert_match /\W*id\W*, \W*original_filename\W*, \W*processed_at\W*/, Implicit.default_select(false)
|
50
|
+
assert_match /\W*implicits\W*.original_filename/, Implicit.default_select(true)
|
51
|
+
end
|
52
|
+
|
53
|
+
test "it doesn't load the columns_to_load_on_demand straight away when finding the records" do
|
54
|
+
record = Implicit.find(:first)
|
55
|
+
assert_not_equal nil, record
|
56
|
+
assert_not_loaded record, "file_data"
|
57
|
+
assert_not_loaded record, "processing_log"
|
58
|
+
|
59
|
+
record = Implicit.find(:all).first
|
60
|
+
assert_not_equal nil, record
|
61
|
+
assert_not_loaded record, "file_data"
|
62
|
+
assert_not_loaded record, "processing_log"
|
63
|
+
end
|
64
|
+
|
65
|
+
test "it loads the columns when accessed as an attribute" do
|
66
|
+
record = Implicit.find(:first)
|
67
|
+
assert_equal "This is the file data!", record.file_data
|
68
|
+
assert_equal "Processed 0 entries OK", record.results
|
69
|
+
assert_equal record.results.object_id, record.results.object_id # should not have to re-find
|
70
|
+
|
71
|
+
record = Implicit.find(:all).first
|
72
|
+
assert_not_equal nil, record.file_data
|
73
|
+
end
|
74
|
+
|
75
|
+
test "it loads the columns only once even if nil" do
|
76
|
+
record = Implicit.find(:first)
|
77
|
+
assert_not_loaded record, "file_data"
|
78
|
+
assert_equal "This is the file data!", record.file_data
|
79
|
+
assert_loaded record, "file_data"
|
80
|
+
Implicit.update_all(:file_data => nil)
|
81
|
+
|
82
|
+
record = Implicit.find(:first)
|
83
|
+
assert_not_loaded record, "file_data"
|
84
|
+
assert_equal nil, record.file_data
|
85
|
+
assert_loaded record, "file_data"
|
86
|
+
assert_no_queries do
|
87
|
+
assert_equal nil, record.file_data
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
test "it loads the column when accessed using read_attribute" do
|
92
|
+
record = Implicit.find(:first)
|
93
|
+
assert_equal "This is the file data!", record.read_attribute(:file_data)
|
94
|
+
assert_equal "This is the file data!", record.read_attribute("file_data")
|
95
|
+
assert_equal "Processed 0 entries OK", record.read_attribute("results")
|
96
|
+
assert_equal record.read_attribute(:results).object_id, record.read_attribute("results").object_id # should not have to re-find
|
97
|
+
end
|
98
|
+
|
99
|
+
test "it loads the column when accessed using read_attribute_before_type_cast" do
|
100
|
+
record = Implicit.find(:first)
|
101
|
+
assert_equal "This is the file data!", record.read_attribute_before_type_cast("file_data")
|
102
|
+
assert_equal "Processed 0 entries OK", record.read_attribute_before_type_cast("results")
|
103
|
+
# read_attribute_before_type_cast doesn't tolerate symbol arguments as read_attribute does
|
104
|
+
end
|
105
|
+
|
106
|
+
test "it loads the column when generating #attributes" do
|
107
|
+
attributes = Implicit.find(:first).attributes
|
108
|
+
assert_equal "This is the file data!", attributes["file_data"]
|
109
|
+
end
|
110
|
+
|
111
|
+
test "loads all the columns in one query when generating #attributes" do
|
112
|
+
record = Implicit.find(:first)
|
113
|
+
assert_queries(1) do
|
114
|
+
attributes = record.attributes
|
115
|
+
assert_equal "This is the file data!", attributes["file_data"]
|
116
|
+
assert !attributes["processing_log"].blank?
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
test "it loads the column when generating #to_json" do
|
121
|
+
ActiveRecord::Base.include_root_in_json = true
|
122
|
+
json = Implicit.find(:first).to_json
|
123
|
+
assert_equal "This is the file data!", ActiveSupport::JSON.decode(json)["implicit"]["file_data"]
|
124
|
+
end
|
125
|
+
|
126
|
+
test "it loads the column for #clone" do
|
127
|
+
record = Implicit.find(:first).clone
|
128
|
+
assert_equal "This is the file data!", record.file_data
|
129
|
+
|
130
|
+
record = Implicit.find(:first).clone.tap(&:save!)
|
131
|
+
assert_equal "This is the file data!", Implicit.find(record.id).file_data
|
132
|
+
end
|
133
|
+
|
134
|
+
test "it clears the column on reload, and can load it again" do
|
135
|
+
record = Implicit.find(:first)
|
136
|
+
old_object_id = record.file_data.object_id
|
137
|
+
Implicit.update_all(:file_data => "New file data")
|
138
|
+
|
139
|
+
record.reload
|
140
|
+
|
141
|
+
assert_not_loaded record, "file_data"
|
142
|
+
assert_equal "New file data", record.file_data
|
143
|
+
end
|
144
|
+
|
145
|
+
test "it doesn't override custom :select finds" do
|
146
|
+
record = Implicit.find(:first, :select => "id, file_data")
|
147
|
+
klass = ActiveRecord.const_defined?(:MissingAttributeError) ? ActiveRecord::MissingAttributeError : ActiveModel::MissingAttributeError
|
148
|
+
assert_raise klass do
|
149
|
+
record.processed_at # explicitly not loaded, overriding default
|
150
|
+
end
|
151
|
+
assert_equal "This is the file data!", record.instance_variable_get("@attributes")["file_data"] # already loaded, overriding default
|
152
|
+
end
|
153
|
+
|
154
|
+
test "it raises normal ActiveRecord::RecordNotFound if the record is deleted before the column load" do
|
155
|
+
record = Implicit.find(:first)
|
156
|
+
Implicit.delete_all
|
157
|
+
|
158
|
+
assert_raise ActiveRecord::RecordNotFound do
|
159
|
+
record.file_data
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
test "it doesn't raise on column access if the record is deleted after the column load" do
|
164
|
+
record = Implicit.find(:first)
|
165
|
+
record.file_data
|
166
|
+
Implicit.delete_all
|
167
|
+
|
168
|
+
assert_equal "This is the file data!", record.file_data # check it doesn't raise
|
169
|
+
end
|
170
|
+
|
171
|
+
test "it updates the select strings when columns are changed and the column information is reset" do
|
172
|
+
ActiveRecord::Schema.define(:version => 1) do
|
173
|
+
create_table :dummies, :force => true do |t|
|
174
|
+
t.string :some_field
|
175
|
+
t.binary :big_field
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
class Dummy < ActiveRecord::Base
|
180
|
+
columns_on_demand
|
181
|
+
end
|
182
|
+
|
183
|
+
assert_match /\W*id\W*, \W*some_field\W*/, Dummy.default_select(false)
|
184
|
+
|
185
|
+
ActiveRecord::Schema.define(:version => 2) do
|
186
|
+
create_table :dummies, :force => true do |t|
|
187
|
+
t.string :some_field
|
188
|
+
t.binary :big_field
|
189
|
+
t.string :another_field
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
assert_match /\W*id\W*, \W*some_field\W*/, Dummy.default_select(false)
|
194
|
+
Dummy.reset_column_information
|
195
|
+
assert_match /\W*id\W*, \W*some_field\W*, \W*another_field\W*/, Dummy.default_select(false)
|
196
|
+
end
|
197
|
+
|
198
|
+
test "it handles STI models" do
|
199
|
+
ActiveRecord::Schema.define(:version => 1) do
|
200
|
+
create_table :stis, :force => true do |t|
|
201
|
+
t.string :type
|
202
|
+
t.string :some_field
|
203
|
+
t.binary :big_field
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
class Sti < ActiveRecord::Base
|
208
|
+
columns_on_demand
|
209
|
+
end
|
210
|
+
|
211
|
+
class StiChild < Sti
|
212
|
+
columns_on_demand :some_field
|
213
|
+
end
|
214
|
+
|
215
|
+
assert_match /\W*id\W*, \W*type\W*, \W*some_field\W*/, Sti.default_select(false)
|
216
|
+
assert_match /\W*id\W*, \W*type\W*, \W*big_field\W*/, StiChild.default_select(false)
|
217
|
+
end
|
218
|
+
|
219
|
+
test "it works on child records loaded from associations" do
|
220
|
+
parent = parents(:some_parent)
|
221
|
+
child = parent.children.find(:first)
|
222
|
+
assert_not_loaded child, "test_data"
|
223
|
+
assert_equal "Some test data", child.test_data
|
224
|
+
end
|
225
|
+
|
226
|
+
test "it works on parent records loaded from associations" do
|
227
|
+
child = children(:a_child_of_some_parent)
|
228
|
+
parent = child.parent
|
229
|
+
assert_not_loaded parent, "info"
|
230
|
+
assert_equal "Here's some info.", parent.info
|
231
|
+
end
|
232
|
+
|
233
|
+
test "it doesn't break validates_presence_of" do
|
234
|
+
class ValidatedImplicit < ActiveRecord::Base
|
235
|
+
set_table_name "implicits"
|
236
|
+
columns_on_demand
|
237
|
+
validates_presence_of :original_filename, :file_data, :results
|
238
|
+
end
|
239
|
+
|
240
|
+
assert !ValidatedImplicit.new(:original_filename => "test.txt").valid?
|
241
|
+
instance = ValidatedImplicit.create!(:original_filename => "test.txt", :file_data => "test file data", :results => "test results")
|
242
|
+
instance.update_attributes!({}) # file_data and results are already loaded
|
243
|
+
new_instance = ValidatedImplicit.find(instance.id)
|
244
|
+
new_instance.update_attributes!({}) # file_data and results aren't loaded yet, but will be loaded to validate
|
245
|
+
end
|
246
|
+
|
247
|
+
test "it works with serialized columns" do
|
248
|
+
class Serializing < ActiveRecord::Base
|
249
|
+
columns_on_demand
|
250
|
+
serialize :data
|
251
|
+
end
|
252
|
+
|
253
|
+
data = {:foo => '1', :bar => '2', :baz => '3'}
|
254
|
+
original_record = Serializing.create!(:data => data)
|
255
|
+
assert_equal data, original_record.data
|
256
|
+
|
257
|
+
record = Serializing.find(:first)
|
258
|
+
assert_not_loaded record, "data"
|
259
|
+
assert_equal data, record.data
|
260
|
+
assert_equal false, record.data_changed?
|
261
|
+
assert_equal false, record.changed?
|
262
|
+
assert_equal data, record.data
|
263
|
+
|
264
|
+
record.data = "replacement"
|
265
|
+
assert_equal true, record.data_changed?
|
266
|
+
assert_equal true, record.changed?
|
267
|
+
record.save!
|
268
|
+
|
269
|
+
record = Serializing.find(:first)
|
270
|
+
assert_not_loaded record, "data"
|
271
|
+
assert_equal "replacement", record.data
|
272
|
+
end
|
273
|
+
end
|
data/test/database.yml
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
mysql:
|
2
|
+
adapter: mysql
|
3
|
+
database: columns_on_demand_test
|
4
|
+
username: root
|
5
|
+
mysql2:
|
6
|
+
adapter: mysql2
|
7
|
+
database: columns_on_demand_test
|
8
|
+
username: root
|
9
|
+
postgresql:
|
10
|
+
adapter: postgresql
|
11
|
+
database: columns_on_demand_test
|
12
|
+
sqlite3:
|
13
|
+
adapter: sqlite3
|
14
|
+
database: /tmp/columns_on_demand_test.db
|
@@ -0,0 +1,10 @@
|
|
1
|
+
first_file:
|
2
|
+
original_filename: somefile.txt
|
3
|
+
file_data: This is the file data!
|
4
|
+
processing_log: "Processing somefile.txt\n
|
5
|
+
Processing header\n
|
6
|
+
Processed header\n
|
7
|
+
Processing footer\n
|
8
|
+
Processed footer\n"
|
9
|
+
results: Processed 0 entries OK
|
10
|
+
processed_at: 2008-12-23 21:38:10
|
data/test/schema.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
ActiveRecord::Schema.define(:version => 0) do
|
2
|
+
create_table :explicits, :force => true do |t|
|
3
|
+
t.string :original_filename, :null => false
|
4
|
+
t.binary :file_data
|
5
|
+
t.text :processing_log
|
6
|
+
t.text :results
|
7
|
+
t.datetime :processed_at
|
8
|
+
end
|
9
|
+
|
10
|
+
create_table :implicits, :force => true do |t|
|
11
|
+
t.string :original_filename, :null => false
|
12
|
+
t.binary :file_data
|
13
|
+
t.text :processing_log
|
14
|
+
t.text :results
|
15
|
+
t.datetime :processed_at
|
16
|
+
end
|
17
|
+
|
18
|
+
create_table :parents, :force => true do |t|
|
19
|
+
t.text :info
|
20
|
+
end
|
21
|
+
|
22
|
+
create_table :children, :force => true do |t|
|
23
|
+
t.integer :parent_id, :null => false
|
24
|
+
t.text :test_data
|
25
|
+
end
|
26
|
+
|
27
|
+
create_table :serializings, :force => true do |t|
|
28
|
+
t.binary :data
|
29
|
+
end
|
30
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
RAILS_ROOT = File.expand_path("../../..")
|
2
|
+
if File.exist?("#{RAILS_ROOT}/config/boot.rb")
|
3
|
+
require "#{RAILS_ROOT}/config/boot.rb"
|
4
|
+
else
|
5
|
+
require 'rubygems'
|
6
|
+
end
|
7
|
+
|
8
|
+
puts "Rails: #{ENV['RAILS_VERSION'] || 'default'}"
|
9
|
+
gem 'activesupport', ENV['RAILS_VERSION']
|
10
|
+
gem 'activerecord', ENV['RAILS_VERSION']
|
11
|
+
|
12
|
+
require 'test/unit'
|
13
|
+
require 'active_support'
|
14
|
+
require 'active_support/test_case'
|
15
|
+
require 'active_record'
|
16
|
+
require 'active_record/fixtures'
|
17
|
+
|
18
|
+
begin
|
19
|
+
require 'ruby-debug'
|
20
|
+
Debugger.start
|
21
|
+
rescue LoadError
|
22
|
+
# ruby-debug not installed, no debugging for you
|
23
|
+
end
|
24
|
+
|
25
|
+
ActiveRecord::Base.configurations = YAML::load(IO.read(File.join(File.dirname(__FILE__), "database.yml")))
|
26
|
+
configuration = ActiveRecord::Base.configurations[ENV['RAILS_ENV']]
|
27
|
+
raise "use RAILS_ENV=#{ActiveRecord::Base.configurations.keys.sort.join '/'} to test this plugin" unless configuration
|
28
|
+
ActiveRecord::Base.establish_connection configuration
|
29
|
+
|
30
|
+
ActiveSupport::TestCase.send(:include, ActiveRecord::TestFixtures) if ActiveRecord.const_defined?('TestFixtures')
|
31
|
+
ActiveSupport::TestCase.fixture_path = File.join(File.dirname(__FILE__), "fixtures")
|
32
|
+
|
33
|
+
require File.expand_path(File.join(File.dirname(__FILE__), '../init')) # load columns_on_demand
|
metadata
ADDED
@@ -0,0 +1,194 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: columns_on_demand
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 15
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 2
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 2.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Will Bryant
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-08-18 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: activerecord
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 3
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
version: "0"
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
34
|
+
- !ruby/object:Gem::Dependency
|
35
|
+
name: rake
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
hash: 3
|
43
|
+
segments:
|
44
|
+
- 0
|
45
|
+
version: "0"
|
46
|
+
type: :development
|
47
|
+
version_requirements: *id002
|
48
|
+
- !ruby/object:Gem::Dependency
|
49
|
+
name: mysql
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
hash: 3
|
57
|
+
segments:
|
58
|
+
- 0
|
59
|
+
version: "0"
|
60
|
+
type: :development
|
61
|
+
version_requirements: *id003
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: mysql2
|
64
|
+
prerelease: false
|
65
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ">="
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
hash: 3
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
type: :development
|
75
|
+
version_requirements: *id004
|
76
|
+
- !ruby/object:Gem::Dependency
|
77
|
+
name: pg
|
78
|
+
prerelease: false
|
79
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
80
|
+
none: false
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
hash: 3
|
85
|
+
segments:
|
86
|
+
- 0
|
87
|
+
version: "0"
|
88
|
+
type: :development
|
89
|
+
version_requirements: *id005
|
90
|
+
- !ruby/object:Gem::Dependency
|
91
|
+
name: sqlite3
|
92
|
+
prerelease: false
|
93
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
94
|
+
none: false
|
95
|
+
requirements:
|
96
|
+
- - ">="
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
hash: 3
|
99
|
+
segments:
|
100
|
+
- 0
|
101
|
+
version: "0"
|
102
|
+
type: :development
|
103
|
+
version_requirements: *id006
|
104
|
+
description: |
|
105
|
+
Lazily loads large columns on demand.
|
106
|
+
|
107
|
+
By default, does this for all TEXT (:text) and BLOB (:binary) columns, but a list
|
108
|
+
of specific columns to load on demand can be given.
|
109
|
+
|
110
|
+
This is useful to reduce the memory taken by Rails when loading a number of records
|
111
|
+
that have large columns if those particular columns are actually not required most
|
112
|
+
of the time. In this situation it can also greatly reduce the database query time
|
113
|
+
because loading large BLOB/TEXT columns generally means seeking to other database
|
114
|
+
pages since they are not stored wholly in the record's page itself.
|
115
|
+
|
116
|
+
Although this plugin is mainly used for BLOB and TEXT columns, it will actually
|
117
|
+
work on all types - and is just as useful for large string fields etc.
|
118
|
+
|
119
|
+
|
120
|
+
Compatibility
|
121
|
+
=============
|
122
|
+
|
123
|
+
Supports mysql, mysql2, postgresql, and sqlite3.
|
124
|
+
|
125
|
+
Currently tested against Rails 2.3.14, 3.0.17, 3.1.8, and 3.2.8. Note that
|
126
|
+
3.0 and 3.1 have ActiveRecord regressions that will affect sqlite users.
|
127
|
+
|
128
|
+
email: will.bryant@gmail.com
|
129
|
+
executables: []
|
130
|
+
|
131
|
+
extensions: []
|
132
|
+
|
133
|
+
extra_rdoc_files: []
|
134
|
+
|
135
|
+
files:
|
136
|
+
- .gitignore
|
137
|
+
- Gemfile
|
138
|
+
- MIT-LICENSE
|
139
|
+
- README
|
140
|
+
- Rakefile
|
141
|
+
- columns_on_demand.gemspec
|
142
|
+
- init.rb
|
143
|
+
- lib/columns_on_demand.rb
|
144
|
+
- lib/columns_on_demand/version.rb
|
145
|
+
- test/activerecord_count_queries.rb
|
146
|
+
- test/columns_on_demand_test.rb
|
147
|
+
- test/database.yml
|
148
|
+
- test/fixtures/children.yml
|
149
|
+
- test/fixtures/implicits.yml
|
150
|
+
- test/fixtures/parents.yml
|
151
|
+
- test/schema.rb
|
152
|
+
- test/test_helper.rb
|
153
|
+
homepage: http://github.com/willbryant/columns_on_demand
|
154
|
+
licenses: []
|
155
|
+
|
156
|
+
post_install_message:
|
157
|
+
rdoc_options: []
|
158
|
+
|
159
|
+
require_paths:
|
160
|
+
- lib
|
161
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
162
|
+
none: false
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
hash: 3
|
167
|
+
segments:
|
168
|
+
- 0
|
169
|
+
version: "0"
|
170
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
171
|
+
none: false
|
172
|
+
requirements:
|
173
|
+
- - ">="
|
174
|
+
- !ruby/object:Gem::Version
|
175
|
+
hash: 3
|
176
|
+
segments:
|
177
|
+
- 0
|
178
|
+
version: "0"
|
179
|
+
requirements: []
|
180
|
+
|
181
|
+
rubyforge_project:
|
182
|
+
rubygems_version: 1.8.15
|
183
|
+
signing_key:
|
184
|
+
specification_version: 3
|
185
|
+
summary: Lazily loads large columns on demand.
|
186
|
+
test_files:
|
187
|
+
- test/activerecord_count_queries.rb
|
188
|
+
- test/columns_on_demand_test.rb
|
189
|
+
- test/database.yml
|
190
|
+
- test/fixtures/children.yml
|
191
|
+
- test/fixtures/implicits.yml
|
192
|
+
- test/fixtures/parents.yml
|
193
|
+
- test/schema.rb
|
194
|
+
- test/test_helper.rb
|