activerecord-import 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +45 -0
- data/Rakefile +67 -0
- data/VERSION +1 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +146 -0
- data/lib/activerecord-import/active_record/adapters/mysql_adapter.rb +47 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +11 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +11 -0
- data/lib/activerecord-import/base.rb +16 -0
- data/lib/activerecord-import/import.rb +348 -0
- data/lib/activerecord-import/mysql.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/test/active_record/connection_adapter_test.rb +52 -0
- data/test/adapters/mysql.rb +1 -0
- data/test/adapters/postgresql.rb +1 -0
- data/test/adapters/sqlite3.rb +1 -0
- data/test/import_test.rb +202 -0
- data/test/models/book.rb +3 -0
- data/test/models/group.rb +3 -0
- data/test/models/topic.rb +7 -0
- data/test/mysql/import_test.rb +118 -0
- data/test/schema/generic_schema.rb +98 -0
- data/test/schema/mysql_schema.rb +17 -0
- data/test/schema/version.rb +4 -0
- data/test/support/factories.rb +13 -0
- data/test/support/generate.rb +29 -0
- data/test/support/mysql/assertions.rb +55 -0
- data/test/test_helper.rb +119 -0
- metadata +125 -0
data/README.markdown
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# activerecord-import
|
2
|
+
|
3
|
+
activerecord-import is a library for bulk inserting data using ActiveRecord. By default with ActiveRecord, in order to insert multiple records you have to perform individual save operations on each model, like so:
|
4
|
+
|
5
|
+
10.times do |i|
|
6
|
+
Book.create! :name => "book #{i}"
|
7
|
+
end
|
8
|
+
|
9
|
+
This may work fine if all you have is 10 records, but if you have hundreds, thousands, or millions of records it can turn into a nightmare. This is where activerecord-import comes into play. Here's the equivalent behaviour using the #import method:
|
10
|
+
|
11
|
+
books = []
|
12
|
+
10.times{ |i| books << Book.new(:name => "book #{i}") }
|
13
|
+
Book.import books
|
14
|
+
|
15
|
+
Pretty slick, eh?
|
16
|
+
|
17
|
+
Maybe, just maybe you're thinking, why do I have do instantiate ActiveRecord objects? Will that perform validations? What if I don't want validations? What if I want to take advantage of features like MySQL's on duplicate key update? Well, activerecord-import handles all of these cases and more!
|
18
|
+
|
19
|
+
For more documentation on the matter you can refer to two places:
|
20
|
+
|
21
|
+
1. activerecord-import github wiki: http://wiki.github.com/zdennis/activerecord-import/
|
22
|
+
1. the tests in the code base
|
23
|
+
|
24
|
+
# Upgrading from ar-extensions
|
25
|
+
|
26
|
+
This library replaces the ar-extensions library and is compatible with Rails 3. It provides the exact same API for importing data, but it does not include any additional ar-extensions functionality.
|
27
|
+
|
28
|
+
# License
|
29
|
+
|
30
|
+
This is licensed under the ruby license.
|
31
|
+
|
32
|
+
# Author
|
33
|
+
|
34
|
+
Zach Dennis (zach.dennis@gmail.com)
|
35
|
+
|
36
|
+
# Contributors
|
37
|
+
|
38
|
+
* Blythe Dunham
|
39
|
+
* Gabe da Silveira
|
40
|
+
* Henry Work
|
41
|
+
* James Herdman
|
42
|
+
* Marcus Crafter
|
43
|
+
* Thibaud Guillaume-Gentil
|
44
|
+
* Mark Van Holstyn
|
45
|
+
* Victor Costan
|
data/Rakefile
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'rake'
|
3
|
+
require 'rake/testtask'
|
4
|
+
|
5
|
+
begin
|
6
|
+
require 'jeweler'
|
7
|
+
require 'bundler'
|
8
|
+
Jeweler::Tasks.new do |gem|
|
9
|
+
gem.name = "activerecord-import"
|
10
|
+
gem.summary = %Q{Bulk-loading extension for ActiveRecord}
|
11
|
+
gem.description = %Q{Extraction of the ActiveRecord::Base#import functionality from ar-extensions for Rails 3 and beyond}
|
12
|
+
gem.email = "zach.dennis@gmail.com"
|
13
|
+
gem.homepage = "http://github.com/zdennis/activerecord-import"
|
14
|
+
gem.authors = ["Zach Dennis"]
|
15
|
+
gem.files = FileList["VERSION", "Rakefile", "README*", "lib/**/*"]
|
16
|
+
gem.add_bundler_dependencies
|
17
|
+
|
18
|
+
# gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
|
19
|
+
end
|
20
|
+
Jeweler::GemcutterTasks.new
|
21
|
+
rescue LoadError
|
22
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
23
|
+
end
|
24
|
+
|
25
|
+
namespace :display do
|
26
|
+
task :notice do
|
27
|
+
puts
|
28
|
+
puts "To run tests you must supply the adapter, see rake -T for more information."
|
29
|
+
puts
|
30
|
+
end
|
31
|
+
end
|
32
|
+
task :default => ["display:notice"]
|
33
|
+
|
34
|
+
ADAPTERS = %w(mysql postgresql sqlite3)
|
35
|
+
ADAPTERS.each do |adapter|
|
36
|
+
namespace :test do
|
37
|
+
desc "Runs #{adapter} database tests."
|
38
|
+
Rake::TestTask.new(adapter) do |t|
|
39
|
+
t.test_files = FileList["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/#{adapter}/**/*_test.rb"]
|
40
|
+
end
|
41
|
+
task adapter => :check_dependencies
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
begin
|
46
|
+
require 'rcov/rcovtask'
|
47
|
+
adapter = ENV['ARE_DB']
|
48
|
+
Rcov::RcovTask.new do |test|
|
49
|
+
test.libs << 'test'
|
50
|
+
test.pattern = ["test/adapters/#{adapter}.rb", "test/*_test.rb", "test/#{adapter}/**/*_test.rb"]
|
51
|
+
test.verbose = true
|
52
|
+
end
|
53
|
+
rescue LoadError
|
54
|
+
task :rcov do
|
55
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install rcov"
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
require 'rake/rdoctask'
|
60
|
+
Rake::RDocTask.new do |rdoc|
|
61
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
62
|
+
|
63
|
+
rdoc.rdoc_dir = 'rdoc'
|
64
|
+
rdoc.title = "activerecord-import #{version}"
|
65
|
+
rdoc.rdoc_files.include('README*')
|
66
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
67
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,146 @@
|
|
1
|
+
module ActiveRecord # :nodoc:
|
2
|
+
module ConnectionAdapters # :nodoc:
|
3
|
+
class AbstractAdapter # :nodoc:
|
4
|
+
NO_MAX_PACKET = 0
|
5
|
+
QUERY_OVERHEAD = 8 #This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
6
|
+
|
7
|
+
def next_value_for_sequence(sequence_name)
|
8
|
+
%{#{sequence_name}.nextval}
|
9
|
+
end
|
10
|
+
|
11
|
+
# +sql+ can be a single string or an array. If it is an array all
|
12
|
+
# elements that are in position >= 1 will be appended to the final SQL.
|
13
|
+
def insert_many( sql, values, *args ) # :nodoc:
|
14
|
+
# the number of inserts default
|
15
|
+
number_of_inserts = 0
|
16
|
+
|
17
|
+
base_sql,post_sql = if sql.is_a?( String )
|
18
|
+
[ sql, '' ]
|
19
|
+
elsif sql.is_a?( Array )
|
20
|
+
[ sql.shift, sql.join( ' ' ) ]
|
21
|
+
end
|
22
|
+
|
23
|
+
sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
|
24
|
+
|
25
|
+
# the number of bytes the requested insert statement values will take up
|
26
|
+
values_in_bytes = self.class.sum_sizes( *values )
|
27
|
+
|
28
|
+
# the number of bytes (commas) it will take to comma separate our values
|
29
|
+
comma_separated_bytes = values.size-1
|
30
|
+
|
31
|
+
# the total number of bytes required if this statement is one statement
|
32
|
+
total_bytes = sql_size + values_in_bytes + comma_separated_bytes
|
33
|
+
|
34
|
+
max = max_allowed_packet
|
35
|
+
|
36
|
+
# if we can insert it all as one statement
|
37
|
+
if NO_MAX_PACKET == max or total_bytes < max
|
38
|
+
number_of_inserts += 1
|
39
|
+
sql2insert = base_sql + values.join( ',' ) + post_sql
|
40
|
+
insert( sql2insert, *args )
|
41
|
+
else
|
42
|
+
value_sets = self.class.get_insert_value_sets( values, sql_size, max )
|
43
|
+
value_sets.each do |values|
|
44
|
+
number_of_inserts += 1
|
45
|
+
sql2insert = base_sql + values.join( ',' ) + post_sql
|
46
|
+
insert( sql2insert, *args )
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
number_of_inserts
|
51
|
+
end
|
52
|
+
|
53
|
+
def pre_sql_statements(options)
|
54
|
+
sql = []
|
55
|
+
sql << options[:pre_sql] if options[:pre_sql]
|
56
|
+
sql << options[:command] if options[:command]
|
57
|
+
sql << "IGNORE" if options[:ignore]
|
58
|
+
|
59
|
+
#add keywords like IGNORE or DELAYED
|
60
|
+
if options[:keywords].is_a?(Array)
|
61
|
+
sql.concat(options[:keywords])
|
62
|
+
elsif options[:keywords]
|
63
|
+
sql << options[:keywords].to_s
|
64
|
+
end
|
65
|
+
|
66
|
+
sql
|
67
|
+
end
|
68
|
+
|
69
|
+
# Synchronizes the passed in ActiveRecord instances with the records in
|
70
|
+
# the database by calling +reload+ on each instance.
|
71
|
+
def after_import_synchronize( instances )
|
72
|
+
instances.each { |e| e.reload }
|
73
|
+
end
|
74
|
+
|
75
|
+
# Returns an array of post SQL statements given the passed in options.
|
76
|
+
def post_sql_statements( table_name, options ) # :nodoc:
|
77
|
+
post_sql_statements = []
|
78
|
+
if options[:on_duplicate_key_update]
|
79
|
+
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
|
80
|
+
end
|
81
|
+
|
82
|
+
#custom user post_sql
|
83
|
+
post_sql_statements << options[:post_sql] if options[:post_sql]
|
84
|
+
|
85
|
+
#with rollup
|
86
|
+
post_sql_statements << rollup_sql if options[:rollup]
|
87
|
+
|
88
|
+
post_sql_statements
|
89
|
+
end
|
90
|
+
|
91
|
+
|
92
|
+
# Generates the INSERT statement used in insert multiple value sets.
|
93
|
+
def multiple_value_sets_insert_sql(table_name, column_names, options) # :nodoc:
|
94
|
+
"INSERT #{options[:ignore] ? 'IGNORE ':''}INTO #{table_name} (#{column_names.join(',')}) VALUES "
|
95
|
+
end
|
96
|
+
|
97
|
+
# Returns SQL the VALUES for an INSERT statement given the passed in +columns+
|
98
|
+
# and +array_of_attributes+.
|
99
|
+
def values_sql_for_column_names_and_attributes( columns, array_of_attributes ) # :nodoc:
|
100
|
+
values = []
|
101
|
+
array_of_attributes.each do |arr|
|
102
|
+
my_values = []
|
103
|
+
arr.each_with_index do |val,j|
|
104
|
+
my_values << quote( val, columns[j] )
|
105
|
+
end
|
106
|
+
values << my_values
|
107
|
+
end
|
108
|
+
values_arr = values.map{ |arr| '(' + arr.join( ',' ) + ')' }
|
109
|
+
end
|
110
|
+
|
111
|
+
# Returns the sum of the sizes of the passed in objects. This should
|
112
|
+
# probably be moved outside this class, but to where?
|
113
|
+
def self.sum_sizes( *objects ) # :nodoc:
|
114
|
+
objects.inject( 0 ){|sum,o| sum += o.size }
|
115
|
+
end
|
116
|
+
|
117
|
+
# Returns the maximum number of bytes that the server will allow
|
118
|
+
# in a single packet
|
119
|
+
def max_allowed_packet
|
120
|
+
NO_MAX_PACKET
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.get_insert_value_sets( values, sql_size, max_bytes ) # :nodoc:
|
124
|
+
value_sets = []
|
125
|
+
arr, current_arr_values_size, current_size = [], 0, 0
|
126
|
+
values.each_with_index do |val,i|
|
127
|
+
comma_bytes = arr.size
|
128
|
+
sql_size_thus_far = sql_size + current_size + val.size + comma_bytes
|
129
|
+
if NO_MAX_PACKET == max_bytes or sql_size_thus_far <= max_bytes
|
130
|
+
current_size += val.size
|
131
|
+
arr << val
|
132
|
+
else
|
133
|
+
value_sets << arr
|
134
|
+
arr = [ val ]
|
135
|
+
current_size = val.size
|
136
|
+
end
|
137
|
+
|
138
|
+
# if we're on the last iteration push whatever we have in arr to value_sets
|
139
|
+
value_sets << arr if i == (values.size-1)
|
140
|
+
end
|
141
|
+
[ *value_sets ]
|
142
|
+
end
|
143
|
+
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require "active_record/connection_adapters/mysql_adapter"
|
2
|
+
|
3
|
+
class ActiveRecord::ConnectionAdapters::MysqlAdapter
|
4
|
+
include ActiveRecord::Extensions::Import::ImportSupport
|
5
|
+
include ActiveRecord::Extensions::Import::OnDuplicateKeyUpdateSupport
|
6
|
+
|
7
|
+
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
8
|
+
# in +args+.
|
9
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
10
|
+
sql = ' ON DUPLICATE KEY UPDATE '
|
11
|
+
arg = args.first
|
12
|
+
if arg.is_a?( Array )
|
13
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
|
14
|
+
elsif arg.is_a?( Hash )
|
15
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
|
16
|
+
elsif arg.is_a?( String )
|
17
|
+
sql << arg
|
18
|
+
else
|
19
|
+
raise ArgumentError.new( "Expected Array or Hash" )
|
20
|
+
end
|
21
|
+
sql
|
22
|
+
end
|
23
|
+
|
24
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
|
25
|
+
results = arr.map do |column|
|
26
|
+
qc = quote_column_name( column )
|
27
|
+
"#{table_name}.#{qc}=VALUES(#{qc})"
|
28
|
+
end
|
29
|
+
results.join( ',' )
|
30
|
+
end
|
31
|
+
|
32
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
|
33
|
+
sql = ' ON DUPLICATE KEY UPDATE '
|
34
|
+
results = hsh.map do |column1, column2|
|
35
|
+
qc1 = quote_column_name( column1 )
|
36
|
+
qc2 = quote_column_name( column2 )
|
37
|
+
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
38
|
+
end
|
39
|
+
results.join( ',')
|
40
|
+
end
|
41
|
+
|
42
|
+
#return true if the statement is a duplicate key record error
|
43
|
+
def duplicate_key_update_error?(exception)# :nodoc:
|
44
|
+
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require "active_record/connection_adapters/postgresql_adapter"
|
2
|
+
|
3
|
+
module ActiveRecord # :nodoc:
|
4
|
+
module ConnectionAdapters # :nodoc:
|
5
|
+
class PostgreSQLAdapter # :nodoc:
|
6
|
+
def next_value_for_sequence(sequence_name)
|
7
|
+
%{nextval('#{sequence_name}')}
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require "active_record/connection_adapters/sqlite3_adapter"
|
2
|
+
|
3
|
+
module ActiveRecord # :nodoc:
|
4
|
+
module ConnectionAdapters # :nodoc:
|
5
|
+
class Sqlite3Adapter # :nodoc:
|
6
|
+
def next_value_for_sequence(sequence_name)
|
7
|
+
%{nextval('#{sequence_name}')}
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require "pathname"
|
2
|
+
require "active_record"
|
3
|
+
require "active_record/version"
|
4
|
+
|
5
|
+
module ActiveRecord::Extensions
|
6
|
+
AdapterPath = File.join File.expand_path(File.dirname(__FILE__)), "/active_record/adapters"
|
7
|
+
|
8
|
+
def self.require_adapter(adapter)
|
9
|
+
require File.join(AdapterPath,"/abstract_adapter")
|
10
|
+
require File.join(AdapterPath,"/#{adapter}_adapter")
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
this_dir = Pathname.new File.dirname(__FILE__)
|
15
|
+
require this_dir.join("import")
|
16
|
+
require this_dir.join("active_record/adapters/abstract_adapter")
|
@@ -0,0 +1,348 @@
|
|
1
|
+
require "ostruct"
|
2
|
+
|
3
|
+
module ActiveRecord::Extensions::ConnectionAdapters ; end
|
4
|
+
|
5
|
+
module ActiveRecord::Extensions::Import #:nodoc:
|
6
|
+
module ImportSupport #:nodoc:
|
7
|
+
def supports_import? #:nodoc:
|
8
|
+
true
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
module OnDuplicateKeyUpdateSupport #:nodoc:
|
13
|
+
def supports_on_duplicate_key_update? #:nodoc:
|
14
|
+
true
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class ActiveRecord::Base
|
20
|
+
class << self
|
21
|
+
|
22
|
+
# use tz as set in ActiveRecord::Base
|
23
|
+
tproc = lambda do
|
24
|
+
ActiveRecord::Base.default_timezone == :utc ? Time.now.utc : Time.now
|
25
|
+
end
|
26
|
+
|
27
|
+
AREXT_RAILS_COLUMNS = {
|
28
|
+
:create => { "created_on" => tproc ,
|
29
|
+
"created_at" => tproc },
|
30
|
+
:update => { "updated_on" => tproc ,
|
31
|
+
"updated_at" => tproc }
|
32
|
+
}
|
33
|
+
AREXT_RAILS_COLUMN_NAMES = AREXT_RAILS_COLUMNS[:create].keys + AREXT_RAILS_COLUMNS[:update].keys
|
34
|
+
|
35
|
+
# Returns true if the current database connection adapter
|
36
|
+
# supports import functionality, otherwise returns false.
|
37
|
+
def supports_import?
|
38
|
+
connection.supports_import?
|
39
|
+
rescue NoMethodError
|
40
|
+
false
|
41
|
+
end
|
42
|
+
|
43
|
+
# Returns true if the current database connection adapter
|
44
|
+
# supports on duplicate key update functionality, otherwise
|
45
|
+
# returns false.
|
46
|
+
def supports_on_duplicate_key_update?
|
47
|
+
connection.supports_on_duplicate_key_update?
|
48
|
+
rescue NoMethodError
|
49
|
+
false
|
50
|
+
end
|
51
|
+
|
52
|
+
# Imports a collection of values to the database.
|
53
|
+
#
|
54
|
+
# This is more efficient than using ActiveRecord::Base#create or
|
55
|
+
# ActiveRecord::Base#save multiple times. This method works well if
|
56
|
+
# you want to create more than one record at a time and do not care
|
57
|
+
# about having ActiveRecord objects returned for each record
|
58
|
+
# inserted.
|
59
|
+
#
|
60
|
+
# This can be used with or without validations. It does not utilize
|
61
|
+
# the ActiveRecord::Callbacks during creation/modification while
|
62
|
+
# performing the import.
|
63
|
+
#
|
64
|
+
# == Usage
|
65
|
+
# Model.import array_of_models
|
66
|
+
# Model.import column_names, array_of_values
|
67
|
+
# Model.import column_names, array_of_values, options
|
68
|
+
#
|
69
|
+
# ==== Model.import array_of_models
|
70
|
+
#
|
71
|
+
# With this form you can call _import_ passing in an array of model
|
72
|
+
# objects that you want updated.
|
73
|
+
#
|
74
|
+
# ==== Model.import column_names, array_of_values
|
75
|
+
#
|
76
|
+
# The first parameter +column_names+ is an array of symbols or
|
77
|
+
# strings which specify the columns that you want to update.
|
78
|
+
#
|
79
|
+
# The second parameter, +array_of_values+, is an array of
|
80
|
+
# arrays. Each subarray is a single set of values for a new
|
81
|
+
# record. The order of values in each subarray should match up to
|
82
|
+
# the order of the +column_names+.
|
83
|
+
#
|
84
|
+
# ==== Model.import column_names, array_of_values, options
|
85
|
+
#
|
86
|
+
# The first two parameters are the same as the above form. The third
|
87
|
+
# parameter, +options+, is a hash. This is optional. Please see
|
88
|
+
# below for what +options+ are available.
|
89
|
+
#
|
90
|
+
# == Options
|
91
|
+
# * +validate+ - true|false, tells import whether or not to use \
|
92
|
+
# ActiveRecord validations. Validations are enforced by default.
|
93
|
+
# * +on_duplicate_key_update+ - an Array or Hash, tells import to \
|
94
|
+
# use MySQL's ON DUPLICATE KEY UPDATE ability. See On Duplicate\
|
95
|
+
# Key Update below.
|
96
|
+
# * +synchronize+ - an array of ActiveRecord instances for the model
|
97
|
+
# that you are currently importing data into. This synchronizes
|
98
|
+
# existing model instances in memory with updates from the import.
|
99
|
+
# * +timestamps+ - true|false, tells import to not add timestamps \
|
100
|
+
# (if false) even if record timestamps is disabled in ActiveRecord::Base
|
101
|
+
#
|
102
|
+
# == Examples
|
103
|
+
# class BlogPost < ActiveRecord::Base ; end
|
104
|
+
#
|
105
|
+
# # Example using array of model objects
|
106
|
+
# posts = [ BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT',
|
107
|
+
# BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT2',
|
108
|
+
# BlogPost.new :author_name=>'Zach Dennis', :title=>'AREXT3' ]
|
109
|
+
# BlogPost.import posts
|
110
|
+
#
|
111
|
+
# # Example using column_names and array_of_values
|
112
|
+
# columns = [ :author_name, :title ]
|
113
|
+
# values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ]
|
114
|
+
# BlogPost.import columns, values
|
115
|
+
#
|
116
|
+
# # Example using column_names, array_of_value and options
|
117
|
+
# columns = [ :author_name, :title ]
|
118
|
+
# values = [ [ 'zdennis', 'test post' ], [ 'jdoe', 'another test post' ] ]
|
119
|
+
# BlogPost.import( columns, values, :validate => false )
|
120
|
+
#
|
121
|
+
# # Example synchronizing existing instances in memory
|
122
|
+
# post = BlogPost.find_by_author_name( 'zdennis' )
|
123
|
+
# puts post.author_name # => 'zdennis'
|
124
|
+
# columns = [ :author_name, :title ]
|
125
|
+
# values = [ [ 'yoda', 'test post' ] ]
|
126
|
+
# BlogPost.import posts, :synchronize=>[ post ]
|
127
|
+
# puts post.author_name # => 'yoda'
|
128
|
+
#
|
129
|
+
# == On Duplicate Key Update (MySQL only)
|
130
|
+
#
|
131
|
+
# The :on_duplicate_key_update option can be either an Array or a Hash.
|
132
|
+
#
|
133
|
+
# ==== Using an Array
|
134
|
+
#
|
135
|
+
# The :on_duplicate_key_update option can be an array of column
|
136
|
+
# names. The column names are the only fields that are updated if
|
137
|
+
# a duplicate record is found. Below is an example:
|
138
|
+
#
|
139
|
+
# BlogPost.import columns, values, :on_duplicate_key_update=>[ :date_modified, :content, :author ]
|
140
|
+
#
|
141
|
+
# ==== Using A Hash
|
142
|
+
#
|
143
|
+
# The :on_duplicate_key_update option can be a hash of column name
|
144
|
+
# to model attribute name mappings. This gives you finer grained
|
145
|
+
# control over what fields are updated with what attributes on your
|
146
|
+
# model. Below is an example:
|
147
|
+
#
|
148
|
+
# BlogPost.import columns, attributes, :on_duplicate_key_update=>{ :title => :title }
|
149
|
+
#
|
150
|
+
# = Returns
|
151
|
+
# This returns an object which responds to +failed_instances+ and +num_inserts+.
|
152
|
+
# * failed_instances - an array of objects that fails validation and were not committed to the database. An empty array if no validation is performed.
|
153
|
+
# * num_inserts - the number of insert statements it took to import the data
|
154
|
+
def import( *args )
|
155
|
+
options = { :validate=>true, :timestamps=>true }
|
156
|
+
options.merge!( args.pop ) if args.last.is_a? Hash
|
157
|
+
|
158
|
+
# assume array of model objects
|
159
|
+
if args.last.is_a?( Array ) and args.last.first.is_a? ActiveRecord::Base
|
160
|
+
if args.length == 2
|
161
|
+
models = args.last
|
162
|
+
column_names = args.first
|
163
|
+
else
|
164
|
+
models = args.first
|
165
|
+
column_names = self.column_names.dup
|
166
|
+
end
|
167
|
+
|
168
|
+
array_of_attributes = []
|
169
|
+
models.each do |model|
|
170
|
+
# this next line breaks sqlite.so with a segmentation fault
|
171
|
+
# if model.new_record? || options[:on_duplicate_key_update]
|
172
|
+
attributes = []
|
173
|
+
column_names.each do |name|
|
174
|
+
attributes << model.send( "#{name}_before_type_cast" )
|
175
|
+
end
|
176
|
+
array_of_attributes << attributes
|
177
|
+
# end
|
178
|
+
end
|
179
|
+
# supports 2-element array and array
|
180
|
+
elsif args.size == 2 and args.first.is_a?( Array ) and args.last.is_a?( Array )
|
181
|
+
column_names, array_of_attributes = args
|
182
|
+
else
|
183
|
+
raise ArgumentError.new( "Invalid arguments!" )
|
184
|
+
end
|
185
|
+
|
186
|
+
# Force the primary key col into the insert if it's not
|
187
|
+
# on the list and we are using a sequence and stuff a nil
|
188
|
+
# value for it into each row so the sequencer will fire later
|
189
|
+
if !column_names.include?(primary_key) && sequence_name && connection.prefetch_primary_key?
|
190
|
+
column_names << primary_key
|
191
|
+
array_of_attributes.each { |a| a << nil }
|
192
|
+
end
|
193
|
+
|
194
|
+
is_validating = options.delete( :validate )
|
195
|
+
|
196
|
+
# dup the passed in array so we don't modify it unintentionally
|
197
|
+
array_of_attributes = array_of_attributes.dup
|
198
|
+
|
199
|
+
# record timestamps unless disabled in ActiveRecord::Base
|
200
|
+
if record_timestamps && options.delete( :timestamps )
|
201
|
+
add_special_rails_stamps column_names, array_of_attributes, options
|
202
|
+
end
|
203
|
+
|
204
|
+
return_obj = if is_validating
|
205
|
+
import_with_validations( column_names, array_of_attributes, options )
|
206
|
+
else
|
207
|
+
num_inserts = import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
208
|
+
OpenStruct.new :failed_instances=>[], :num_inserts=>num_inserts
|
209
|
+
end
|
210
|
+
|
211
|
+
if options[:synchronize]
|
212
|
+
synchronize( options[:synchronize] )
|
213
|
+
end
|
214
|
+
|
215
|
+
return_obj.num_inserts = 0 if return_obj.num_inserts.nil?
|
216
|
+
return_obj
|
217
|
+
end
|
218
|
+
|
219
|
+
# TODO import_from_table needs to be implemented.
|
220
|
+
def import_from_table( options ) # :nodoc:
|
221
|
+
end
|
222
|
+
|
223
|
+
# Imports the passed in +column_names+ and +array_of_attributes+
|
224
|
+
# given the passed in +options+ Hash with validations. Returns an
|
225
|
+
# object with the methods +failed_instances+ and +num_inserts+.
|
226
|
+
# +failed_instances+ is an array of instances that failed validations.
|
227
|
+
# +num_inserts+ is the number of inserts it took to import the data. See
|
228
|
+
# ActiveRecord::Base.import for more information on
|
229
|
+
# +column_names+, +array_of_attributes+ and +options+.
|
230
|
+
def import_with_validations( column_names, array_of_attributes, options={} )
|
231
|
+
failed_instances = []
|
232
|
+
|
233
|
+
# create instances for each of our column/value sets
|
234
|
+
arr = validations_array_for_column_names_and_attributes( column_names, array_of_attributes )
|
235
|
+
|
236
|
+
# keep track of the instance and the position it is currently at. if this fails
|
237
|
+
# validation we'll use the index to remove it from the array_of_attributes
|
238
|
+
arr.each_with_index do |hsh,i|
|
239
|
+
instance = new( hsh )
|
240
|
+
if not instance.valid?
|
241
|
+
array_of_attributes[ i ] = nil
|
242
|
+
failed_instances << instance
|
243
|
+
end
|
244
|
+
end
|
245
|
+
array_of_attributes.compact!
|
246
|
+
|
247
|
+
num_inserts = array_of_attributes.empty? ? 0 : import_without_validations_or_callbacks( column_names, array_of_attributes, options )
|
248
|
+
OpenStruct.new :failed_instances=>failed_instances, :num_inserts => num_inserts
|
249
|
+
end
|
250
|
+
|
251
|
+
# Imports the passed in +column_names+ and +array_of_attributes+
|
252
|
+
# given the passed in +options+ Hash. This will return the number
|
253
|
+
# of insert operations it took to create these records without
|
254
|
+
# validations or callbacks. See ActiveRecord::Base.import for more
|
255
|
+
# information on +column_names+, +array_of_attributes_ and
|
256
|
+
# +options+.
|
257
|
+
def import_without_validations_or_callbacks( column_names, array_of_attributes, options={} )
|
258
|
+
escaped_column_names = quote_column_names( column_names )
|
259
|
+
columns = []
|
260
|
+
array_of_attributes.first.each_with_index { |arr,i| columns << columns_hash[ column_names[i] ] }
|
261
|
+
|
262
|
+
if not supports_import?
|
263
|
+
columns_sql = "(" + escaped_column_names.join( ',' ) + ")"
|
264
|
+
insert_statements, values = [], []
|
265
|
+
number_inserted = 0
|
266
|
+
array_of_attributes.each do |arr|
|
267
|
+
my_values = []
|
268
|
+
arr.each_with_index do |val,j|
|
269
|
+
if !sequence_name.blank? && column_names[j] == primary_key && val.nil?
|
270
|
+
my_values << connection.next_value_for_sequence(sequence_name)
|
271
|
+
else
|
272
|
+
my_values << connection.quote( val, columns[j] )
|
273
|
+
end
|
274
|
+
end
|
275
|
+
insert_statements << "INSERT INTO #{quoted_table_name} #{columns_sql} VALUES(" + my_values.join( ',' ) + ")"
|
276
|
+
connection.execute( insert_statements.last )
|
277
|
+
number_inserted += 1
|
278
|
+
end
|
279
|
+
else
|
280
|
+
# generate the sql
|
281
|
+
insert_sql = connection.multiple_value_sets_insert_sql( quoted_table_name, escaped_column_names, options )
|
282
|
+
values_sql = connection.values_sql_for_column_names_and_attributes( columns, array_of_attributes )
|
283
|
+
post_sql_statements = connection.post_sql_statements( quoted_table_name, options )
|
284
|
+
|
285
|
+
# perform the inserts
|
286
|
+
number_inserted = connection.insert_many( [ insert_sql, post_sql_statements ].flatten,
|
287
|
+
values_sql,
|
288
|
+
"#{self.class.name} Create Many Without Validations Or Callbacks" )
|
289
|
+
end
|
290
|
+
number_inserted
|
291
|
+
end
|
292
|
+
|
293
|
+
# Returns an array of quoted column names
|
294
|
+
def quote_column_names( names )
|
295
|
+
names.map{ |name| connection.quote_column_name( name ) }
|
296
|
+
end
|
297
|
+
|
298
|
+
|
299
|
+
private
|
300
|
+
|
301
|
+
def add_special_rails_stamps( column_names, array_of_attributes, options )
|
302
|
+
AREXT_RAILS_COLUMNS[:create].each_pair do |key, blk|
|
303
|
+
if self.column_names.include?(key)
|
304
|
+
value = blk.call
|
305
|
+
if index=column_names.index(key)
|
306
|
+
# replace every instance of the array of attributes with our value
|
307
|
+
array_of_attributes.each{ |arr| arr[index] = value }
|
308
|
+
else
|
309
|
+
column_names << key
|
310
|
+
array_of_attributes.each { |arr| arr << value }
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
AREXT_RAILS_COLUMNS[:update].each_pair do |key, blk|
|
316
|
+
if self.column_names.include?(key)
|
317
|
+
value = blk.call
|
318
|
+
if index=column_names.index(key)
|
319
|
+
# replace every instance of the array of attributes with our value
|
320
|
+
array_of_attributes.each{ |arr| arr[index] = value }
|
321
|
+
else
|
322
|
+
column_names << key
|
323
|
+
array_of_attributes.each { |arr| arr << value }
|
324
|
+
end
|
325
|
+
|
326
|
+
if options[:on_duplicate_key_update]
|
327
|
+
options[:on_duplicate_key_update] << key.to_sym if options[:on_duplicate_key_update].is_a?(Array)
|
328
|
+
options[:on_duplicate_key_update][key.to_sym] = key.to_sym if options[:on_duplicate_key_update].is_a?(Hash)
|
329
|
+
else
|
330
|
+
options[:on_duplicate_key_update] = [ key.to_sym ]
|
331
|
+
end
|
332
|
+
end
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
# Returns an Array of Hashes for the passed in +column_names+ and +array_of_attributes+.
|
337
|
+
def validations_array_for_column_names_and_attributes( column_names, array_of_attributes ) # :nodoc:
|
338
|
+
arr = []
|
339
|
+
array_of_attributes.each do |attributes|
|
340
|
+
c = 0
|
341
|
+
hsh = attributes.inject( {} ){|hsh,attr| hsh[ column_names[c] ] = attr ; c+=1 ; hsh }
|
342
|
+
arr << hsh
|
343
|
+
end
|
344
|
+
arr
|
345
|
+
end
|
346
|
+
|
347
|
+
end
|
348
|
+
end
|