activerecord_save_many 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- data/README.rdoc +5 -0
- data/VERSION +1 -1
- data/lib/activerecord_save_many.rb +1 -134
- data/lib/activerecord_save_many/save_many.rb +135 -0
- data/spec/spec_helper.rb +1 -1
- metadata +43 -21
data/README.rdoc
CHANGED
@@ -3,6 +3,11 @@
|
|
3
3
|
adds a save_many method to ActiveRecord classes. save_many generates reasonable
|
4
4
|
SQL for inserting or updating many ActiveRecord instances in one go
|
5
5
|
|
6
|
+
== Install
|
7
|
+
|
8
|
+
gem source --add http://gemcutter.org
|
9
|
+
gem install activerecord_save_many
|
10
|
+
|
6
11
|
== Note on Patches/Pull Requests
|
7
12
|
|
8
13
|
* Fork the project.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.1
|
@@ -1,136 +1,3 @@
|
|
1
|
-
require 'set'
|
2
1
|
require 'activerecord_execute_raw'
|
2
|
+
require 'active_record_save_many/save_many'
|
3
3
|
|
4
|
-
module ActiveRecord
|
5
|
-
module SaveMany
|
6
|
-
MAX_QUERY_SIZE = 1024 * 1024
|
7
|
-
# options for the create_many method
|
8
|
-
OPTIONS_KEYS = [:columns, :max_rows, :async, :ignore, :update, :updates].to_set
|
9
|
-
|
10
|
-
# options for the create_many_sql method
|
11
|
-
SQL_OPTIONS_KEYS = [:async, :ignore, :update, :updates].to_set
|
12
|
-
|
13
|
-
class << self
|
14
|
-
attr_accessor :default_max_rows
|
15
|
-
end
|
16
|
-
self.default_max_rows = 50000
|
17
|
-
|
18
|
-
module Functions
|
19
|
-
def rails_env
|
20
|
-
RAILS_ENV if defined? RAILS_ENV
|
21
|
-
end
|
22
|
-
module_function :rails_env
|
23
|
-
|
24
|
-
def disable_async?
|
25
|
-
# for predictable tests we disable delayed insert during testing
|
26
|
-
rails_env()=="test"
|
27
|
-
end
|
28
|
-
module_function :disable_async?
|
29
|
-
|
30
|
-
def check_options(permitted, options)
|
31
|
-
unknown_keys = options.keys.to_set - permitted
|
32
|
-
raise "unknown options: #{unknown_keys.to_a.join(", ")}" if !unknown_keys.empty?
|
33
|
-
end
|
34
|
-
module_function :check_options
|
35
|
-
|
36
|
-
# slice an array into smaller arrays with maximum size max_size
|
37
|
-
def slice_array(max_length, arr)
|
38
|
-
slices = []
|
39
|
-
(0..arr.length-1).step( max_length ){ |i| slices << arr.slice(i,max_length) }
|
40
|
-
slices
|
41
|
-
end
|
42
|
-
module_function :slice_array
|
43
|
-
|
44
|
-
def add_columns(klass, values, options)
|
45
|
-
columns = options[:columns] || klass.columns.map(&:name)
|
46
|
-
|
47
|
-
# add a :type column automatically for STI, if not already present
|
48
|
-
if klass.superclass!=ActiveRecord::Base && !columns.include?(:type)
|
49
|
-
columns = [:type, *columns]
|
50
|
-
values = values.map{|vals| [klass.to_s, *vals]}
|
51
|
-
end
|
52
|
-
|
53
|
-
[columns, values]
|
54
|
-
end
|
55
|
-
module_function :add_columns
|
56
|
-
end
|
57
|
-
|
58
|
-
module ClassMethods
|
59
|
-
def save_many_max_rows=(max_rows)
|
60
|
-
@save_many_max_rows=max_rows
|
61
|
-
end
|
62
|
-
|
63
|
-
def save_many_max_rows
|
64
|
-
@save_many_max_rows || ActiveRecord::SaveMany::default_max_rows
|
65
|
-
end
|
66
|
-
|
67
|
-
def save_many(values, options={})
|
68
|
-
Functions::check_options(ActiveRecord::SaveMany::OPTIONS_KEYS , options)
|
69
|
-
return if values.nil? || values.empty?
|
70
|
-
|
71
|
-
columns, values = Functions::add_columns(self, values, options)
|
72
|
-
|
73
|
-
# if more than max_rows, execute multiple sql statements
|
74
|
-
max_rows = options[:max_rows] || save_many_max_rows
|
75
|
-
batches = Functions::slice_array(max_rows, values)
|
76
|
-
|
77
|
-
batches.each do |rows|
|
78
|
-
rows = rows.map do |obj|
|
79
|
-
if obj.is_a? ActiveRecord::Base
|
80
|
-
obj.send( :callback, :before_save )
|
81
|
-
if obj.id
|
82
|
-
obj.send( :callback, :before_update)
|
83
|
-
else
|
84
|
-
obj.send( :callback, :before_create )
|
85
|
-
end
|
86
|
-
raise "#{obj.errors.full_messages.join(', ')}" if !obj.valid?
|
87
|
-
end
|
88
|
-
columns.map{|col| obj[col]}
|
89
|
-
end
|
90
|
-
|
91
|
-
sql = connection.save_many_sql(self,
|
92
|
-
table_name,
|
93
|
-
columns,
|
94
|
-
rows,
|
95
|
-
{ :ignore=>options[:ignore],
|
96
|
-
:async=>options[:async] && !Functions::disable_async?(),
|
97
|
-
:update=>options[:update] || options[:updates],
|
98
|
-
:updates=>options[:updates] || {} })
|
99
|
-
|
100
|
-
connection.execute_raw sql
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
|
105
|
-
module MySQL
|
106
|
-
def save_many_sql(klass, table_name, columns, rows, options)
|
107
|
-
Functions::check_options(SQL_OPTIONS_KEYS, options)
|
108
|
-
|
109
|
-
sql = ["insert", ("delayed" if options[:async]), ("ignore" if options[:ignore])].compact.join(' ') +
|
110
|
-
" into #{table_name} (#{columns.join(',')}) values " +
|
111
|
-
rows.map{|vals| "(" + vals.map{|v| klass.quote_value(v)}.join(",") +")"}.join(",") +
|
112
|
-
(" on duplicate key update "+columns.map{|c| options[:updates][c] || "#{c}=values(#{c})"}.join(",") if options[:update]).to_s
|
113
|
-
end
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
class Base
|
118
|
-
class << self
|
119
|
-
include ActiveRecord::SaveMany::ClassMethods
|
120
|
-
end
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
module JdbcSpec
|
125
|
-
module MySQL
|
126
|
-
include ActiveRecord::SaveMany::MySQL
|
127
|
-
end
|
128
|
-
end
|
129
|
-
|
130
|
-
module ActiveRecord
|
131
|
-
module ConnectionAdapters
|
132
|
-
class MysqlAdapter
|
133
|
-
include ActiveRecord::SaveMany::MySQL
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
@@ -0,0 +1,135 @@
|
|
1
|
+
require 'set'
|
2
|
+
|
3
|
+
module ActiveRecord
|
4
|
+
module SaveMany
|
5
|
+
MAX_QUERY_SIZE = 1024 * 1024
|
6
|
+
# options for the create_many method
|
7
|
+
OPTIONS_KEYS = [:columns, :max_rows, :async, :ignore, :update, :updates].to_set
|
8
|
+
|
9
|
+
# options for the create_many_sql method
|
10
|
+
SQL_OPTIONS_KEYS = [:async, :ignore, :update, :updates].to_set
|
11
|
+
|
12
|
+
class << self
|
13
|
+
attr_accessor :default_max_rows
|
14
|
+
end
|
15
|
+
self.default_max_rows = 50000
|
16
|
+
|
17
|
+
module Functions
|
18
|
+
def rails_env
|
19
|
+
RAILS_ENV if defined? RAILS_ENV
|
20
|
+
end
|
21
|
+
module_function :rails_env
|
22
|
+
|
23
|
+
def disable_async?
|
24
|
+
# for predictable tests we disable delayed insert during testing
|
25
|
+
rails_env()=="test"
|
26
|
+
end
|
27
|
+
module_function :disable_async?
|
28
|
+
|
29
|
+
def check_options(permitted, options)
|
30
|
+
unknown_keys = options.keys.to_set - permitted
|
31
|
+
raise "unknown options: #{unknown_keys.to_a.join(", ")}" if !unknown_keys.empty?
|
32
|
+
end
|
33
|
+
module_function :check_options
|
34
|
+
|
35
|
+
# slice an array into smaller arrays with maximum size max_size
|
36
|
+
def slice_array(max_length, arr)
|
37
|
+
slices = []
|
38
|
+
(0..arr.length-1).step( max_length ){ |i| slices << arr.slice(i,max_length) }
|
39
|
+
slices
|
40
|
+
end
|
41
|
+
module_function :slice_array
|
42
|
+
|
43
|
+
def add_columns(klass, values, options)
|
44
|
+
columns = options[:columns] || klass.columns.map(&:name)
|
45
|
+
|
46
|
+
# add a :type column automatically for STI, if not already present
|
47
|
+
if klass.superclass!=ActiveRecord::Base && !columns.include?(:type)
|
48
|
+
columns = [:type, *columns]
|
49
|
+
values = values.map{|vals| [klass.to_s, *vals]}
|
50
|
+
end
|
51
|
+
|
52
|
+
[columns, values]
|
53
|
+
end
|
54
|
+
module_function :add_columns
|
55
|
+
end
|
56
|
+
|
57
|
+
module ClassMethods
|
58
|
+
def save_many_max_rows=(max_rows)
|
59
|
+
@save_many_max_rows=max_rows
|
60
|
+
end
|
61
|
+
|
62
|
+
def save_many_max_rows
|
63
|
+
@save_many_max_rows || ActiveRecord::SaveMany::default_max_rows
|
64
|
+
end
|
65
|
+
|
66
|
+
def save_many(values, options={})
|
67
|
+
Functions::check_options(ActiveRecord::SaveMany::OPTIONS_KEYS , options)
|
68
|
+
return if values.nil? || values.empty?
|
69
|
+
|
70
|
+
columns, values = Functions::add_columns(self, values, options)
|
71
|
+
|
72
|
+
# if more than max_rows, execute multiple sql statements
|
73
|
+
max_rows = options[:max_rows] || save_many_max_rows
|
74
|
+
batches = Functions::slice_array(max_rows, values)
|
75
|
+
|
76
|
+
batches.each do |rows|
|
77
|
+
rows = rows.map do |obj|
|
78
|
+
if obj.is_a? ActiveRecord::Base
|
79
|
+
obj.send( :callback, :before_save )
|
80
|
+
if obj.id
|
81
|
+
obj.send( :callback, :before_update)
|
82
|
+
else
|
83
|
+
obj.send( :callback, :before_create )
|
84
|
+
end
|
85
|
+
raise "#{obj.errors.full_messages.join(', ')}" if !obj.valid?
|
86
|
+
end
|
87
|
+
columns.map{|col| obj[col]}
|
88
|
+
end
|
89
|
+
|
90
|
+
sql = connection.save_many_sql(self,
|
91
|
+
table_name,
|
92
|
+
columns,
|
93
|
+
rows,
|
94
|
+
{ :ignore=>options[:ignore],
|
95
|
+
:async=>options[:async] && !Functions::disable_async?(),
|
96
|
+
:update=>options[:update] || options[:updates],
|
97
|
+
:updates=>options[:updates] || {} })
|
98
|
+
|
99
|
+
connection.execute_raw sql
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
module MySQL
|
105
|
+
def save_many_sql(klass, table_name, columns, rows, options)
|
106
|
+
Functions::check_options(SQL_OPTIONS_KEYS, options)
|
107
|
+
|
108
|
+
sql = ["insert", ("delayed" if options[:async]), ("ignore" if options[:ignore])].compact.join(' ') +
|
109
|
+
" into #{table_name} (#{columns.join(',')}) values " +
|
110
|
+
rows.map{|vals| "(" + vals.map{|v| klass.quote_value(v)}.join(",") +")"}.join(",") +
|
111
|
+
(" on duplicate key update "+columns.map{|c| options[:updates][c] || "#{c}=values(#{c})"}.join(",") if options[:update]).to_s
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
class Base
|
117
|
+
class << self
|
118
|
+
include ActiveRecord::SaveMany::ClassMethods
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
module JdbcSpec
|
124
|
+
module MySQL
|
125
|
+
include ActiveRecord::SaveMany::MySQL
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
module ActiveRecord
|
130
|
+
module ConnectionAdapters
|
131
|
+
class MysqlAdapter
|
132
|
+
include ActiveRecord::SaveMany::MySQL
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord_save_many
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 6
|
8
|
+
- 1
|
9
|
+
version: 0.6.1
|
5
10
|
platform: ruby
|
6
11
|
authors:
|
7
12
|
- mccraigmccraig
|
@@ -9,49 +14,63 @@ autorequire:
|
|
9
14
|
bindir: bin
|
10
15
|
cert_chain: []
|
11
16
|
|
12
|
-
date: 2010-
|
17
|
+
date: 2010-03-11 00:00:00 +00:00
|
13
18
|
default_executable:
|
14
19
|
dependencies:
|
15
20
|
- !ruby/object:Gem::Dependency
|
16
21
|
name: activerecord_execute_raw
|
17
|
-
|
18
|
-
|
19
|
-
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
20
24
|
requirements:
|
21
25
|
- - ">="
|
22
26
|
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
- 1
|
30
|
+
- 0
|
23
31
|
version: 0.1.0
|
24
|
-
|
32
|
+
type: :runtime
|
33
|
+
version_requirements: *id001
|
25
34
|
- !ruby/object:Gem::Dependency
|
26
35
|
name: rspec
|
27
|
-
|
28
|
-
|
29
|
-
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
prerelease: false
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
30
38
|
requirements:
|
31
39
|
- - ">="
|
32
40
|
- !ruby/object:Gem::Version
|
41
|
+
segments:
|
42
|
+
- 1
|
43
|
+
- 2
|
44
|
+
- 9
|
33
45
|
version: 1.2.9
|
34
|
-
|
46
|
+
type: :development
|
47
|
+
version_requirements: *id002
|
35
48
|
- !ruby/object:Gem::Dependency
|
36
49
|
name: yard
|
37
|
-
|
38
|
-
|
39
|
-
version_requirements: !ruby/object:Gem::Requirement
|
50
|
+
prerelease: false
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
40
52
|
requirements:
|
41
53
|
- - ">="
|
42
54
|
- !ruby/object:Gem::Version
|
55
|
+
segments:
|
56
|
+
- 0
|
43
57
|
version: "0"
|
44
|
-
|
58
|
+
type: :development
|
59
|
+
version_requirements: *id003
|
45
60
|
- !ruby/object:Gem::Dependency
|
46
61
|
name: rr
|
47
|
-
|
48
|
-
|
49
|
-
version_requirements: !ruby/object:Gem::Requirement
|
62
|
+
prerelease: false
|
63
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
50
64
|
requirements:
|
51
65
|
- - ">="
|
52
66
|
- !ruby/object:Gem::Version
|
67
|
+
segments:
|
68
|
+
- 0
|
69
|
+
- 10
|
70
|
+
- 5
|
53
71
|
version: 0.10.5
|
54
|
-
|
72
|
+
type: :development
|
73
|
+
version_requirements: *id004
|
55
74
|
description: adds save_many method to ActiveRecord classes for efficient bulk insert and update operations
|
56
75
|
email: craig@trampolinesystems.com
|
57
76
|
executables: []
|
@@ -69,6 +88,7 @@ files:
|
|
69
88
|
- Rakefile
|
70
89
|
- VERSION
|
71
90
|
- lib/activerecord_save_many.rb
|
91
|
+
- lib/activerecord_save_many/save_many.rb
|
72
92
|
- spec/activerecord_save_many_spec.rb
|
73
93
|
- spec/spec.opts
|
74
94
|
- spec/spec_helper.rb
|
@@ -85,18 +105,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
85
105
|
requirements:
|
86
106
|
- - ">="
|
87
107
|
- !ruby/object:Gem::Version
|
108
|
+
segments:
|
109
|
+
- 0
|
88
110
|
version: "0"
|
89
|
-
version:
|
90
111
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
91
112
|
requirements:
|
92
113
|
- - ">="
|
93
114
|
- !ruby/object:Gem::Version
|
115
|
+
segments:
|
116
|
+
- 0
|
94
117
|
version: "0"
|
95
|
-
version:
|
96
118
|
requirements: []
|
97
119
|
|
98
120
|
rubyforge_project:
|
99
|
-
rubygems_version: 1.3.
|
121
|
+
rubygems_version: 1.3.6
|
100
122
|
signing_key:
|
101
123
|
specification_version: 3
|
102
124
|
summary: efficient bulk inserts and updates for ActiveRecord
|