activerecord-import 0.14.1 → 1.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +74 -8
- data/Brewfile +3 -1
- data/CHANGELOG.md +448 -2
- data/Gemfile +26 -19
- data/LICENSE +21 -56
- data/README.markdown +568 -32
- data/Rakefile +5 -1
- data/activerecord-import.gemspec +8 -7
- data/benchmarks/README +2 -2
- data/benchmarks/benchmark.rb +8 -1
- data/benchmarks/lib/base.rb +2 -0
- data/benchmarks/lib/cli_parser.rb +5 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/gemfiles/4.2.gemfile +4 -3
- data/gemfiles/5.0.gemfile +4 -3
- data/gemfiles/5.1.gemfile +4 -0
- data/gemfiles/5.2.gemfile +4 -0
- data/gemfiles/6.0.gemfile +4 -0
- data/gemfiles/6.1.gemfile +4 -0
- data/gemfiles/7.0.gemfile +4 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +12 -16
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +35 -18
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +107 -28
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +141 -18
- data/lib/activerecord-import/base.rb +15 -8
- data/lib/activerecord-import/import.rb +642 -178
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +5 -3
- data/lib/activerecord-import/value_sets_parser.rb +29 -3
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +5 -16
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +3 -0
- data/test/adapters/makara_postgis.rb +3 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +498 -32
- data/test/jdbcmysql/import_test.rb +2 -1
- data/test/jdbcpostgresql/import_test.rb +2 -1
- data/test/jdbcsqlite3/import_test.rb +6 -0
- data/test/makara_postgis/import_test.rb +10 -0
- data/test/models/account.rb +5 -0
- data/test/models/alarm.rb +4 -0
- data/test/models/animal.rb +8 -0
- data/test/models/bike_maker.rb +9 -0
- data/test/models/book.rb +4 -0
- data/test/models/car.rb +5 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +8 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +6 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +8 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +7 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +16 -0
- data/test/models/user.rb +5 -0
- data/test/models/user_token.rb +6 -0
- data/test/models/vendor.rb +9 -0
- data/test/models/widget.rb +18 -0
- data/test/mysql2/import_test.rb +2 -0
- data/test/mysql2_makara/import_test.rb +2 -0
- data/test/mysqlspatial2/import_test.rb +2 -0
- data/test/postgis/import_test.rb +6 -0
- data/test/postgresql/import_test.rb +2 -4
- data/test/schema/generic_schema.rb +88 -3
- data/test/schema/jdbcpostgresql_schema.rb +3 -0
- data/test/schema/mysql2_schema.rb +21 -0
- data/test/schema/postgis_schema.rb +3 -0
- data/test/schema/postgresql_schema.rb +63 -0
- data/test/schema/sqlite3_schema.rb +15 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -50
- data/test/support/active_support/test_case_extensions.rb +8 -1
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +17 -8
- data/test/support/generate.rb +10 -8
- data/test/support/mysql/import_examples.rb +17 -3
- data/test/support/postgresql/import_examples.rb +442 -9
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +45 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +278 -1
- data/test/support/shared_examples/recursive_import.rb +137 -12
- data/test/support/sqlite3/import_examples.rb +232 -0
- data/test/synchronize_test.rb +10 -0
- data/test/test_helper.rb +44 -3
- data/test/value_sets_bytes_parser_test.rb +15 -2
- data/test/value_sets_records_parser_test.rb +2 -0
- metadata +74 -22
- data/.travis.yml +0 -52
- data/gemfiles/3.2.gemfile +0 -3
- data/gemfiles/4.0.gemfile +0 -3
- data/gemfiles/4.1.gemfile +0 -3
- data/test/schema/mysql_schema.rb +0 -16
@@ -0,0 +1,232 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
def should_support_sqlite3_import_functionality
|
4
|
+
if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
|
5
|
+
should_support_sqlite_upsert_functionality
|
6
|
+
end
|
7
|
+
|
8
|
+
describe "#supports_imports?" do
|
9
|
+
it "should support import" do
|
10
|
+
assert ActiveRecord::Base.supports_import?
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe "#import" do
|
15
|
+
it "imports with a single insert on SQLite 3.7.11 or higher" do
|
16
|
+
assert_difference "Topic.count", +507 do
|
17
|
+
result = Topic.import Build(7, :topics)
|
18
|
+
assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
19
|
+
assert_equal 7, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
20
|
+
|
21
|
+
result = Topic.import Build(500, :topics)
|
22
|
+
assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
23
|
+
assert_equal 507, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it "imports with a two inserts on SQLite 3.7.11 or higher" do
|
28
|
+
assert_difference "Topic.count", +501 do
|
29
|
+
result = Topic.import Build(501, :topics)
|
30
|
+
assert_equal 2, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
31
|
+
assert_equal 501, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it "imports with a five inserts on SQLite 3.7.11 or higher" do
|
36
|
+
assert_difference "Topic.count", +2500 do
|
37
|
+
result = Topic.import Build(2500, :topics)
|
38
|
+
assert_equal 5, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
39
|
+
assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def should_support_sqlite_upsert_functionality
|
46
|
+
should_support_basic_on_duplicate_key_update
|
47
|
+
should_support_on_duplicate_key_ignore
|
48
|
+
|
49
|
+
describe "#import" do
|
50
|
+
extend ActiveSupport::TestCase::ImportAssertions
|
51
|
+
|
52
|
+
macro(:perform_import) { raise "supply your own #perform_import in a context below" }
|
53
|
+
macro(:updated_topic) { Topic.find(@topic.id) }
|
54
|
+
|
55
|
+
context "with :on_duplicate_key_ignore and validation checks turned off" do
|
56
|
+
let(:columns) { %w( id title author_name author_email_address parent_id ) }
|
57
|
+
let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
|
58
|
+
let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
|
59
|
+
|
60
|
+
setup do
|
61
|
+
Topic.import columns, values, validate: false
|
62
|
+
end
|
63
|
+
|
64
|
+
it "should not update any records" do
|
65
|
+
result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
|
66
|
+
assert_equal [], result.ids
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context "with :on_duplicate_key_update and validation checks turned off" do
|
71
|
+
asssertion_group(:should_support_on_duplicate_key_update) do
|
72
|
+
should_not_update_fields_not_mentioned
|
73
|
+
should_update_foreign_keys
|
74
|
+
should_not_update_created_at_on_timestamp_columns
|
75
|
+
should_update_updated_at_on_timestamp_columns
|
76
|
+
end
|
77
|
+
|
78
|
+
context "using a hash" do
|
79
|
+
context "with :columns a hash" do
|
80
|
+
let(:columns) { %w( id title author_name author_email_address parent_id ) }
|
81
|
+
let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
|
82
|
+
let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
|
83
|
+
|
84
|
+
macro(:perform_import) do |*opts|
|
85
|
+
Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
|
86
|
+
end
|
87
|
+
|
88
|
+
setup do
|
89
|
+
Topic.import columns, values, validate: false
|
90
|
+
@topic = Topic.find 99
|
91
|
+
end
|
92
|
+
|
93
|
+
it "should not modify the passed in :on_duplicate_key_update columns array" do
|
94
|
+
assert_nothing_raised do
|
95
|
+
columns = %w(title author_name).freeze
|
96
|
+
Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context "using string hash map" do
|
101
|
+
let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
|
102
|
+
should_support_on_duplicate_key_update
|
103
|
+
should_update_fields_mentioned
|
104
|
+
end
|
105
|
+
|
106
|
+
context "using string hash map, but specifying column mismatches" do
|
107
|
+
let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
|
108
|
+
should_support_on_duplicate_key_update
|
109
|
+
should_update_fields_mentioned_with_hash_mappings
|
110
|
+
end
|
111
|
+
|
112
|
+
context "using symbol hash map" do
|
113
|
+
let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
|
114
|
+
should_support_on_duplicate_key_update
|
115
|
+
should_update_fields_mentioned
|
116
|
+
end
|
117
|
+
|
118
|
+
context "using symbol hash map, but specifying column mismatches" do
|
119
|
+
let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
|
120
|
+
should_support_on_duplicate_key_update
|
121
|
+
should_update_fields_mentioned_with_hash_mappings
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
context 'with :index_predicate' do
|
126
|
+
let(:columns) { %w( id device_id alarm_type status metadata ) }
|
127
|
+
let(:values) { [[99, 17, 1, 1, 'foo']] }
|
128
|
+
let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
|
129
|
+
|
130
|
+
macro(:perform_import) do |*opts|
|
131
|
+
Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
|
132
|
+
end
|
133
|
+
|
134
|
+
macro(:updated_alarm) { Alarm.find(@alarm.id) }
|
135
|
+
|
136
|
+
setup do
|
137
|
+
Alarm.import columns, values, validate: false
|
138
|
+
@alarm = Alarm.find 99
|
139
|
+
end
|
140
|
+
|
141
|
+
context 'supports on duplicate key update for partial indexes' do
|
142
|
+
it 'should not update created_at timestamp columns' do
|
143
|
+
Timecop.freeze Chronic.parse("5 minutes from now") do
|
144
|
+
perform_import
|
145
|
+
assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
it 'should update updated_at timestamp columns' do
|
150
|
+
time = Chronic.parse("5 minutes from now")
|
151
|
+
Timecop.freeze time do
|
152
|
+
perform_import
|
153
|
+
assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
it 'should not update fields not mentioned' do
|
158
|
+
perform_import
|
159
|
+
assert_equal 'foo', updated_alarm.metadata
|
160
|
+
end
|
161
|
+
|
162
|
+
it 'should update fields mentioned with hash mappings' do
|
163
|
+
perform_import
|
164
|
+
assert_equal 2, updated_alarm.status
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
context 'with :condition' do
|
170
|
+
let(:columns) { %w( id device_id alarm_type status metadata) }
|
171
|
+
let(:values) { [[99, 17, 1, 1, 'foo']] }
|
172
|
+
let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
|
173
|
+
|
174
|
+
macro(:perform_import) do |*opts|
|
175
|
+
Alarm.import(
|
176
|
+
columns,
|
177
|
+
updated_values,
|
178
|
+
opts.extract_options!.merge(
|
179
|
+
on_duplicate_key_update: {
|
180
|
+
conflict_target: [:id],
|
181
|
+
condition: "alarms.metadata NOT LIKE '%foo%'",
|
182
|
+
columns: [:metadata]
|
183
|
+
},
|
184
|
+
validate: false
|
185
|
+
)
|
186
|
+
)
|
187
|
+
end
|
188
|
+
|
189
|
+
macro(:updated_alarm) { Alarm.find(@alarm.id) }
|
190
|
+
|
191
|
+
setup do
|
192
|
+
Alarm.import columns, values, validate: false
|
193
|
+
@alarm = Alarm.find 99
|
194
|
+
end
|
195
|
+
|
196
|
+
it 'should not update fields not matched' do
|
197
|
+
perform_import
|
198
|
+
assert_equal 'foo', updated_alarm.metadata
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
context "with no :conflict_target" do
|
203
|
+
context "with no primary key" do
|
204
|
+
it "raises ArgumentError" do
|
205
|
+
error = assert_raises ArgumentError do
|
206
|
+
Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
|
207
|
+
end
|
208
|
+
assert_match(/Expected :conflict_target to be specified/, error.message)
|
209
|
+
end
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
context "with no :columns" do
|
214
|
+
let(:columns) { %w( id title author_name author_email_address ) }
|
215
|
+
let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
|
216
|
+
let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
|
217
|
+
|
218
|
+
macro(:perform_import) do |*opts|
|
219
|
+
Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
|
220
|
+
end
|
221
|
+
|
222
|
+
setup do
|
223
|
+
Topic.import columns, values, validate: false
|
224
|
+
@topic = Topic.find 100
|
225
|
+
end
|
226
|
+
|
227
|
+
should_update_updated_at_on_timestamp_columns
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|
data/test/synchronize_test.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require File.expand_path('../test_helper', __FILE__)
|
2
4
|
|
3
5
|
describe ".synchronize" do
|
@@ -30,4 +32,12 @@ describe ".synchronize" do
|
|
30
32
|
assert_equal false, topics[1].changed?, "the second record was dirty"
|
31
33
|
assert_equal false, topics[2].changed?, "the third record was dirty"
|
32
34
|
end
|
35
|
+
|
36
|
+
it "ignores default scope" do
|
37
|
+
# update records outside of ActiveRecord knowing about it
|
38
|
+
Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
|
39
|
+
|
40
|
+
Topic.synchronize topics
|
41
|
+
assert_equal false, topics[0].approved
|
42
|
+
end
|
33
43
|
end
|
data/test/test_helper.rb
CHANGED
@@ -1,4 +1,7 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'pathname'
|
4
|
+
require 'rake'
|
2
5
|
test_dir = Pathname.new File.dirname(__FILE__)
|
3
6
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
7
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
@@ -18,13 +21,29 @@ require "active_support/test_case"
|
|
18
21
|
|
19
22
|
if ActiveSupport::VERSION::STRING < "4.0"
|
20
23
|
require 'test/unit'
|
24
|
+
require 'mocha/test_unit'
|
21
25
|
else
|
22
26
|
require 'active_support/testing/autorun'
|
27
|
+
require "mocha/mini_test"
|
23
28
|
end
|
24
29
|
|
25
30
|
require 'timecop'
|
26
31
|
require 'chronic'
|
27
32
|
|
33
|
+
begin
|
34
|
+
require 'composite_primary_keys'
|
35
|
+
rescue LoadError
|
36
|
+
ENV["SKIP_COMPOSITE_PK"] = "true"
|
37
|
+
end
|
38
|
+
|
39
|
+
# Support MySQL 5.7
|
40
|
+
if ActiveSupport::VERSION::STRING < "4.1"
|
41
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
42
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
43
|
+
NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
28
47
|
require "ruby-debug" if RUBY_VERSION.to_f < 1.9
|
29
48
|
|
30
49
|
adapter = ENV["ARE_DB"] || "sqlite3"
|
@@ -32,8 +51,30 @@ adapter = ENV["ARE_DB"] || "sqlite3"
|
|
32
51
|
FileUtils.mkdir_p 'log'
|
33
52
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
34
53
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
35
|
-
|
36
|
-
ActiveRecord
|
54
|
+
|
55
|
+
if ActiveRecord.respond_to?(:use_yaml_unsafe_load)
|
56
|
+
ActiveRecord.use_yaml_unsafe_load = true
|
57
|
+
elsif ActiveRecord::Base.respond_to?(:use_yaml_unsafe_load)
|
58
|
+
ActiveRecord::Base.use_yaml_unsafe_load = true
|
59
|
+
end
|
60
|
+
|
61
|
+
if ENV['AR_VERSION'].to_f >= 6.0
|
62
|
+
yaml_config = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new('3.2.1')
|
63
|
+
YAML.safe_load_file(test_dir.join("database.yml"), aliases: true)[adapter]
|
64
|
+
else
|
65
|
+
YAML.load_file(test_dir.join("database.yml"))[adapter]
|
66
|
+
end
|
67
|
+
config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
|
68
|
+
ActiveRecord::Base.configurations.configurations << config
|
69
|
+
else
|
70
|
+
ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
71
|
+
end
|
72
|
+
|
73
|
+
if ActiveRecord.respond_to?(:default_timezone)
|
74
|
+
ActiveRecord.default_timezone = :utc
|
75
|
+
else
|
76
|
+
ActiveRecord::Base.default_timezone = :utc
|
77
|
+
end
|
37
78
|
|
38
79
|
require "activerecord-import"
|
39
80
|
ActiveRecord::Base.establish_connection :test
|
@@ -42,7 +83,7 @@ ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
|
|
42
83
|
ActiveRecord::Base.logger.info hsh[:sql]
|
43
84
|
end
|
44
85
|
|
45
|
-
require "
|
86
|
+
require "factory_bot"
|
46
87
|
Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
|
47
88
|
|
48
89
|
# Load base/generic schema
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require File.expand_path(File.dirname(__FILE__) + '/test_helper')
|
2
4
|
|
3
5
|
require 'activerecord-import/value_sets_parser'
|
@@ -8,6 +10,15 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
8
10
|
let(:base_sql) { "INSERT INTO atable (a,b,c)" }
|
9
11
|
let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
|
10
12
|
|
13
|
+
context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
|
14
|
+
it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
|
15
|
+
error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
|
16
|
+
parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
|
17
|
+
end
|
18
|
+
assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
11
22
|
context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
|
12
23
|
it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
|
13
24
|
value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
|
@@ -54,7 +65,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
54
65
|
values = [
|
55
66
|
"('1','2','3')",
|
56
67
|
"('4','5','6')",
|
57
|
-
"('7','8','9')"
|
68
|
+
"('7','8','9')"
|
69
|
+
]
|
58
70
|
|
59
71
|
base_sql_size_in_bytes = 15
|
60
72
|
max_bytes = 30
|
@@ -79,7 +91,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
79
91
|
# each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
|
80
92
|
values = [
|
81
93
|
"('é')",
|
82
|
-
"('é')"
|
94
|
+
"('é')"
|
95
|
+
]
|
83
96
|
|
84
97
|
base_sql_size_in_bytes = 15
|
85
98
|
max_bytes = 26
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: activerecord-import
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zach Dennis
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-10-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '4.2'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
26
|
+
version: '4.2'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: rake
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,18 +38,17 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
-
description:
|
42
|
-
for Rails 3 and beyond
|
41
|
+
description: A library for bulk inserting data using ActiveRecord.
|
43
42
|
email:
|
44
43
|
- zach.dennis@gmail.com
|
45
44
|
executables: []
|
46
45
|
extensions: []
|
47
46
|
extra_rdoc_files: []
|
48
47
|
files:
|
48
|
+
- ".github/workflows/test.yaml"
|
49
49
|
- ".gitignore"
|
50
50
|
- ".rubocop.yml"
|
51
51
|
- ".rubocop_todo.yml"
|
52
|
-
- ".travis.yml"
|
53
52
|
- Brewfile
|
54
53
|
- CHANGELOG.md
|
55
54
|
- Gemfile
|
@@ -68,16 +67,19 @@ files:
|
|
68
67
|
- benchmarks/models/test_innodb.rb
|
69
68
|
- benchmarks/models/test_memory.rb
|
70
69
|
- benchmarks/models/test_myisam.rb
|
71
|
-
- benchmarks/schema/
|
72
|
-
- gemfiles/3.2.gemfile
|
73
|
-
- gemfiles/4.0.gemfile
|
74
|
-
- gemfiles/4.1.gemfile
|
70
|
+
- benchmarks/schema/mysql2_schema.rb
|
75
71
|
- gemfiles/4.2.gemfile
|
76
72
|
- gemfiles/5.0.gemfile
|
73
|
+
- gemfiles/5.1.gemfile
|
74
|
+
- gemfiles/5.2.gemfile
|
75
|
+
- gemfiles/6.0.gemfile
|
76
|
+
- gemfiles/6.1.gemfile
|
77
|
+
- gemfiles/7.0.gemfile
|
77
78
|
- lib/activerecord-import.rb
|
78
79
|
- lib/activerecord-import/active_record/adapters/abstract_adapter.rb
|
79
80
|
- lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
|
80
81
|
- lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb
|
82
|
+
- lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb
|
81
83
|
- lib/activerecord-import/active_record/adapters/mysql2_adapter.rb
|
82
84
|
- lib/activerecord-import/active_record/adapters/postgresql_adapter.rb
|
83
85
|
- lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb
|
@@ -98,6 +100,8 @@ files:
|
|
98
100
|
- lib/activerecord-import/version.rb
|
99
101
|
- test/adapters/jdbcmysql.rb
|
100
102
|
- test/adapters/jdbcpostgresql.rb
|
103
|
+
- test/adapters/jdbcsqlite3.rb
|
104
|
+
- test/adapters/makara_postgis.rb
|
101
105
|
- test/adapters/mysql2.rb
|
102
106
|
- test/adapters/mysql2_makara.rb
|
103
107
|
- test/adapters/mysql2spatial.rb
|
@@ -108,18 +112,37 @@ files:
|
|
108
112
|
- test/adapters/spatialite.rb
|
109
113
|
- test/adapters/sqlite3.rb
|
110
114
|
- test/database.yml.sample
|
115
|
+
- test/github/database.yml
|
111
116
|
- test/import_test.rb
|
112
117
|
- test/jdbcmysql/import_test.rb
|
113
118
|
- test/jdbcpostgresql/import_test.rb
|
119
|
+
- test/jdbcsqlite3/import_test.rb
|
120
|
+
- test/makara_postgis/import_test.rb
|
121
|
+
- test/models/account.rb
|
122
|
+
- test/models/alarm.rb
|
123
|
+
- test/models/animal.rb
|
124
|
+
- test/models/bike_maker.rb
|
114
125
|
- test/models/book.rb
|
126
|
+
- test/models/car.rb
|
127
|
+
- test/models/card.rb
|
115
128
|
- test/models/chapter.rb
|
129
|
+
- test/models/customer.rb
|
130
|
+
- test/models/deck.rb
|
131
|
+
- test/models/dictionary.rb
|
116
132
|
- test/models/discount.rb
|
117
133
|
- test/models/end_note.rb
|
118
134
|
- test/models/group.rb
|
135
|
+
- test/models/order.rb
|
136
|
+
- test/models/playing_card.rb
|
119
137
|
- test/models/promotion.rb
|
120
138
|
- test/models/question.rb
|
121
139
|
- test/models/rule.rb
|
140
|
+
- test/models/tag.rb
|
141
|
+
- test/models/tag_alias.rb
|
122
142
|
- test/models/topic.rb
|
143
|
+
- test/models/user.rb
|
144
|
+
- test/models/user_token.rb
|
145
|
+
- test/models/vendor.rb
|
123
146
|
- test/models/widget.rb
|
124
147
|
- test/mysql2/import_test.rb
|
125
148
|
- test/mysql2_makara/import_test.rb
|
@@ -127,7 +150,11 @@ files:
|
|
127
150
|
- test/postgis/import_test.rb
|
128
151
|
- test/postgresql/import_test.rb
|
129
152
|
- test/schema/generic_schema.rb
|
130
|
-
- test/schema/
|
153
|
+
- test/schema/jdbcpostgresql_schema.rb
|
154
|
+
- test/schema/mysql2_schema.rb
|
155
|
+
- test/schema/postgis_schema.rb
|
156
|
+
- test/schema/postgresql_schema.rb
|
157
|
+
- test/schema/sqlite3_schema.rb
|
131
158
|
- test/schema/version.rb
|
132
159
|
- test/sqlite3/import_test.rb
|
133
160
|
- test/support/active_support/test_case_extensions.rb
|
@@ -136,16 +163,17 @@ files:
|
|
136
163
|
- test/support/generate.rb
|
137
164
|
- test/support/mysql/import_examples.rb
|
138
165
|
- test/support/postgresql/import_examples.rb
|
166
|
+
- test/support/shared_examples/on_duplicate_key_ignore.rb
|
139
167
|
- test/support/shared_examples/on_duplicate_key_update.rb
|
140
168
|
- test/support/shared_examples/recursive_import.rb
|
169
|
+
- test/support/sqlite3/import_examples.rb
|
141
170
|
- test/synchronize_test.rb
|
142
171
|
- test/test_helper.rb
|
143
|
-
- test/travis/database.yml
|
144
172
|
- test/value_sets_bytes_parser_test.rb
|
145
173
|
- test/value_sets_records_parser_test.rb
|
146
|
-
homepage:
|
174
|
+
homepage: https://github.com/zdennis/activerecord-import
|
147
175
|
licenses:
|
148
|
-
-
|
176
|
+
- MIT
|
149
177
|
metadata: {}
|
150
178
|
post_install_message:
|
151
179
|
rdoc_options: []
|
@@ -155,21 +183,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
155
183
|
requirements:
|
156
184
|
- - ">="
|
157
185
|
- !ruby/object:Gem::Version
|
158
|
-
version:
|
186
|
+
version: 2.4.0
|
159
187
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
160
188
|
requirements:
|
161
189
|
- - ">="
|
162
190
|
- !ruby/object:Gem::Version
|
163
191
|
version: '0'
|
164
192
|
requirements: []
|
165
|
-
|
166
|
-
rubygems_version: 2.5.1
|
193
|
+
rubygems_version: 3.0.3.1
|
167
194
|
signing_key:
|
168
195
|
specification_version: 4
|
169
|
-
summary: Bulk
|
196
|
+
summary: Bulk insert extension for ActiveRecord
|
170
197
|
test_files:
|
171
198
|
- test/adapters/jdbcmysql.rb
|
172
199
|
- test/adapters/jdbcpostgresql.rb
|
200
|
+
- test/adapters/jdbcsqlite3.rb
|
201
|
+
- test/adapters/makara_postgis.rb
|
173
202
|
- test/adapters/mysql2.rb
|
174
203
|
- test/adapters/mysql2_makara.rb
|
175
204
|
- test/adapters/mysql2spatial.rb
|
@@ -180,18 +209,37 @@ test_files:
|
|
180
209
|
- test/adapters/spatialite.rb
|
181
210
|
- test/adapters/sqlite3.rb
|
182
211
|
- test/database.yml.sample
|
212
|
+
- test/github/database.yml
|
183
213
|
- test/import_test.rb
|
184
214
|
- test/jdbcmysql/import_test.rb
|
185
215
|
- test/jdbcpostgresql/import_test.rb
|
216
|
+
- test/jdbcsqlite3/import_test.rb
|
217
|
+
- test/makara_postgis/import_test.rb
|
218
|
+
- test/models/account.rb
|
219
|
+
- test/models/alarm.rb
|
220
|
+
- test/models/animal.rb
|
221
|
+
- test/models/bike_maker.rb
|
186
222
|
- test/models/book.rb
|
223
|
+
- test/models/car.rb
|
224
|
+
- test/models/card.rb
|
187
225
|
- test/models/chapter.rb
|
226
|
+
- test/models/customer.rb
|
227
|
+
- test/models/deck.rb
|
228
|
+
- test/models/dictionary.rb
|
188
229
|
- test/models/discount.rb
|
189
230
|
- test/models/end_note.rb
|
190
231
|
- test/models/group.rb
|
232
|
+
- test/models/order.rb
|
233
|
+
- test/models/playing_card.rb
|
191
234
|
- test/models/promotion.rb
|
192
235
|
- test/models/question.rb
|
193
236
|
- test/models/rule.rb
|
237
|
+
- test/models/tag.rb
|
238
|
+
- test/models/tag_alias.rb
|
194
239
|
- test/models/topic.rb
|
240
|
+
- test/models/user.rb
|
241
|
+
- test/models/user_token.rb
|
242
|
+
- test/models/vendor.rb
|
195
243
|
- test/models/widget.rb
|
196
244
|
- test/mysql2/import_test.rb
|
197
245
|
- test/mysql2_makara/import_test.rb
|
@@ -199,7 +247,11 @@ test_files:
|
|
199
247
|
- test/postgis/import_test.rb
|
200
248
|
- test/postgresql/import_test.rb
|
201
249
|
- test/schema/generic_schema.rb
|
202
|
-
- test/schema/
|
250
|
+
- test/schema/jdbcpostgresql_schema.rb
|
251
|
+
- test/schema/mysql2_schema.rb
|
252
|
+
- test/schema/postgis_schema.rb
|
253
|
+
- test/schema/postgresql_schema.rb
|
254
|
+
- test/schema/sqlite3_schema.rb
|
203
255
|
- test/schema/version.rb
|
204
256
|
- test/sqlite3/import_test.rb
|
205
257
|
- test/support/active_support/test_case_extensions.rb
|
@@ -208,11 +260,11 @@ test_files:
|
|
208
260
|
- test/support/generate.rb
|
209
261
|
- test/support/mysql/import_examples.rb
|
210
262
|
- test/support/postgresql/import_examples.rb
|
263
|
+
- test/support/shared_examples/on_duplicate_key_ignore.rb
|
211
264
|
- test/support/shared_examples/on_duplicate_key_update.rb
|
212
265
|
- test/support/shared_examples/recursive_import.rb
|
266
|
+
- test/support/sqlite3/import_examples.rb
|
213
267
|
- test/synchronize_test.rb
|
214
268
|
- test/test_helper.rb
|
215
|
-
- test/travis/database.yml
|
216
269
|
- test/value_sets_bytes_parser_test.rb
|
217
270
|
- test/value_sets_records_parser_test.rb
|
218
|
-
has_rdoc:
|
data/.travis.yml
DELETED
@@ -1,52 +0,0 @@
|
|
1
|
-
language: ruby
|
2
|
-
cache: bundler
|
3
|
-
rvm:
|
4
|
-
- 2.2.4
|
5
|
-
|
6
|
-
env:
|
7
|
-
global:
|
8
|
-
# https://github.com/discourse/discourse/blob/master/.travis.yml
|
9
|
-
- RUBY_GC_MALLOC_LIMIT=50000000
|
10
|
-
matrix:
|
11
|
-
- AR_VERSION=3.2
|
12
|
-
- AR_VERSION=4.0
|
13
|
-
- AR_VERSION=4.1
|
14
|
-
- AR_VERSION=4.2
|
15
|
-
- AR_VERSION=5.0
|
16
|
-
|
17
|
-
matrix:
|
18
|
-
include:
|
19
|
-
- rvm: jruby-9.0.5.0
|
20
|
-
env: AR_VERSION=4.2
|
21
|
-
script:
|
22
|
-
- bundle exec rake test:jdbcmysql
|
23
|
-
- bundle exec rake test:jdbcpostgresql
|
24
|
-
|
25
|
-
fast_finish: true
|
26
|
-
|
27
|
-
before_script:
|
28
|
-
- mysql -e 'create database activerecord_import_test;'
|
29
|
-
- psql -c 'create database activerecord_import_test;' -U postgres
|
30
|
-
- psql -U postgres -c "create extension postgis"
|
31
|
-
- cp test/travis/database.yml test/database.yml
|
32
|
-
|
33
|
-
addons:
|
34
|
-
apt:
|
35
|
-
sources:
|
36
|
-
- travis-ci/sqlite3
|
37
|
-
packages:
|
38
|
-
- sqlite3
|
39
|
-
|
40
|
-
script:
|
41
|
-
- bundle exec rake test:mysql2
|
42
|
-
- bundle exec rake test:mysql2_makara
|
43
|
-
- bundle exec rake test:mysql2spatial
|
44
|
-
- bundle exec rake test:postgis
|
45
|
-
- bundle exec rake test:postgresql
|
46
|
-
- bundle exec rake test:postgresql_makara
|
47
|
-
- bundle exec rake test:seamless_database_pool
|
48
|
-
- bundle exec rake test:spatialite
|
49
|
-
- bundle exec rake test:sqlite3
|
50
|
-
- bundle exec rubocop
|
51
|
-
|
52
|
-
sudo: false
|
data/gemfiles/3.2.gemfile
DELETED
data/gemfiles/4.0.gemfile
DELETED