activerecord-import 0.14.1 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/test.yaml +107 -0
- data/.gitignore +1 -0
- data/.rubocop.yml +74 -8
- data/Brewfile +3 -1
- data/CHANGELOG.md +448 -2
- data/Gemfile +26 -19
- data/LICENSE +21 -56
- data/README.markdown +568 -32
- data/Rakefile +5 -1
- data/activerecord-import.gemspec +8 -7
- data/benchmarks/README +2 -2
- data/benchmarks/benchmark.rb +8 -1
- data/benchmarks/lib/base.rb +2 -0
- data/benchmarks/lib/cli_parser.rb +5 -2
- data/benchmarks/lib/float.rb +2 -0
- data/benchmarks/lib/mysql2_benchmark.rb +2 -0
- data/benchmarks/lib/output_to_csv.rb +2 -0
- data/benchmarks/lib/output_to_html.rb +4 -2
- data/benchmarks/models/test_innodb.rb +2 -0
- data/benchmarks/models/test_memory.rb +2 -0
- data/benchmarks/models/test_myisam.rb +2 -0
- data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
- data/gemfiles/4.2.gemfile +4 -3
- data/gemfiles/5.0.gemfile +4 -3
- data/gemfiles/5.1.gemfile +4 -0
- data/gemfiles/5.2.gemfile +4 -0
- data/gemfiles/6.0.gemfile +4 -0
- data/gemfiles/6.1.gemfile +4 -0
- data/gemfiles/7.0.gemfile +4 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +8 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +12 -16
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +35 -18
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +107 -28
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +141 -18
- data/lib/activerecord-import/base.rb +15 -8
- data/lib/activerecord-import/import.rb +642 -178
- data/lib/activerecord-import/mysql2.rb +2 -0
- data/lib/activerecord-import/postgresql.rb +2 -0
- data/lib/activerecord-import/sqlite3.rb +2 -0
- data/lib/activerecord-import/synchronize.rb +5 -3
- data/lib/activerecord-import/value_sets_parser.rb +29 -3
- data/lib/activerecord-import/version.rb +3 -1
- data/lib/activerecord-import.rb +5 -16
- data/test/adapters/jdbcmysql.rb +2 -0
- data/test/adapters/jdbcpostgresql.rb +2 -0
- data/test/adapters/jdbcsqlite3.rb +3 -0
- data/test/adapters/makara_postgis.rb +3 -0
- data/test/adapters/mysql2.rb +2 -0
- data/test/adapters/mysql2_makara.rb +2 -0
- data/test/adapters/mysql2spatial.rb +2 -0
- data/test/adapters/postgis.rb +2 -0
- data/test/adapters/postgresql.rb +2 -0
- data/test/adapters/postgresql_makara.rb +2 -0
- data/test/adapters/seamless_database_pool.rb +2 -0
- data/test/adapters/spatialite.rb +2 -0
- data/test/adapters/sqlite3.rb +2 -0
- data/test/{travis → github}/database.yml +7 -1
- data/test/import_test.rb +498 -32
- data/test/jdbcmysql/import_test.rb +2 -1
- data/test/jdbcpostgresql/import_test.rb +2 -1
- data/test/jdbcsqlite3/import_test.rb +6 -0
- data/test/makara_postgis/import_test.rb +10 -0
- data/test/models/account.rb +5 -0
- data/test/models/alarm.rb +4 -0
- data/test/models/animal.rb +8 -0
- data/test/models/bike_maker.rb +9 -0
- data/test/models/book.rb +4 -0
- data/test/models/car.rb +5 -0
- data/test/models/card.rb +5 -0
- data/test/models/chapter.rb +2 -0
- data/test/models/customer.rb +8 -0
- data/test/models/deck.rb +8 -0
- data/test/models/dictionary.rb +6 -0
- data/test/models/discount.rb +2 -0
- data/test/models/end_note.rb +2 -0
- data/test/models/group.rb +2 -0
- data/test/models/order.rb +8 -0
- data/test/models/playing_card.rb +4 -0
- data/test/models/promotion.rb +2 -0
- data/test/models/question.rb +2 -0
- data/test/models/rule.rb +2 -0
- data/test/models/tag.rb +7 -0
- data/test/models/tag_alias.rb +5 -0
- data/test/models/topic.rb +16 -0
- data/test/models/user.rb +5 -0
- data/test/models/user_token.rb +6 -0
- data/test/models/vendor.rb +9 -0
- data/test/models/widget.rb +18 -0
- data/test/mysql2/import_test.rb +2 -0
- data/test/mysql2_makara/import_test.rb +2 -0
- data/test/mysqlspatial2/import_test.rb +2 -0
- data/test/postgis/import_test.rb +6 -0
- data/test/postgresql/import_test.rb +2 -4
- data/test/schema/generic_schema.rb +88 -3
- data/test/schema/jdbcpostgresql_schema.rb +3 -0
- data/test/schema/mysql2_schema.rb +21 -0
- data/test/schema/postgis_schema.rb +3 -0
- data/test/schema/postgresql_schema.rb +63 -0
- data/test/schema/sqlite3_schema.rb +15 -0
- data/test/schema/version.rb +2 -0
- data/test/sqlite3/import_test.rb +4 -50
- data/test/support/active_support/test_case_extensions.rb +8 -1
- data/test/support/assertions.rb +2 -0
- data/test/support/factories.rb +17 -8
- data/test/support/generate.rb +10 -8
- data/test/support/mysql/import_examples.rb +17 -3
- data/test/support/postgresql/import_examples.rb +442 -9
- data/test/support/shared_examples/on_duplicate_key_ignore.rb +45 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +278 -1
- data/test/support/shared_examples/recursive_import.rb +137 -12
- data/test/support/sqlite3/import_examples.rb +232 -0
- data/test/synchronize_test.rb +10 -0
- data/test/test_helper.rb +44 -3
- data/test/value_sets_bytes_parser_test.rb +15 -2
- data/test/value_sets_records_parser_test.rb +2 -0
- metadata +74 -22
- data/.travis.yml +0 -52
- data/gemfiles/3.2.gemfile +0 -3
- data/gemfiles/4.0.gemfile +0 -3
- data/gemfiles/4.1.gemfile +0 -3
- data/test/schema/mysql_schema.rb +0 -16
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
def should_support_sqlite3_import_functionality
|
|
4
|
+
if ActiveRecord::Base.connection.supports_on_duplicate_key_update?
|
|
5
|
+
should_support_sqlite_upsert_functionality
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
describe "#supports_imports?" do
|
|
9
|
+
it "should support import" do
|
|
10
|
+
assert ActiveRecord::Base.supports_import?
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
describe "#import" do
|
|
15
|
+
it "imports with a single insert on SQLite 3.7.11 or higher" do
|
|
16
|
+
assert_difference "Topic.count", +507 do
|
|
17
|
+
result = Topic.import Build(7, :topics)
|
|
18
|
+
assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
19
|
+
assert_equal 7, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
20
|
+
|
|
21
|
+
result = Topic.import Build(500, :topics)
|
|
22
|
+
assert_equal 1, result.num_inserts, "Failed to issue a single INSERT statement. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
23
|
+
assert_equal 507, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it "imports with a two inserts on SQLite 3.7.11 or higher" do
|
|
28
|
+
assert_difference "Topic.count", +501 do
|
|
29
|
+
result = Topic.import Build(501, :topics)
|
|
30
|
+
assert_equal 2, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
31
|
+
assert_equal 501, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it "imports with a five inserts on SQLite 3.7.11 or higher" do
|
|
36
|
+
assert_difference "Topic.count", +2500 do
|
|
37
|
+
result = Topic.import Build(2500, :topics)
|
|
38
|
+
assert_equal 5, result.num_inserts, "Failed to issue a two INSERT statements. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
39
|
+
assert_equal 2500, Topic.count, "Failed to insert all records. Make sure you have a supported version of SQLite3 (3.7.11 or higher) installed"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def should_support_sqlite_upsert_functionality
|
|
46
|
+
should_support_basic_on_duplicate_key_update
|
|
47
|
+
should_support_on_duplicate_key_ignore
|
|
48
|
+
|
|
49
|
+
describe "#import" do
|
|
50
|
+
extend ActiveSupport::TestCase::ImportAssertions
|
|
51
|
+
|
|
52
|
+
macro(:perform_import) { raise "supply your own #perform_import in a context below" }
|
|
53
|
+
macro(:updated_topic) { Topic.find(@topic.id) }
|
|
54
|
+
|
|
55
|
+
context "with :on_duplicate_key_ignore and validation checks turned off" do
|
|
56
|
+
let(:columns) { %w( id title author_name author_email_address parent_id ) }
|
|
57
|
+
let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
|
|
58
|
+
let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
|
|
59
|
+
|
|
60
|
+
setup do
|
|
61
|
+
Topic.import columns, values, validate: false
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
it "should not update any records" do
|
|
65
|
+
result = Topic.import columns, updated_values, on_duplicate_key_ignore: true, validate: false
|
|
66
|
+
assert_equal [], result.ids
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
context "with :on_duplicate_key_update and validation checks turned off" do
|
|
71
|
+
asssertion_group(:should_support_on_duplicate_key_update) do
|
|
72
|
+
should_not_update_fields_not_mentioned
|
|
73
|
+
should_update_foreign_keys
|
|
74
|
+
should_not_update_created_at_on_timestamp_columns
|
|
75
|
+
should_update_updated_at_on_timestamp_columns
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
context "using a hash" do
|
|
79
|
+
context "with :columns a hash" do
|
|
80
|
+
let(:columns) { %w( id title author_name author_email_address parent_id ) }
|
|
81
|
+
let(:values) { [[99, "Book", "John Doe", "john@doe.com", 17]] }
|
|
82
|
+
let(:updated_values) { [[99, "Book - 2nd Edition", "Author Should Not Change", "johndoe@example.com", 57]] }
|
|
83
|
+
|
|
84
|
+
macro(:perform_import) do |*opts|
|
|
85
|
+
Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id, columns: update_columns }, validate: false)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
setup do
|
|
89
|
+
Topic.import columns, values, validate: false
|
|
90
|
+
@topic = Topic.find 99
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
it "should not modify the passed in :on_duplicate_key_update columns array" do
|
|
94
|
+
assert_nothing_raised do
|
|
95
|
+
columns = %w(title author_name).freeze
|
|
96
|
+
Topic.import columns, [%w(foo, bar)], on_duplicate_key_update: { columns: columns }
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
context "using string hash map" do
|
|
101
|
+
let(:update_columns) { { "title" => "title", "author_email_address" => "author_email_address", "parent_id" => "parent_id" } }
|
|
102
|
+
should_support_on_duplicate_key_update
|
|
103
|
+
should_update_fields_mentioned
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
context "using string hash map, but specifying column mismatches" do
|
|
107
|
+
let(:update_columns) { { "title" => "author_email_address", "author_email_address" => "title", "parent_id" => "parent_id" } }
|
|
108
|
+
should_support_on_duplicate_key_update
|
|
109
|
+
should_update_fields_mentioned_with_hash_mappings
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
context "using symbol hash map" do
|
|
113
|
+
let(:update_columns) { { title: :title, author_email_address: :author_email_address, parent_id: :parent_id } }
|
|
114
|
+
should_support_on_duplicate_key_update
|
|
115
|
+
should_update_fields_mentioned
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
context "using symbol hash map, but specifying column mismatches" do
|
|
119
|
+
let(:update_columns) { { title: :author_email_address, author_email_address: :title, parent_id: :parent_id } }
|
|
120
|
+
should_support_on_duplicate_key_update
|
|
121
|
+
should_update_fields_mentioned_with_hash_mappings
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
context 'with :index_predicate' do
|
|
126
|
+
let(:columns) { %w( id device_id alarm_type status metadata ) }
|
|
127
|
+
let(:values) { [[99, 17, 1, 1, 'foo']] }
|
|
128
|
+
let(:updated_values) { [[99, 17, 1, 2, 'bar']] }
|
|
129
|
+
|
|
130
|
+
macro(:perform_import) do |*opts|
|
|
131
|
+
Alarm.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: [:device_id, :alarm_type], index_predicate: 'status <> 0', columns: [:status] }, validate: false)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
macro(:updated_alarm) { Alarm.find(@alarm.id) }
|
|
135
|
+
|
|
136
|
+
setup do
|
|
137
|
+
Alarm.import columns, values, validate: false
|
|
138
|
+
@alarm = Alarm.find 99
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
context 'supports on duplicate key update for partial indexes' do
|
|
142
|
+
it 'should not update created_at timestamp columns' do
|
|
143
|
+
Timecop.freeze Chronic.parse("5 minutes from now") do
|
|
144
|
+
perform_import
|
|
145
|
+
assert_in_delta @alarm.created_at.to_i, updated_alarm.created_at.to_i, 1
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
it 'should update updated_at timestamp columns' do
|
|
150
|
+
time = Chronic.parse("5 minutes from now")
|
|
151
|
+
Timecop.freeze time do
|
|
152
|
+
perform_import
|
|
153
|
+
assert_in_delta time.to_i, updated_alarm.updated_at.to_i, 1
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
it 'should not update fields not mentioned' do
|
|
158
|
+
perform_import
|
|
159
|
+
assert_equal 'foo', updated_alarm.metadata
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
it 'should update fields mentioned with hash mappings' do
|
|
163
|
+
perform_import
|
|
164
|
+
assert_equal 2, updated_alarm.status
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
context 'with :condition' do
|
|
170
|
+
let(:columns) { %w( id device_id alarm_type status metadata) }
|
|
171
|
+
let(:values) { [[99, 17, 1, 1, 'foo']] }
|
|
172
|
+
let(:updated_values) { [[99, 17, 1, 1, 'bar']] }
|
|
173
|
+
|
|
174
|
+
macro(:perform_import) do |*opts|
|
|
175
|
+
Alarm.import(
|
|
176
|
+
columns,
|
|
177
|
+
updated_values,
|
|
178
|
+
opts.extract_options!.merge(
|
|
179
|
+
on_duplicate_key_update: {
|
|
180
|
+
conflict_target: [:id],
|
|
181
|
+
condition: "alarms.metadata NOT LIKE '%foo%'",
|
|
182
|
+
columns: [:metadata]
|
|
183
|
+
},
|
|
184
|
+
validate: false
|
|
185
|
+
)
|
|
186
|
+
)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
macro(:updated_alarm) { Alarm.find(@alarm.id) }
|
|
190
|
+
|
|
191
|
+
setup do
|
|
192
|
+
Alarm.import columns, values, validate: false
|
|
193
|
+
@alarm = Alarm.find 99
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
it 'should not update fields not matched' do
|
|
197
|
+
perform_import
|
|
198
|
+
assert_equal 'foo', updated_alarm.metadata
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
context "with no :conflict_target" do
|
|
203
|
+
context "with no primary key" do
|
|
204
|
+
it "raises ArgumentError" do
|
|
205
|
+
error = assert_raises ArgumentError do
|
|
206
|
+
Rule.import Build(3, :rules), on_duplicate_key_update: [:condition_text], validate: false
|
|
207
|
+
end
|
|
208
|
+
assert_match(/Expected :conflict_target to be specified/, error.message)
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
context "with no :columns" do
|
|
214
|
+
let(:columns) { %w( id title author_name author_email_address ) }
|
|
215
|
+
let(:values) { [[100, "Book", "John Doe", "john@doe.com"]] }
|
|
216
|
+
let(:updated_values) { [[100, "Title Should Not Change", "Author Should Not Change", "john@nogo.com"]] }
|
|
217
|
+
|
|
218
|
+
macro(:perform_import) do |*opts|
|
|
219
|
+
Topic.import columns, updated_values, opts.extract_options!.merge(on_duplicate_key_update: { conflict_target: :id }, validate: false)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
setup do
|
|
223
|
+
Topic.import columns, values, validate: false
|
|
224
|
+
@topic = Topic.find 100
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
should_update_updated_at_on_timestamp_columns
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
data/test/synchronize_test.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require File.expand_path('../test_helper', __FILE__)
|
|
2
4
|
|
|
3
5
|
describe ".synchronize" do
|
|
@@ -30,4 +32,12 @@ describe ".synchronize" do
|
|
|
30
32
|
assert_equal false, topics[1].changed?, "the second record was dirty"
|
|
31
33
|
assert_equal false, topics[2].changed?, "the third record was dirty"
|
|
32
34
|
end
|
|
35
|
+
|
|
36
|
+
it "ignores default scope" do
|
|
37
|
+
# update records outside of ActiveRecord knowing about it
|
|
38
|
+
Topic.connection.execute( "UPDATE #{Topic.table_name} SET approved='0' WHERE id=#{topics[0].id}", "Updating record 1 without ActiveRecord" )
|
|
39
|
+
|
|
40
|
+
Topic.synchronize topics
|
|
41
|
+
assert_equal false, topics[0].approved
|
|
42
|
+
end
|
|
33
43
|
end
|
data/test/test_helper.rb
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'pathname'
|
|
4
|
+
require 'rake'
|
|
2
5
|
test_dir = Pathname.new File.dirname(__FILE__)
|
|
3
6
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
|
4
7
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
@@ -18,13 +21,29 @@ require "active_support/test_case"
|
|
|
18
21
|
|
|
19
22
|
if ActiveSupport::VERSION::STRING < "4.0"
|
|
20
23
|
require 'test/unit'
|
|
24
|
+
require 'mocha/test_unit'
|
|
21
25
|
else
|
|
22
26
|
require 'active_support/testing/autorun'
|
|
27
|
+
require "mocha/mini_test"
|
|
23
28
|
end
|
|
24
29
|
|
|
25
30
|
require 'timecop'
|
|
26
31
|
require 'chronic'
|
|
27
32
|
|
|
33
|
+
begin
|
|
34
|
+
require 'composite_primary_keys'
|
|
35
|
+
rescue LoadError
|
|
36
|
+
ENV["SKIP_COMPOSITE_PK"] = "true"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Support MySQL 5.7
|
|
40
|
+
if ActiveSupport::VERSION::STRING < "4.1"
|
|
41
|
+
require "active_record/connection_adapters/mysql2_adapter"
|
|
42
|
+
class ActiveRecord::ConnectionAdapters::Mysql2Adapter
|
|
43
|
+
NATIVE_DATABASE_TYPES[:primary_key] = "int(11) auto_increment PRIMARY KEY"
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
28
47
|
require "ruby-debug" if RUBY_VERSION.to_f < 1.9
|
|
29
48
|
|
|
30
49
|
adapter = ENV["ARE_DB"] || "sqlite3"
|
|
@@ -32,8 +51,30 @@ adapter = ENV["ARE_DB"] || "sqlite3"
|
|
|
32
51
|
FileUtils.mkdir_p 'log'
|
|
33
52
|
ActiveRecord::Base.logger = Logger.new("log/test.log")
|
|
34
53
|
ActiveRecord::Base.logger.level = Logger::DEBUG
|
|
35
|
-
|
|
36
|
-
ActiveRecord
|
|
54
|
+
|
|
55
|
+
if ActiveRecord.respond_to?(:use_yaml_unsafe_load)
|
|
56
|
+
ActiveRecord.use_yaml_unsafe_load = true
|
|
57
|
+
elsif ActiveRecord::Base.respond_to?(:use_yaml_unsafe_load)
|
|
58
|
+
ActiveRecord::Base.use_yaml_unsafe_load = true
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
if ENV['AR_VERSION'].to_f >= 6.0
|
|
62
|
+
yaml_config = if Gem::Version.new(Psych::VERSION) >= Gem::Version.new('3.2.1')
|
|
63
|
+
YAML.safe_load_file(test_dir.join("database.yml"), aliases: true)[adapter]
|
|
64
|
+
else
|
|
65
|
+
YAML.load_file(test_dir.join("database.yml"))[adapter]
|
|
66
|
+
end
|
|
67
|
+
config = ActiveRecord::DatabaseConfigurations::HashConfig.new("test", adapter, yaml_config)
|
|
68
|
+
ActiveRecord::Base.configurations.configurations << config
|
|
69
|
+
else
|
|
70
|
+
ActiveRecord::Base.configurations["test"] = YAML.load_file(test_dir.join("database.yml"))[adapter]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
if ActiveRecord.respond_to?(:default_timezone)
|
|
74
|
+
ActiveRecord.default_timezone = :utc
|
|
75
|
+
else
|
|
76
|
+
ActiveRecord::Base.default_timezone = :utc
|
|
77
|
+
end
|
|
37
78
|
|
|
38
79
|
require "activerecord-import"
|
|
39
80
|
ActiveRecord::Base.establish_connection :test
|
|
@@ -42,7 +83,7 @@ ActiveSupport::Notifications.subscribe(/active_record.sql/) do |_, _, _, _, hsh|
|
|
|
42
83
|
ActiveRecord::Base.logger.info hsh[:sql]
|
|
43
84
|
end
|
|
44
85
|
|
|
45
|
-
require "
|
|
86
|
+
require "factory_bot"
|
|
46
87
|
Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each { |file| require file }
|
|
47
88
|
|
|
48
89
|
# Load base/generic schema
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require File.expand_path(File.dirname(__FILE__) + '/test_helper')
|
|
2
4
|
|
|
3
5
|
require 'activerecord-import/value_sets_parser'
|
|
@@ -8,6 +10,15 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
|
8
10
|
let(:base_sql) { "INSERT INTO atable (a,b,c)" }
|
|
9
11
|
let(:values) { ["(1,2,3)", "(2,3,4)", "(3,4,5)"] }
|
|
10
12
|
|
|
13
|
+
context "when the max allowed bytes is 30 and the base SQL is 26 bytes" do
|
|
14
|
+
it "should raise ActiveRecord::Import::ValueSetTooLargeError" do
|
|
15
|
+
error = assert_raises ActiveRecord::Import::ValueSetTooLargeError do
|
|
16
|
+
parser.parse values, reserved_bytes: base_sql.size, max_bytes: 30
|
|
17
|
+
end
|
|
18
|
+
assert_match(/33 bytes exceeds the max allowed for an insert \[30\]/, error.message)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
11
22
|
context "when the max allowed bytes is 33 and the base SQL is 26 bytes" do
|
|
12
23
|
it "should return 3 value sets when given 3 value sets of 7 bytes a piece" do
|
|
13
24
|
value_sets = parser.parse values, reserved_bytes: base_sql.size, max_bytes: 33
|
|
@@ -54,7 +65,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
|
54
65
|
values = [
|
|
55
66
|
"('1','2','3')",
|
|
56
67
|
"('4','5','6')",
|
|
57
|
-
"('7','8','9')"
|
|
68
|
+
"('7','8','9')"
|
|
69
|
+
]
|
|
58
70
|
|
|
59
71
|
base_sql_size_in_bytes = 15
|
|
60
72
|
max_bytes = 30
|
|
@@ -79,7 +91,8 @@ describe ActiveRecord::Import::ValueSetsBytesParser do
|
|
|
79
91
|
# each accented e should be 2 bytes, so each entry is 6 bytes instead of 5
|
|
80
92
|
values = [
|
|
81
93
|
"('é')",
|
|
82
|
-
"('é')"
|
|
94
|
+
"('é')"
|
|
95
|
+
]
|
|
83
96
|
|
|
84
97
|
base_sql_size_in_bytes = 15
|
|
85
98
|
max_bytes = 26
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: activerecord-import
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 1.4.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Zach Dennis
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-10-07 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activerecord
|
|
@@ -16,14 +16,14 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - ">="
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '
|
|
19
|
+
version: '4.2'
|
|
20
20
|
type: :runtime
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '
|
|
26
|
+
version: '4.2'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: rake
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,18 +38,17 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
-
description:
|
|
42
|
-
for Rails 3 and beyond
|
|
41
|
+
description: A library for bulk inserting data using ActiveRecord.
|
|
43
42
|
email:
|
|
44
43
|
- zach.dennis@gmail.com
|
|
45
44
|
executables: []
|
|
46
45
|
extensions: []
|
|
47
46
|
extra_rdoc_files: []
|
|
48
47
|
files:
|
|
48
|
+
- ".github/workflows/test.yaml"
|
|
49
49
|
- ".gitignore"
|
|
50
50
|
- ".rubocop.yml"
|
|
51
51
|
- ".rubocop_todo.yml"
|
|
52
|
-
- ".travis.yml"
|
|
53
52
|
- Brewfile
|
|
54
53
|
- CHANGELOG.md
|
|
55
54
|
- Gemfile
|
|
@@ -68,16 +67,19 @@ files:
|
|
|
68
67
|
- benchmarks/models/test_innodb.rb
|
|
69
68
|
- benchmarks/models/test_memory.rb
|
|
70
69
|
- benchmarks/models/test_myisam.rb
|
|
71
|
-
- benchmarks/schema/
|
|
72
|
-
- gemfiles/3.2.gemfile
|
|
73
|
-
- gemfiles/4.0.gemfile
|
|
74
|
-
- gemfiles/4.1.gemfile
|
|
70
|
+
- benchmarks/schema/mysql2_schema.rb
|
|
75
71
|
- gemfiles/4.2.gemfile
|
|
76
72
|
- gemfiles/5.0.gemfile
|
|
73
|
+
- gemfiles/5.1.gemfile
|
|
74
|
+
- gemfiles/5.2.gemfile
|
|
75
|
+
- gemfiles/6.0.gemfile
|
|
76
|
+
- gemfiles/6.1.gemfile
|
|
77
|
+
- gemfiles/7.0.gemfile
|
|
77
78
|
- lib/activerecord-import.rb
|
|
78
79
|
- lib/activerecord-import/active_record/adapters/abstract_adapter.rb
|
|
79
80
|
- lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb
|
|
80
81
|
- lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb
|
|
82
|
+
- lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb
|
|
81
83
|
- lib/activerecord-import/active_record/adapters/mysql2_adapter.rb
|
|
82
84
|
- lib/activerecord-import/active_record/adapters/postgresql_adapter.rb
|
|
83
85
|
- lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb
|
|
@@ -98,6 +100,8 @@ files:
|
|
|
98
100
|
- lib/activerecord-import/version.rb
|
|
99
101
|
- test/adapters/jdbcmysql.rb
|
|
100
102
|
- test/adapters/jdbcpostgresql.rb
|
|
103
|
+
- test/adapters/jdbcsqlite3.rb
|
|
104
|
+
- test/adapters/makara_postgis.rb
|
|
101
105
|
- test/adapters/mysql2.rb
|
|
102
106
|
- test/adapters/mysql2_makara.rb
|
|
103
107
|
- test/adapters/mysql2spatial.rb
|
|
@@ -108,18 +112,37 @@ files:
|
|
|
108
112
|
- test/adapters/spatialite.rb
|
|
109
113
|
- test/adapters/sqlite3.rb
|
|
110
114
|
- test/database.yml.sample
|
|
115
|
+
- test/github/database.yml
|
|
111
116
|
- test/import_test.rb
|
|
112
117
|
- test/jdbcmysql/import_test.rb
|
|
113
118
|
- test/jdbcpostgresql/import_test.rb
|
|
119
|
+
- test/jdbcsqlite3/import_test.rb
|
|
120
|
+
- test/makara_postgis/import_test.rb
|
|
121
|
+
- test/models/account.rb
|
|
122
|
+
- test/models/alarm.rb
|
|
123
|
+
- test/models/animal.rb
|
|
124
|
+
- test/models/bike_maker.rb
|
|
114
125
|
- test/models/book.rb
|
|
126
|
+
- test/models/car.rb
|
|
127
|
+
- test/models/card.rb
|
|
115
128
|
- test/models/chapter.rb
|
|
129
|
+
- test/models/customer.rb
|
|
130
|
+
- test/models/deck.rb
|
|
131
|
+
- test/models/dictionary.rb
|
|
116
132
|
- test/models/discount.rb
|
|
117
133
|
- test/models/end_note.rb
|
|
118
134
|
- test/models/group.rb
|
|
135
|
+
- test/models/order.rb
|
|
136
|
+
- test/models/playing_card.rb
|
|
119
137
|
- test/models/promotion.rb
|
|
120
138
|
- test/models/question.rb
|
|
121
139
|
- test/models/rule.rb
|
|
140
|
+
- test/models/tag.rb
|
|
141
|
+
- test/models/tag_alias.rb
|
|
122
142
|
- test/models/topic.rb
|
|
143
|
+
- test/models/user.rb
|
|
144
|
+
- test/models/user_token.rb
|
|
145
|
+
- test/models/vendor.rb
|
|
123
146
|
- test/models/widget.rb
|
|
124
147
|
- test/mysql2/import_test.rb
|
|
125
148
|
- test/mysql2_makara/import_test.rb
|
|
@@ -127,7 +150,11 @@ files:
|
|
|
127
150
|
- test/postgis/import_test.rb
|
|
128
151
|
- test/postgresql/import_test.rb
|
|
129
152
|
- test/schema/generic_schema.rb
|
|
130
|
-
- test/schema/
|
|
153
|
+
- test/schema/jdbcpostgresql_schema.rb
|
|
154
|
+
- test/schema/mysql2_schema.rb
|
|
155
|
+
- test/schema/postgis_schema.rb
|
|
156
|
+
- test/schema/postgresql_schema.rb
|
|
157
|
+
- test/schema/sqlite3_schema.rb
|
|
131
158
|
- test/schema/version.rb
|
|
132
159
|
- test/sqlite3/import_test.rb
|
|
133
160
|
- test/support/active_support/test_case_extensions.rb
|
|
@@ -136,16 +163,17 @@ files:
|
|
|
136
163
|
- test/support/generate.rb
|
|
137
164
|
- test/support/mysql/import_examples.rb
|
|
138
165
|
- test/support/postgresql/import_examples.rb
|
|
166
|
+
- test/support/shared_examples/on_duplicate_key_ignore.rb
|
|
139
167
|
- test/support/shared_examples/on_duplicate_key_update.rb
|
|
140
168
|
- test/support/shared_examples/recursive_import.rb
|
|
169
|
+
- test/support/sqlite3/import_examples.rb
|
|
141
170
|
- test/synchronize_test.rb
|
|
142
171
|
- test/test_helper.rb
|
|
143
|
-
- test/travis/database.yml
|
|
144
172
|
- test/value_sets_bytes_parser_test.rb
|
|
145
173
|
- test/value_sets_records_parser_test.rb
|
|
146
|
-
homepage:
|
|
174
|
+
homepage: https://github.com/zdennis/activerecord-import
|
|
147
175
|
licenses:
|
|
148
|
-
-
|
|
176
|
+
- MIT
|
|
149
177
|
metadata: {}
|
|
150
178
|
post_install_message:
|
|
151
179
|
rdoc_options: []
|
|
@@ -155,21 +183,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
155
183
|
requirements:
|
|
156
184
|
- - ">="
|
|
157
185
|
- !ruby/object:Gem::Version
|
|
158
|
-
version:
|
|
186
|
+
version: 2.4.0
|
|
159
187
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
160
188
|
requirements:
|
|
161
189
|
- - ">="
|
|
162
190
|
- !ruby/object:Gem::Version
|
|
163
191
|
version: '0'
|
|
164
192
|
requirements: []
|
|
165
|
-
|
|
166
|
-
rubygems_version: 2.5.1
|
|
193
|
+
rubygems_version: 3.0.3.1
|
|
167
194
|
signing_key:
|
|
168
195
|
specification_version: 4
|
|
169
|
-
summary: Bulk
|
|
196
|
+
summary: Bulk insert extension for ActiveRecord
|
|
170
197
|
test_files:
|
|
171
198
|
- test/adapters/jdbcmysql.rb
|
|
172
199
|
- test/adapters/jdbcpostgresql.rb
|
|
200
|
+
- test/adapters/jdbcsqlite3.rb
|
|
201
|
+
- test/adapters/makara_postgis.rb
|
|
173
202
|
- test/adapters/mysql2.rb
|
|
174
203
|
- test/adapters/mysql2_makara.rb
|
|
175
204
|
- test/adapters/mysql2spatial.rb
|
|
@@ -180,18 +209,37 @@ test_files:
|
|
|
180
209
|
- test/adapters/spatialite.rb
|
|
181
210
|
- test/adapters/sqlite3.rb
|
|
182
211
|
- test/database.yml.sample
|
|
212
|
+
- test/github/database.yml
|
|
183
213
|
- test/import_test.rb
|
|
184
214
|
- test/jdbcmysql/import_test.rb
|
|
185
215
|
- test/jdbcpostgresql/import_test.rb
|
|
216
|
+
- test/jdbcsqlite3/import_test.rb
|
|
217
|
+
- test/makara_postgis/import_test.rb
|
|
218
|
+
- test/models/account.rb
|
|
219
|
+
- test/models/alarm.rb
|
|
220
|
+
- test/models/animal.rb
|
|
221
|
+
- test/models/bike_maker.rb
|
|
186
222
|
- test/models/book.rb
|
|
223
|
+
- test/models/car.rb
|
|
224
|
+
- test/models/card.rb
|
|
187
225
|
- test/models/chapter.rb
|
|
226
|
+
- test/models/customer.rb
|
|
227
|
+
- test/models/deck.rb
|
|
228
|
+
- test/models/dictionary.rb
|
|
188
229
|
- test/models/discount.rb
|
|
189
230
|
- test/models/end_note.rb
|
|
190
231
|
- test/models/group.rb
|
|
232
|
+
- test/models/order.rb
|
|
233
|
+
- test/models/playing_card.rb
|
|
191
234
|
- test/models/promotion.rb
|
|
192
235
|
- test/models/question.rb
|
|
193
236
|
- test/models/rule.rb
|
|
237
|
+
- test/models/tag.rb
|
|
238
|
+
- test/models/tag_alias.rb
|
|
194
239
|
- test/models/topic.rb
|
|
240
|
+
- test/models/user.rb
|
|
241
|
+
- test/models/user_token.rb
|
|
242
|
+
- test/models/vendor.rb
|
|
195
243
|
- test/models/widget.rb
|
|
196
244
|
- test/mysql2/import_test.rb
|
|
197
245
|
- test/mysql2_makara/import_test.rb
|
|
@@ -199,7 +247,11 @@ test_files:
|
|
|
199
247
|
- test/postgis/import_test.rb
|
|
200
248
|
- test/postgresql/import_test.rb
|
|
201
249
|
- test/schema/generic_schema.rb
|
|
202
|
-
- test/schema/
|
|
250
|
+
- test/schema/jdbcpostgresql_schema.rb
|
|
251
|
+
- test/schema/mysql2_schema.rb
|
|
252
|
+
- test/schema/postgis_schema.rb
|
|
253
|
+
- test/schema/postgresql_schema.rb
|
|
254
|
+
- test/schema/sqlite3_schema.rb
|
|
203
255
|
- test/schema/version.rb
|
|
204
256
|
- test/sqlite3/import_test.rb
|
|
205
257
|
- test/support/active_support/test_case_extensions.rb
|
|
@@ -208,11 +260,11 @@ test_files:
|
|
|
208
260
|
- test/support/generate.rb
|
|
209
261
|
- test/support/mysql/import_examples.rb
|
|
210
262
|
- test/support/postgresql/import_examples.rb
|
|
263
|
+
- test/support/shared_examples/on_duplicate_key_ignore.rb
|
|
211
264
|
- test/support/shared_examples/on_duplicate_key_update.rb
|
|
212
265
|
- test/support/shared_examples/recursive_import.rb
|
|
266
|
+
- test/support/sqlite3/import_examples.rb
|
|
213
267
|
- test/synchronize_test.rb
|
|
214
268
|
- test/test_helper.rb
|
|
215
|
-
- test/travis/database.yml
|
|
216
269
|
- test/value_sets_bytes_parser_test.rb
|
|
217
270
|
- test/value_sets_records_parser_test.rb
|
|
218
|
-
has_rdoc:
|
data/.travis.yml
DELETED
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
language: ruby
|
|
2
|
-
cache: bundler
|
|
3
|
-
rvm:
|
|
4
|
-
- 2.2.4
|
|
5
|
-
|
|
6
|
-
env:
|
|
7
|
-
global:
|
|
8
|
-
# https://github.com/discourse/discourse/blob/master/.travis.yml
|
|
9
|
-
- RUBY_GC_MALLOC_LIMIT=50000000
|
|
10
|
-
matrix:
|
|
11
|
-
- AR_VERSION=3.2
|
|
12
|
-
- AR_VERSION=4.0
|
|
13
|
-
- AR_VERSION=4.1
|
|
14
|
-
- AR_VERSION=4.2
|
|
15
|
-
- AR_VERSION=5.0
|
|
16
|
-
|
|
17
|
-
matrix:
|
|
18
|
-
include:
|
|
19
|
-
- rvm: jruby-9.0.5.0
|
|
20
|
-
env: AR_VERSION=4.2
|
|
21
|
-
script:
|
|
22
|
-
- bundle exec rake test:jdbcmysql
|
|
23
|
-
- bundle exec rake test:jdbcpostgresql
|
|
24
|
-
|
|
25
|
-
fast_finish: true
|
|
26
|
-
|
|
27
|
-
before_script:
|
|
28
|
-
- mysql -e 'create database activerecord_import_test;'
|
|
29
|
-
- psql -c 'create database activerecord_import_test;' -U postgres
|
|
30
|
-
- psql -U postgres -c "create extension postgis"
|
|
31
|
-
- cp test/travis/database.yml test/database.yml
|
|
32
|
-
|
|
33
|
-
addons:
|
|
34
|
-
apt:
|
|
35
|
-
sources:
|
|
36
|
-
- travis-ci/sqlite3
|
|
37
|
-
packages:
|
|
38
|
-
- sqlite3
|
|
39
|
-
|
|
40
|
-
script:
|
|
41
|
-
- bundle exec rake test:mysql2
|
|
42
|
-
- bundle exec rake test:mysql2_makara
|
|
43
|
-
- bundle exec rake test:mysql2spatial
|
|
44
|
-
- bundle exec rake test:postgis
|
|
45
|
-
- bundle exec rake test:postgresql
|
|
46
|
-
- bundle exec rake test:postgresql_makara
|
|
47
|
-
- bundle exec rake test:seamless_database_pool
|
|
48
|
-
- bundle exec rake test:spatialite
|
|
49
|
-
- bundle exec rake test:sqlite3
|
|
50
|
-
- bundle exec rubocop
|
|
51
|
-
|
|
52
|
-
sudo: false
|
data/gemfiles/3.2.gemfile
DELETED
data/gemfiles/4.0.gemfile
DELETED