activerecord-import-uuid 0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +32 -0
- data/.rubocop.yml +49 -0
- data/.rubocop_todo.yml +36 -0
- data/.travis.yml +52 -0
- data/Brewfile +3 -0
- data/CHANGELOG.md +87 -0
- data/Gemfile +54 -0
- data/LICENSE +56 -0
- data/README.markdown +101 -0
- data/Rakefile +66 -0
- data/activerecord-import.gemspec +23 -0
- data/benchmarks/README +32 -0
- data/benchmarks/benchmark.rb +67 -0
- data/benchmarks/lib/base.rb +138 -0
- data/benchmarks/lib/cli_parser.rb +106 -0
- data/benchmarks/lib/float.rb +15 -0
- data/benchmarks/lib/mysql2_benchmark.rb +19 -0
- data/benchmarks/lib/output_to_csv.rb +19 -0
- data/benchmarks/lib/output_to_html.rb +64 -0
- data/benchmarks/models/test_innodb.rb +3 -0
- data/benchmarks/models/test_memory.rb +3 -0
- data/benchmarks/models/test_myisam.rb +3 -0
- data/benchmarks/schema/mysql_schema.rb +16 -0
- data/gemfiles/3.2.gemfile +3 -0
- data/gemfiles/4.0.gemfile +3 -0
- data/gemfiles/4.1.gemfile +3 -0
- data/gemfiles/4.2.gemfile +7 -0
- data/gemfiles/5.0.gemfile +3 -0
- data/lib/activerecord-import.rb +19 -0
- data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +9 -0
- data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -0
- data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +6 -0
- data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +6 -0
- data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +6 -0
- data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +7 -0
- data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +6 -0
- data/lib/activerecord-import/adapters/abstract_adapter.rb +78 -0
- data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +5 -0
- data/lib/activerecord-import/adapters/mysql2_adapter.rb +5 -0
- data/lib/activerecord-import/adapters/mysql_adapter.rb +114 -0
- data/lib/activerecord-import/adapters/postgresql_adapter.rb +144 -0
- data/lib/activerecord-import/adapters/sqlite3_adapter.rb +51 -0
- data/lib/activerecord-import/base.rb +38 -0
- data/lib/activerecord-import/import.rb +660 -0
- data/lib/activerecord-import/mysql2.rb +7 -0
- data/lib/activerecord-import/postgresql.rb +7 -0
- data/lib/activerecord-import/sqlite3.rb +7 -0
- data/lib/activerecord-import/synchronize.rb +66 -0
- data/lib/activerecord-import/value_sets_parser.rb +55 -0
- data/lib/activerecord-import/version.rb +5 -0
- data/test/adapters/jdbcmysql.rb +1 -0
- data/test/adapters/jdbcpostgresql.rb +1 -0
- data/test/adapters/mysql2.rb +1 -0
- data/test/adapters/mysql2_makara.rb +1 -0
- data/test/adapters/mysql2spatial.rb +1 -0
- data/test/adapters/postgis.rb +1 -0
- data/test/adapters/postgresql.rb +1 -0
- data/test/adapters/postgresql_makara.rb +1 -0
- data/test/adapters/seamless_database_pool.rb +1 -0
- data/test/adapters/spatialite.rb +1 -0
- data/test/adapters/sqlite3.rb +1 -0
- data/test/database.yml.sample +52 -0
- data/test/import_test.rb +574 -0
- data/test/jdbcmysql/import_test.rb +6 -0
- data/test/jdbcpostgresql/import_test.rb +5 -0
- data/test/models/book.rb +7 -0
- data/test/models/chapter.rb +4 -0
- data/test/models/discount.rb +3 -0
- data/test/models/end_note.rb +4 -0
- data/test/models/group.rb +3 -0
- data/test/models/promotion.rb +3 -0
- data/test/models/question.rb +3 -0
- data/test/models/rule.rb +3 -0
- data/test/models/topic.rb +9 -0
- data/test/models/widget.rb +24 -0
- data/test/mysql2/import_test.rb +5 -0
- data/test/mysql2_makara/import_test.rb +6 -0
- data/test/mysqlspatial2/import_test.rb +6 -0
- data/test/postgis/import_test.rb +4 -0
- data/test/postgresql/import_test.rb +8 -0
- data/test/schema/generic_schema.rb +144 -0
- data/test/schema/mysql_schema.rb +16 -0
- data/test/schema/version.rb +10 -0
- data/test/sqlite3/import_test.rb +52 -0
- data/test/support/active_support/test_case_extensions.rb +70 -0
- data/test/support/assertions.rb +73 -0
- data/test/support/factories.rb +57 -0
- data/test/support/generate.rb +29 -0
- data/test/support/mysql/import_examples.rb +85 -0
- data/test/support/postgresql/import_examples.rb +242 -0
- data/test/support/shared_examples/on_duplicate_key_update.rb +103 -0
- data/test/support/shared_examples/recursive_import.rb +122 -0
- data/test/synchronize_test.rb +33 -0
- data/test/test_helper.rb +59 -0
- data/test/travis/database.yml +62 -0
- data/test/value_sets_bytes_parser_test.rb +93 -0
- data/test/value_sets_records_parser_test.rb +32 -0
- metadata +225 -0
@@ -0,0 +1,19 @@
|
|
1
|
+
class Mysql2Benchmark < BenchmarkBase
|
2
|
+
def benchmark_all( array_of_cols_and_vals )
|
3
|
+
methods = self.methods.find_all { |m| m =~ /benchmark_/ }
|
4
|
+
methods.delete_if { |m| m =~ /benchmark_(all|model)/ }
|
5
|
+
methods.each { |method| send( method, array_of_cols_and_vals ) }
|
6
|
+
end
|
7
|
+
|
8
|
+
def benchmark_myisam( array_of_cols_and_vals )
|
9
|
+
bm_model( TestMyISAM, array_of_cols_and_vals )
|
10
|
+
end
|
11
|
+
|
12
|
+
def benchmark_innodb( array_of_cols_and_vals )
|
13
|
+
bm_model( TestInnoDb, array_of_cols_and_vals )
|
14
|
+
end
|
15
|
+
|
16
|
+
def benchmark_memory( array_of_cols_and_vals )
|
17
|
+
bm_model( TestMemory, array_of_cols_and_vals )
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module OutputToCSV
|
4
|
+
def self.output_results( filename, results )
|
5
|
+
CSV.open( filename, 'w' ) do |csv|
|
6
|
+
# Iterate over each result set, which contains many results
|
7
|
+
results.each do |result_set|
|
8
|
+
columns = []
|
9
|
+
times = []
|
10
|
+
result_set.each do |result|
|
11
|
+
columns << result.description
|
12
|
+
times << result.tms.real
|
13
|
+
end
|
14
|
+
csv << columns
|
15
|
+
csv << times
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'erb'
|
2
|
+
|
3
|
+
module OutputToHTML
|
4
|
+
TEMPLATE_HEADER = <<"EOT".freeze
|
5
|
+
<div>
|
6
|
+
All times are rounded to the nearest thousandth for display purposes. Speedups next to each time are computed
|
7
|
+
before any rounding occurs. Also, all speedup calculations are computed by comparing a given time against
|
8
|
+
the very first column (which is always the default ActiveRecord::Base.create method.
|
9
|
+
</div>
|
10
|
+
EOT
|
11
|
+
|
12
|
+
TEMPLATE = <<"EOT".freeze
|
13
|
+
<style>
|
14
|
+
td#benchmarkTitle {
|
15
|
+
border: 1px solid black;
|
16
|
+
padding: 2px;
|
17
|
+
font-size: 0.8em;
|
18
|
+
background-color: black;
|
19
|
+
color: white;
|
20
|
+
}
|
21
|
+
td#benchmarkCell {
|
22
|
+
border: 1px solid black;
|
23
|
+
padding: 2px;
|
24
|
+
font-size: 0.8em;
|
25
|
+
}
|
26
|
+
</style>
|
27
|
+
<table>
|
28
|
+
<tr>
|
29
|
+
<% columns.each do |col| %>
|
30
|
+
<td id="benchmarkTitle"><%= col %></td>
|
31
|
+
<% end %>
|
32
|
+
</tr>
|
33
|
+
<tr>
|
34
|
+
<% times.each do |time| %>
|
35
|
+
<td id="benchmarkCell"><%= time %></td>
|
36
|
+
<% end %>
|
37
|
+
</tr>
|
38
|
+
<tr><td> </td></tr>
|
39
|
+
</table>
|
40
|
+
EOT
|
41
|
+
|
42
|
+
def self.output_results( filename, results )
|
43
|
+
html = ''
|
44
|
+
results.each do |result_set|
|
45
|
+
columns = []
|
46
|
+
times = []
|
47
|
+
result_set.each do |result|
|
48
|
+
columns << result.description
|
49
|
+
if result.failed
|
50
|
+
times << "failed"
|
51
|
+
else
|
52
|
+
time = result.tms.real.round_to( 3 )
|
53
|
+
speedup = ( result_set.first.tms.real / result.tms.real ).round
|
54
|
+
times << (result == result_set.first ? time.to_s : "#{time} (#{speedup}x speedup)")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
template = ERB.new( TEMPLATE, 0, "%<>")
|
59
|
+
html << template.result( binding )
|
60
|
+
end
|
61
|
+
|
62
|
+
File.open( filename, 'w' ) { |file| file.write( TEMPLATE_HEADER + html ) }
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
ActiveRecord::Schema.define do
|
2
|
+
create_table :test_myisam, options: 'ENGINE=MyISAM', force: true do |t|
|
3
|
+
t.column :my_name, :string, null: false
|
4
|
+
t.column :description, :string
|
5
|
+
end
|
6
|
+
|
7
|
+
create_table :test_innodb, options: 'ENGINE=InnoDb', force: true do |t|
|
8
|
+
t.column :my_name, :string, null: false
|
9
|
+
t.column :description, :string
|
10
|
+
end
|
11
|
+
|
12
|
+
create_table :test_memory, options: 'ENGINE=Memory', force: true do |t|
|
13
|
+
t.column :my_name, :string, null: false
|
14
|
+
t.column :description, :string
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# rubocop:disable Style/FileName
|
2
|
+
|
3
|
+
ActiveSupport.on_load(:active_record) do
|
4
|
+
class ActiveRecord::Base
|
5
|
+
class << self
|
6
|
+
def establish_connection_with_activerecord_import(*args)
|
7
|
+
conn = establish_connection_without_activerecord_import(*args)
|
8
|
+
if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
|
9
|
+
require "activerecord-import/base"
|
10
|
+
end
|
11
|
+
|
12
|
+
ActiveRecord::Import.load_from_connection_pool connection_pool
|
13
|
+
conn
|
14
|
+
end
|
15
|
+
alias establish_connection_without_activerecord_import establish_connection
|
16
|
+
alias establish_connection establish_connection_with_activerecord_import
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,7 @@
|
|
1
|
+
require "seamless_database_pool"
|
2
|
+
require "active_record/connection_adapters/seamless_database_pool_adapter"
|
3
|
+
require "activerecord-import/adapters/mysql_adapter"
|
4
|
+
|
5
|
+
class ActiveRecord::ConnectionAdapters::SeamlessDatabasePoolAdapter
|
6
|
+
include ActiveRecord::Import::MysqlAdapter
|
7
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
module ActiveRecord::Import::AbstractAdapter
|
2
|
+
module InstanceMethods
|
3
|
+
def next_value_for_sequence(sequence_name)
|
4
|
+
%(#{sequence_name}.nextval)
|
5
|
+
end
|
6
|
+
|
7
|
+
def insert_many( sql, values, *args ) # :nodoc:
|
8
|
+
number_of_inserts = 1
|
9
|
+
|
10
|
+
base_sql, post_sql = if sql.is_a?( String )
|
11
|
+
[sql, '']
|
12
|
+
elsif sql.is_a?( Array )
|
13
|
+
[sql.shift, sql.join( ' ' )]
|
14
|
+
end
|
15
|
+
|
16
|
+
sql2insert = base_sql + values.join( ',' ) + post_sql
|
17
|
+
insert( sql2insert, *args )
|
18
|
+
|
19
|
+
[number_of_inserts, []]
|
20
|
+
end
|
21
|
+
|
22
|
+
def pre_sql_statements(options)
|
23
|
+
sql = []
|
24
|
+
sql << options[:pre_sql] if options[:pre_sql]
|
25
|
+
sql << options[:command] if options[:command]
|
26
|
+
sql << "IGNORE" if options[:ignore]
|
27
|
+
|
28
|
+
# add keywords like IGNORE or DELAYED
|
29
|
+
if options[:keywords].is_a?(Array)
|
30
|
+
sql.concat(options[:keywords])
|
31
|
+
elsif options[:keywords]
|
32
|
+
sql << options[:keywords].to_s
|
33
|
+
end
|
34
|
+
|
35
|
+
sql
|
36
|
+
end
|
37
|
+
|
38
|
+
# Synchronizes the passed in ActiveRecord instances with the records in
|
39
|
+
# the database by calling +reload+ on each instance.
|
40
|
+
def after_import_synchronize( instances )
|
41
|
+
instances.each(&:reload)
|
42
|
+
end
|
43
|
+
|
44
|
+
# Returns an array of post SQL statements given the passed in options.
|
45
|
+
def post_sql_statements( table_name, options ) # :nodoc:
|
46
|
+
post_sql_statements = []
|
47
|
+
|
48
|
+
if supports_on_duplicate_key_update?
|
49
|
+
if options[:on_duplicate_key_ignore] && respond_to?(:sql_for_on_duplicate_key_ignore)
|
50
|
+
# Options :recursive and :on_duplicate_key_ignore are mutually exclusive
|
51
|
+
unless options[:recursive]
|
52
|
+
post_sql_statements << sql_for_on_duplicate_key_ignore( table_name, options[:on_duplicate_key_ignore] )
|
53
|
+
end
|
54
|
+
elsif options[:on_duplicate_key_update]
|
55
|
+
post_sql_statements << sql_for_on_duplicate_key_update( table_name, options[:on_duplicate_key_update] )
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
# custom user post_sql
|
60
|
+
post_sql_statements << options[:post_sql] if options[:post_sql]
|
61
|
+
|
62
|
+
# with rollup
|
63
|
+
post_sql_statements << rollup_sql if options[:rollup]
|
64
|
+
|
65
|
+
post_sql_statements
|
66
|
+
end
|
67
|
+
|
68
|
+
# Returns the maximum number of bytes that the server will allow
|
69
|
+
# in a single packet
|
70
|
+
def max_allowed_packet
|
71
|
+
NO_MAX_PACKET
|
72
|
+
end
|
73
|
+
|
74
|
+
def supports_on_duplicate_key_update?
|
75
|
+
false
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
module ActiveRecord::Import::MysqlAdapter
|
2
|
+
include ActiveRecord::Import::ImportSupport
|
3
|
+
include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
|
4
|
+
|
5
|
+
NO_MAX_PACKET = 0
|
6
|
+
QUERY_OVERHEAD = 8 # This was shown to be true for MySQL, but it's not clear where the overhead is from.
|
7
|
+
|
8
|
+
# +sql+ can be a single string or an array. If it is an array all
|
9
|
+
# elements that are in position >= 1 will be appended to the final SQL.
|
10
|
+
def insert_many( sql, values, *args ) # :nodoc:
|
11
|
+
# the number of inserts default
|
12
|
+
number_of_inserts = 0
|
13
|
+
|
14
|
+
base_sql, post_sql = if sql.is_a?( String )
|
15
|
+
[sql, '']
|
16
|
+
elsif sql.is_a?( Array )
|
17
|
+
[sql.shift, sql.join( ' ' )]
|
18
|
+
end
|
19
|
+
|
20
|
+
sql_size = QUERY_OVERHEAD + base_sql.size + post_sql.size
|
21
|
+
|
22
|
+
# the number of bytes the requested insert statement values will take up
|
23
|
+
values_in_bytes = values.sum(&:bytesize)
|
24
|
+
|
25
|
+
# the number of bytes (commas) it will take to comma separate our values
|
26
|
+
comma_separated_bytes = values.size - 1
|
27
|
+
|
28
|
+
# the total number of bytes required if this statement is one statement
|
29
|
+
total_bytes = sql_size + values_in_bytes + comma_separated_bytes
|
30
|
+
|
31
|
+
max = max_allowed_packet
|
32
|
+
|
33
|
+
# if we can insert it all as one statement
|
34
|
+
if NO_MAX_PACKET == max || total_bytes < max
|
35
|
+
number_of_inserts += 1
|
36
|
+
sql2insert = base_sql + values.join( ',' ) + post_sql
|
37
|
+
insert( sql2insert, *args )
|
38
|
+
else
|
39
|
+
value_sets = ::ActiveRecord::Import::ValueSetsBytesParser.parse(values,
|
40
|
+
reserved_bytes: sql_size,
|
41
|
+
max_bytes: max)
|
42
|
+
value_sets.each do |value_set|
|
43
|
+
number_of_inserts += 1
|
44
|
+
sql2insert = base_sql + value_set.join( ',' ) + post_sql
|
45
|
+
insert( sql2insert, *args )
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
[number_of_inserts, []]
|
50
|
+
end
|
51
|
+
|
52
|
+
# Returns the maximum number of bytes that the server will allow
|
53
|
+
# in a single packet
|
54
|
+
def max_allowed_packet # :nodoc:
|
55
|
+
@max_allowed_packet ||= begin
|
56
|
+
result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
|
57
|
+
# original Mysql gem responds to #fetch_row while Mysql2 responds to #first
|
58
|
+
val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
|
59
|
+
val.to_i
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# Add a column to be updated on duplicate key update
|
64
|
+
def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
|
65
|
+
if options.include?(:on_duplicate_key_update)
|
66
|
+
columns = options[:on_duplicate_key_update]
|
67
|
+
case columns
|
68
|
+
when Array then columns << column.to_sym unless columns.include?(column.to_sym)
|
69
|
+
when Hash then columns[column.to_sym] = column.to_sym
|
70
|
+
end
|
71
|
+
else
|
72
|
+
options[:on_duplicate_key_update] = [column.to_sym]
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
# Returns a generated ON DUPLICATE KEY UPDATE statement given the passed
|
77
|
+
# in +args+.
|
78
|
+
def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
|
79
|
+
sql = ' ON DUPLICATE KEY UPDATE '
|
80
|
+
arg = args.first
|
81
|
+
if arg.is_a?( Array )
|
82
|
+
sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
|
83
|
+
elsif arg.is_a?( Hash )
|
84
|
+
sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
|
85
|
+
elsif arg.is_a?( String )
|
86
|
+
sql << arg
|
87
|
+
else
|
88
|
+
raise ArgumentError, "Expected Array or Hash"
|
89
|
+
end
|
90
|
+
sql
|
91
|
+
end
|
92
|
+
|
93
|
+
def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
|
94
|
+
results = arr.map do |column|
|
95
|
+
qc = quote_column_name( column )
|
96
|
+
"#{table_name}.#{qc}=VALUES(#{qc})"
|
97
|
+
end
|
98
|
+
results.join( ',' )
|
99
|
+
end
|
100
|
+
|
101
|
+
def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
|
102
|
+
results = hsh.map do |column1, column2|
|
103
|
+
qc1 = quote_column_name( column1 )
|
104
|
+
qc2 = quote_column_name( column2 )
|
105
|
+
"#{table_name}.#{qc1}=VALUES( #{qc2} )"
|
106
|
+
end
|
107
|
+
results.join( ',')
|
108
|
+
end
|
109
|
+
|
110
|
+
# Return true if the statement is a duplicate key record error
|
111
|
+
def duplicate_key_update_error?(exception) # :nodoc:
|
112
|
+
exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
|
113
|
+
end
|
114
|
+
end
|