activerecord-import 0.19.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/.travis.yml +22 -12
  3. data/CHANGELOG.md +166 -0
  4. data/Gemfile +13 -10
  5. data/README.markdown +548 -5
  6. data/Rakefile +2 -1
  7. data/benchmarks/lib/cli_parser.rb +2 -1
  8. data/gemfiles/5.1.gemfile +1 -0
  9. data/gemfiles/5.2.gemfile +2 -0
  10. data/lib/activerecord-import/adapters/abstract_adapter.rb +2 -2
  11. data/lib/activerecord-import/adapters/mysql_adapter.rb +16 -10
  12. data/lib/activerecord-import/adapters/postgresql_adapter.rb +59 -15
  13. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +126 -3
  14. data/lib/activerecord-import/base.rb +4 -6
  15. data/lib/activerecord-import/import.rb +384 -126
  16. data/lib/activerecord-import/synchronize.rb +1 -1
  17. data/lib/activerecord-import/value_sets_parser.rb +14 -0
  18. data/lib/activerecord-import/version.rb +1 -1
  19. data/lib/activerecord-import.rb +2 -15
  20. data/test/adapters/makara_postgis.rb +1 -0
  21. data/test/import_test.rb +148 -14
  22. data/test/makara_postgis/import_test.rb +8 -0
  23. data/test/models/account.rb +3 -0
  24. data/test/models/bike_maker.rb +7 -0
  25. data/test/models/topic.rb +10 -0
  26. data/test/models/user.rb +3 -0
  27. data/test/models/user_token.rb +4 -0
  28. data/test/schema/generic_schema.rb +20 -0
  29. data/test/schema/mysql2_schema.rb +19 -0
  30. data/test/schema/postgresql_schema.rb +1 -0
  31. data/test/schema/sqlite3_schema.rb +13 -0
  32. data/test/support/factories.rb +9 -8
  33. data/test/support/generate.rb +6 -6
  34. data/test/support/mysql/import_examples.rb +14 -2
  35. data/test/support/postgresql/import_examples.rb +142 -0
  36. data/test/support/shared_examples/on_duplicate_key_update.rb +252 -1
  37. data/test/support/shared_examples/recursive_import.rb +41 -11
  38. data/test/support/sqlite3/import_examples.rb +187 -10
  39. data/test/synchronize_test.rb +8 -0
  40. data/test/test_helper.rb +9 -1
  41. data/test/value_sets_bytes_parser_test.rb +13 -2
  42. metadata +20 -5
  43. data/test/schema/mysql_schema.rb +0 -16
@@ -49,14 +49,14 @@ module ActiveRecord::Import::MysqlAdapter
49
49
  end
50
50
  end
51
51
 
52
- [number_of_inserts, []]
52
+ ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
53
53
  end
54
54
 
55
55
  # Returns the maximum number of bytes that the server will allow
56
56
  # in a single packet
57
57
  def max_allowed_packet # :nodoc:
58
58
  @max_allowed_packet ||= begin
59
- result = execute( "SHOW VARIABLES like 'max_allowed_packet';" )
59
+ result = execute( "SHOW VARIABLES like 'max_allowed_packet'" )
60
60
  # original Mysql gem responds to #fetch_row while Mysql2 responds to #first
61
61
  val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
62
62
  val.to_i
@@ -71,14 +71,11 @@ module ActiveRecord::Import::MysqlAdapter
71
71
 
72
72
  # Add a column to be updated on duplicate key update
73
73
  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
74
- if options.include?(:on_duplicate_key_update)
75
- columns = options[:on_duplicate_key_update]
74
+ if (columns = options[:on_duplicate_key_update])
76
75
  case columns
77
76
  when Array then columns << column.to_sym unless columns.include?(column.to_sym)
78
77
  when Hash then columns[column.to_sym] = column.to_sym
79
78
  end
80
- elsif !options[:ignore] && !options[:on_duplicate_key_ignore]
81
- options[:on_duplicate_key_update] = [column.to_sym]
82
79
  end
83
80
  end
84
81
 
@@ -87,10 +84,11 @@ module ActiveRecord::Import::MysqlAdapter
87
84
  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
88
85
  sql = ' ON DUPLICATE KEY UPDATE '
89
86
  arg = args.first
87
+ locking_column = args.last
90
88
  if arg.is_a?( Array )
91
- sql << sql_for_on_duplicate_key_update_as_array( table_name, arg )
89
+ sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arg )
92
90
  elsif arg.is_a?( Hash )
93
- sql << sql_for_on_duplicate_key_update_as_hash( table_name, arg )
91
+ sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, arg )
94
92
  elsif arg.is_a?( String )
95
93
  sql << arg
96
94
  else
@@ -99,20 +97,22 @@ module ActiveRecord::Import::MysqlAdapter
99
97
  sql
100
98
  end
101
99
 
102
- def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
100
+ def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
103
101
  results = arr.map do |column|
104
102
  qc = quote_column_name( column )
105
103
  "#{table_name}.#{qc}=VALUES(#{qc})"
106
104
  end
105
+ increment_locking_column!(results, table_name, locking_column)
107
106
  results.join( ',' )
108
107
  end
109
108
 
110
- def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
109
+ def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
111
110
  results = hsh.map do |column1, column2|
112
111
  qc1 = quote_column_name( column1 )
113
112
  qc2 = quote_column_name( column2 )
114
113
  "#{table_name}.#{qc1}=VALUES( #{qc2} )"
115
114
  end
115
+ increment_locking_column!(results, table_name, locking_column)
116
116
  results.join( ',')
117
117
  end
118
118
 
@@ -120,4 +120,10 @@ module ActiveRecord::Import::MysqlAdapter
120
120
  def duplicate_key_update_error?(exception) # :nodoc:
121
121
  exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('Duplicate entry')
122
122
  end
123
+
124
+ def increment_locking_column!(results, table_name, locking_column)
125
+ if locking_column.present?
126
+ results << "#{table_name}.`#{locking_column}`=`#{locking_column}`+1"
127
+ end
128
+ end
123
129
  end
@@ -5,9 +5,10 @@ module ActiveRecord::Import::PostgreSQLAdapter
5
5
  MIN_VERSION_FOR_UPSERT = 90_500
6
6
 
7
7
  def insert_many( sql, values, options = {}, *args ) # :nodoc:
8
- primary_key = options[:primary_key]
9
8
  number_of_inserts = 1
9
+ returned_values = []
10
10
  ids = []
11
+ results = []
11
12
 
12
13
  base_sql, post_sql = if sql.is_a?( String )
13
14
  [sql, '']
@@ -17,11 +18,12 @@ module ActiveRecord::Import::PostgreSQLAdapter
17
18
 
18
19
  sql2insert = base_sql + values.join( ',' ) + post_sql
19
20
 
20
- if primary_key.blank? || options[:no_returning]
21
+ columns = returning_columns(options)
22
+ if columns.blank? || options[:no_returning]
21
23
  insert( sql2insert, *args )
22
24
  else
23
- ids = if primary_key.is_a?( Array )
24
- # Select composite primary keys
25
+ returned_values = if columns.size > 1
26
+ # Select composite columns
25
27
  select_rows( sql2insert, *args )
26
28
  else
27
29
  select_values( sql2insert, *args )
@@ -29,7 +31,34 @@ module ActiveRecord::Import::PostgreSQLAdapter
29
31
  query_cache.clear if query_cache_enabled
30
32
  end
31
33
 
32
- [number_of_inserts, ids]
34
+ if options[:returning].blank?
35
+ ids = returned_values
36
+ elsif options[:primary_key].blank?
37
+ results = returned_values
38
+ else
39
+ # split primary key and returning columns
40
+ ids, results = split_ids_and_results(returned_values, columns, options)
41
+ end
42
+
43
+ ActiveRecord::Import::Result.new([], number_of_inserts, ids, results)
44
+ end
45
+
46
+ def split_ids_and_results(values, columns, options)
47
+ ids = []
48
+ results = []
49
+ id_indexes = Array(options[:primary_key]).map { |key| columns.index(key) }
50
+ returning_indexes = Array(options[:returning]).map { |key| columns.index(key) }
51
+
52
+ values.each do |value|
53
+ value_array = Array(value)
54
+ ids << id_indexes.map { |i| value_array[i] }
55
+ results << returning_indexes.map { |i| value_array[i] }
56
+ end
57
+
58
+ ids.map!(&:first) if id_indexes.size == 1
59
+ results.map!(&:first) if returning_indexes.size == 1
60
+
61
+ [ids, results]
33
62
  end
34
63
 
35
64
  def next_value_for_sequence(sequence_name)
@@ -50,14 +79,21 @@ module ActiveRecord::Import::PostgreSQLAdapter
50
79
 
51
80
  sql += super(table_name, options)
52
81
 
53
- unless options[:primary_key].blank? || options[:no_returning]
54
- primary_key = Array(options[:primary_key])
55
- sql << " RETURNING \"#{primary_key.join('", "')}\""
82
+ columns = returning_columns(options)
83
+ unless columns.blank? || options[:no_returning]
84
+ sql << " RETURNING \"#{columns.join('", "')}\""
56
85
  end
57
86
 
58
87
  sql
59
88
  end
60
89
 
90
+ def returning_columns(options)
91
+ columns = []
92
+ columns += Array(options[:primary_key]) if options[:primary_key].present?
93
+ columns |= Array(options[:returning]) if options[:returning].present?
94
+ columns
95
+ end
96
+
61
97
  # Add a column to be updated on duplicate key update
62
98
  def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
63
99
  arg = options[:on_duplicate_key_update]
@@ -83,7 +119,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
83
119
  # Returns a generated ON CONFLICT DO UPDATE statement given the passed
84
120
  # in +args+.
85
121
  def sql_for_on_duplicate_key_update( table_name, *args ) # :nodoc:
86
- arg, primary_key = args
122
+ arg, primary_key, locking_column = args
87
123
  arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
88
124
  return unless arg.is_a?( Hash )
89
125
 
@@ -103,9 +139,9 @@ module ActiveRecord::Import::PostgreSQLAdapter
103
139
 
104
140
  sql << "#{conflict_target}DO UPDATE SET "
105
141
  if columns.is_a?( Array )
106
- sql << sql_for_on_duplicate_key_update_as_array( table_name, columns )
142
+ sql << sql_for_on_duplicate_key_update_as_array( table_name, locking_column, columns )
107
143
  elsif columns.is_a?( Hash )
108
- sql << sql_for_on_duplicate_key_update_as_hash( table_name, columns )
144
+ sql << sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, columns )
109
145
  elsif columns.is_a?( String )
110
146
  sql << columns
111
147
  else
@@ -117,20 +153,22 @@ module ActiveRecord::Import::PostgreSQLAdapter
117
153
  sql
118
154
  end
119
155
 
120
- def sql_for_on_duplicate_key_update_as_array( table_name, arr ) # :nodoc:
156
+ def sql_for_on_duplicate_key_update_as_array( table_name, locking_column, arr ) # :nodoc:
121
157
  results = arr.map do |column|
122
158
  qc = quote_column_name( column )
123
159
  "#{qc}=EXCLUDED.#{qc}"
124
160
  end
161
+ increment_locking_column!(results, locking_column)
125
162
  results.join( ',' )
126
163
  end
127
164
 
128
- def sql_for_on_duplicate_key_update_as_hash( table_name, hsh ) # :nodoc:
165
+ def sql_for_on_duplicate_key_update_as_hash( table_name, locking_column, hsh ) # :nodoc:
129
166
  results = hsh.map do |column1, column2|
130
167
  qc1 = quote_column_name( column1 )
131
168
  qc2 = quote_column_name( column2 )
132
169
  "#{qc1}=EXCLUDED.#{qc2}"
133
170
  end
171
+ increment_locking_column!(results, locking_column)
134
172
  results.join( ',' )
135
173
  end
136
174
 
@@ -141,7 +179,7 @@ module ActiveRecord::Import::PostgreSQLAdapter
141
179
  if constraint_name.present?
142
180
  "ON CONSTRAINT #{constraint_name} "
143
181
  elsif conflict_target.present?
144
- '(' << Array( conflict_target ).reject( &:empty? ).join( ', ' ) << ') '.tap do |sql|
182
+ '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
145
183
  sql << "WHERE #{index_predicate} " if index_predicate
146
184
  end
147
185
  end
@@ -161,7 +199,13 @@ module ActiveRecord::Import::PostgreSQLAdapter
161
199
  current_version >= MIN_VERSION_FOR_UPSERT
162
200
  end
163
201
 
164
- def support_setting_primary_key_of_imported_objects?
202
+ def supports_setting_primary_key_of_imported_objects?
165
203
  true
166
204
  end
205
+
206
+ def increment_locking_column!(results, locking_column)
207
+ if locking_column.present?
208
+ results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
209
+ end
210
+ end
167
211
  end
@@ -1,7 +1,9 @@
1
1
  module ActiveRecord::Import::SQLite3Adapter
2
2
  include ActiveRecord::Import::ImportSupport
3
+ include ActiveRecord::Import::OnDuplicateKeyUpdateSupport
3
4
 
4
5
  MIN_VERSION_FOR_IMPORT = "3.7.11".freeze
6
+ MIN_VERSION_FOR_UPSERT = "3.24.0".freeze
5
7
  SQLITE_LIMIT_COMPOUND_SELECT = 500
6
8
 
7
9
  # Override our conformance to ActiveRecord::Import::ImportSupport interface
@@ -15,6 +17,10 @@ module ActiveRecord::Import::SQLite3Adapter
15
17
  end
16
18
  end
17
19
 
20
+ def supports_on_duplicate_key_update?(current_version = sqlite_version)
21
+ current_version >= MIN_VERSION_FOR_UPSERT
22
+ end
23
+
18
24
  # +sql+ can be a single string or an array. If it is an array all
19
25
  # elements that are in position >= 1 will be appended to the final SQL.
20
26
  def insert_many( sql, values, _options = {}, *args ) # :nodoc:
@@ -37,19 +43,136 @@ module ActiveRecord::Import::SQLite3Adapter
37
43
  end
38
44
  end
39
45
 
40
- [number_of_inserts, []]
46
+ ActiveRecord::Import::Result.new([], number_of_inserts, [], [])
41
47
  end
42
48
 
43
- def pre_sql_statements( options)
49
+ def pre_sql_statements( options )
44
50
  sql = []
45
51
  # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
46
- if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:recursive]
52
+ if !supports_on_duplicate_key_update? && (options[:ignore] || options[:on_duplicate_key_ignore])
47
53
  sql << "OR IGNORE"
48
54
  end
49
55
  sql + super
50
56
  end
51
57
 
58
+ def post_sql_statements( table_name, options ) # :nodoc:
59
+ sql = []
60
+
61
+ if supports_on_duplicate_key_update?
62
+ # Options :recursive and :on_duplicate_key_ignore are mutually exclusive
63
+ if (options[:ignore] || options[:on_duplicate_key_ignore]) && !options[:on_duplicate_key_update]
64
+ sql << sql_for_on_duplicate_key_ignore( options[:on_duplicate_key_ignore] )
65
+ end
66
+ end
67
+
68
+ sql + super
69
+ end
70
+
52
71
  def next_value_for_sequence(sequence_name)
53
72
  %{nextval('#{sequence_name}')}
54
73
  end
74
+
75
+ # Add a column to be updated on duplicate key update
76
+ def add_column_for_on_duplicate_key_update( column, options = {} ) # :nodoc:
77
+ arg = options[:on_duplicate_key_update]
78
+ if arg.is_a?( Hash )
79
+ columns = arg.fetch( :columns ) { arg[:columns] = [] }
80
+ case columns
81
+ when Array then columns << column.to_sym unless columns.include?( column.to_sym )
82
+ when Hash then columns[column.to_sym] = column.to_sym
83
+ end
84
+ elsif arg.is_a?( Array )
85
+ arg << column.to_sym unless arg.include?( column.to_sym )
86
+ end
87
+ end
88
+
89
+ # Returns a generated ON CONFLICT DO NOTHING statement given the passed
90
+ # in +args+.
91
+ def sql_for_on_duplicate_key_ignore( *args ) # :nodoc:
92
+ arg = args.first
93
+ conflict_target = sql_for_conflict_target( arg ) if arg.is_a?( Hash )
94
+ " ON CONFLICT #{conflict_target}DO NOTHING"
95
+ end
96
+
97
+ # Returns a generated ON CONFLICT DO UPDATE statement given the passed
98
+ # in +args+.
99
+ def sql_for_on_duplicate_key_update( _table_name, *args ) # :nodoc:
100
+ arg, primary_key, locking_column = args
101
+ arg = { columns: arg } if arg.is_a?( Array ) || arg.is_a?( String )
102
+ return unless arg.is_a?( Hash )
103
+
104
+ sql = ' ON CONFLICT '
105
+ conflict_target = sql_for_conflict_target( arg )
106
+
107
+ columns = arg.fetch( :columns, [] )
108
+ condition = arg[:condition]
109
+ if columns.respond_to?( :empty? ) && columns.empty?
110
+ return sql << "#{conflict_target}DO NOTHING"
111
+ end
112
+
113
+ conflict_target ||= sql_for_default_conflict_target( primary_key )
114
+ unless conflict_target
115
+ raise ArgumentError, 'Expected :conflict_target to be specified'
116
+ end
117
+
118
+ sql << "#{conflict_target}DO UPDATE SET "
119
+ if columns.is_a?( Array )
120
+ sql << sql_for_on_duplicate_key_update_as_array( locking_column, columns )
121
+ elsif columns.is_a?( Hash )
122
+ sql << sql_for_on_duplicate_key_update_as_hash( locking_column, columns )
123
+ elsif columns.is_a?( String )
124
+ sql << columns
125
+ else
126
+ raise ArgumentError, 'Expected :columns to be an Array or Hash'
127
+ end
128
+
129
+ sql << " WHERE #{condition}" if condition.present?
130
+
131
+ sql
132
+ end
133
+
134
+ def sql_for_on_duplicate_key_update_as_array( locking_column, arr ) # :nodoc:
135
+ results = arr.map do |column|
136
+ qc = quote_column_name( column )
137
+ "#{qc}=EXCLUDED.#{qc}"
138
+ end
139
+ increment_locking_column!(results, locking_column)
140
+ results.join( ',' )
141
+ end
142
+
143
+ def sql_for_on_duplicate_key_update_as_hash( locking_column, hsh ) # :nodoc:
144
+ results = hsh.map do |column1, column2|
145
+ qc1 = quote_column_name( column1 )
146
+ qc2 = quote_column_name( column2 )
147
+ "#{qc1}=EXCLUDED.#{qc2}"
148
+ end
149
+ increment_locking_column!(results, locking_column)
150
+ results.join( ',' )
151
+ end
152
+
153
+ def sql_for_conflict_target( args = {} )
154
+ conflict_target = args[:conflict_target]
155
+ index_predicate = args[:index_predicate]
156
+ if conflict_target.present?
157
+ '(' << Array( conflict_target ).reject( &:blank? ).join( ', ' ) << ') '.tap do |sql|
158
+ sql << "WHERE #{index_predicate} " if index_predicate
159
+ end
160
+ end
161
+ end
162
+
163
+ def sql_for_default_conflict_target( primary_key )
164
+ conflict_target = Array(primary_key).join(', ')
165
+ "(#{conflict_target}) " if conflict_target.present?
166
+ end
167
+
168
+ # Return true if the statement is a duplicate key record error
169
+ def duplicate_key_update_error?(exception) # :nodoc:
170
+ exception.is_a?(ActiveRecord::StatementInvalid) && exception.to_s.include?('duplicate key')
171
+ end
172
+
173
+ def increment_locking_column!(results, locking_column)
174
+ if locking_column.present?
175
+ results << "\"#{locking_column}\"=EXCLUDED.\"#{locking_column}\"+1"
176
+ end
177
+ end
55
178
  end
@@ -11,6 +11,7 @@ module ActiveRecord::Import
11
11
  when 'mysql2spatial' then 'mysql2'
12
12
  when 'spatialite' then 'sqlite3'
13
13
  when 'postgresql_makara' then 'postgresql'
14
+ when 'makara_postgis' then 'postgresql'
14
15
  when 'postgis' then 'postgresql'
15
16
  else adapter
16
17
  end
@@ -18,12 +19,9 @@ module ActiveRecord::Import
18
19
 
19
20
  # Loads the import functionality for a specific database adapter
20
21
  def self.require_adapter(adapter)
21
- require File.join(ADAPTER_PATH, "/abstract_adapter")
22
- begin
23
- require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
24
- rescue LoadError
25
- # fallback
26
- end
22
+ require File.join(ADAPTER_PATH, "/#{base_adapter(adapter)}_adapter")
23
+ rescue LoadError
24
+ # fallback
27
25
  end
28
26
 
29
27
  # Loads the import functionality for the passed in ActiveRecord connection