activerecord-import 0.14.1 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/test.yaml +107 -0
  3. data/.gitignore +1 -0
  4. data/.rubocop.yml +74 -8
  5. data/Brewfile +3 -1
  6. data/CHANGELOG.md +448 -2
  7. data/Gemfile +26 -19
  8. data/LICENSE +21 -56
  9. data/README.markdown +568 -32
  10. data/Rakefile +5 -1
  11. data/activerecord-import.gemspec +8 -7
  12. data/benchmarks/README +2 -2
  13. data/benchmarks/benchmark.rb +8 -1
  14. data/benchmarks/lib/base.rb +2 -0
  15. data/benchmarks/lib/cli_parser.rb +5 -2
  16. data/benchmarks/lib/float.rb +2 -0
  17. data/benchmarks/lib/mysql2_benchmark.rb +2 -0
  18. data/benchmarks/lib/output_to_csv.rb +2 -0
  19. data/benchmarks/lib/output_to_html.rb +4 -2
  20. data/benchmarks/models/test_innodb.rb +2 -0
  21. data/benchmarks/models/test_memory.rb +2 -0
  22. data/benchmarks/models/test_myisam.rb +2 -0
  23. data/benchmarks/schema/{mysql_schema.rb → mysql2_schema.rb} +2 -0
  24. data/gemfiles/4.2.gemfile +4 -3
  25. data/gemfiles/5.0.gemfile +4 -3
  26. data/gemfiles/5.1.gemfile +4 -0
  27. data/gemfiles/5.2.gemfile +4 -0
  28. data/gemfiles/6.0.gemfile +4 -0
  29. data/gemfiles/6.1.gemfile +4 -0
  30. data/gemfiles/7.0.gemfile +4 -0
  31. data/lib/activerecord-import/active_record/adapters/abstract_adapter.rb +2 -0
  32. data/lib/activerecord-import/active_record/adapters/jdbcmysql_adapter.rb +6 -4
  33. data/lib/activerecord-import/active_record/adapters/jdbcpostgresql_adapter.rb +2 -0
  34. data/lib/activerecord-import/active_record/adapters/jdbcsqlite3_adapter.rb +8 -0
  35. data/lib/activerecord-import/active_record/adapters/mysql2_adapter.rb +2 -0
  36. data/lib/activerecord-import/active_record/adapters/postgresql_adapter.rb +2 -0
  37. data/lib/activerecord-import/active_record/adapters/seamless_database_pool_adapter.rb +2 -0
  38. data/lib/activerecord-import/active_record/adapters/sqlite3_adapter.rb +2 -0
  39. data/lib/activerecord-import/adapters/abstract_adapter.rb +12 -16
  40. data/lib/activerecord-import/adapters/em_mysql2_adapter.rb +2 -0
  41. data/lib/activerecord-import/adapters/mysql2_adapter.rb +2 -0
  42. data/lib/activerecord-import/adapters/mysql_adapter.rb +35 -18
  43. data/lib/activerecord-import/adapters/postgresql_adapter.rb +107 -28
  44. data/lib/activerecord-import/adapters/sqlite3_adapter.rb +141 -18
  45. data/lib/activerecord-import/base.rb +15 -8
  46. data/lib/activerecord-import/import.rb +642 -178
  47. data/lib/activerecord-import/mysql2.rb +2 -0
  48. data/lib/activerecord-import/postgresql.rb +2 -0
  49. data/lib/activerecord-import/sqlite3.rb +2 -0
  50. data/lib/activerecord-import/synchronize.rb +5 -3
  51. data/lib/activerecord-import/value_sets_parser.rb +29 -3
  52. data/lib/activerecord-import/version.rb +3 -1
  53. data/lib/activerecord-import.rb +5 -16
  54. data/test/adapters/jdbcmysql.rb +2 -0
  55. data/test/adapters/jdbcpostgresql.rb +2 -0
  56. data/test/adapters/jdbcsqlite3.rb +3 -0
  57. data/test/adapters/makara_postgis.rb +3 -0
  58. data/test/adapters/mysql2.rb +2 -0
  59. data/test/adapters/mysql2_makara.rb +2 -0
  60. data/test/adapters/mysql2spatial.rb +2 -0
  61. data/test/adapters/postgis.rb +2 -0
  62. data/test/adapters/postgresql.rb +2 -0
  63. data/test/adapters/postgresql_makara.rb +2 -0
  64. data/test/adapters/seamless_database_pool.rb +2 -0
  65. data/test/adapters/spatialite.rb +2 -0
  66. data/test/adapters/sqlite3.rb +2 -0
  67. data/test/{travis → github}/database.yml +7 -1
  68. data/test/import_test.rb +498 -32
  69. data/test/jdbcmysql/import_test.rb +2 -1
  70. data/test/jdbcpostgresql/import_test.rb +2 -1
  71. data/test/jdbcsqlite3/import_test.rb +6 -0
  72. data/test/makara_postgis/import_test.rb +10 -0
  73. data/test/models/account.rb +5 -0
  74. data/test/models/alarm.rb +4 -0
  75. data/test/models/animal.rb +8 -0
  76. data/test/models/bike_maker.rb +9 -0
  77. data/test/models/book.rb +4 -0
  78. data/test/models/car.rb +5 -0
  79. data/test/models/card.rb +5 -0
  80. data/test/models/chapter.rb +2 -0
  81. data/test/models/customer.rb +8 -0
  82. data/test/models/deck.rb +8 -0
  83. data/test/models/dictionary.rb +6 -0
  84. data/test/models/discount.rb +2 -0
  85. data/test/models/end_note.rb +2 -0
  86. data/test/models/group.rb +2 -0
  87. data/test/models/order.rb +8 -0
  88. data/test/models/playing_card.rb +4 -0
  89. data/test/models/promotion.rb +2 -0
  90. data/test/models/question.rb +2 -0
  91. data/test/models/rule.rb +2 -0
  92. data/test/models/tag.rb +7 -0
  93. data/test/models/tag_alias.rb +5 -0
  94. data/test/models/topic.rb +16 -0
  95. data/test/models/user.rb +5 -0
  96. data/test/models/user_token.rb +6 -0
  97. data/test/models/vendor.rb +9 -0
  98. data/test/models/widget.rb +18 -0
  99. data/test/mysql2/import_test.rb +2 -0
  100. data/test/mysql2_makara/import_test.rb +2 -0
  101. data/test/mysqlspatial2/import_test.rb +2 -0
  102. data/test/postgis/import_test.rb +6 -0
  103. data/test/postgresql/import_test.rb +2 -4
  104. data/test/schema/generic_schema.rb +88 -3
  105. data/test/schema/jdbcpostgresql_schema.rb +3 -0
  106. data/test/schema/mysql2_schema.rb +21 -0
  107. data/test/schema/postgis_schema.rb +3 -0
  108. data/test/schema/postgresql_schema.rb +63 -0
  109. data/test/schema/sqlite3_schema.rb +15 -0
  110. data/test/schema/version.rb +2 -0
  111. data/test/sqlite3/import_test.rb +4 -50
  112. data/test/support/active_support/test_case_extensions.rb +8 -1
  113. data/test/support/assertions.rb +2 -0
  114. data/test/support/factories.rb +17 -8
  115. data/test/support/generate.rb +10 -8
  116. data/test/support/mysql/import_examples.rb +17 -3
  117. data/test/support/postgresql/import_examples.rb +442 -9
  118. data/test/support/shared_examples/on_duplicate_key_ignore.rb +45 -0
  119. data/test/support/shared_examples/on_duplicate_key_update.rb +278 -1
  120. data/test/support/shared_examples/recursive_import.rb +137 -12
  121. data/test/support/sqlite3/import_examples.rb +232 -0
  122. data/test/synchronize_test.rb +10 -0
  123. data/test/test_helper.rb +44 -3
  124. data/test/value_sets_bytes_parser_test.rb +15 -2
  125. data/test/value_sets_records_parser_test.rb +2 -0
  126. metadata +74 -22
  127. data/.travis.yml +0 -52
  128. data/gemfiles/3.2.gemfile +0 -3
  129. data/gemfiles/4.0.gemfile +0 -3
  130. data/gemfiles/4.1.gemfile +0 -3
  131. data/test/schema/mysql_schema.rb +0 -16
data/lib/activerecord-import/mysql2.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  warn <<-MSG
2
4
  [DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
3
5
  is deprecated. Update to autorequire using 'require "activerecord-import"'. See
data/lib/activerecord-import/postgresql.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  warn <<-MSG
2
4
  [DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
3
5
  is deprecated. Update to autorequire using 'require "activerecord-import"'. See
data/lib/activerecord-import/sqlite3.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  warn <<-MSG
2
4
  [DEPRECATION] loading activerecord-import via 'require "activerecord-import/<adapter-name>"'
3
5
  is deprecated. Update to autorequire using 'require "activerecord-import"'. See
data/lib/activerecord-import/synchronize.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module ActiveRecord # :nodoc:
2
4
  class Base # :nodoc:
3
5
  # Synchronizes the passed in ActiveRecord instances with data
@@ -31,7 +33,7 @@ module ActiveRecord # :nodoc:
31
33
 
32
34
  klass = instances.first.class
33
35
 
34
- fresh_instances = klass.where(conditions).order(order)
36
+ fresh_instances = klass.unscoped.where(conditions).order(order)
35
37
  instances.each do |instance|
36
38
  matched_instance = fresh_instances.detect do |fresh_instance|
37
39
  keys.all? { |key| fresh_instance.send(key) == instance.send(key) }
@@ -39,8 +41,8 @@ module ActiveRecord # :nodoc:
39
41
 
40
42
  next unless matched_instance
41
43
 
42
- instance.send :clear_aggregation_cache
43
- instance.send :clear_association_cache
44
+ instance.instance_variable_set :@association_cache, {}
45
+ instance.send :clear_aggregation_cache if instance.respond_to?(:clear_aggregation_cache, true)
44
46
  instance.instance_variable_set :@attributes, matched_instance.instance_variable_get(:@attributes)
45
47
 
46
48
  if instance.respond_to?(:clear_changes_information)
data/lib/activerecord-import/value_sets_parser.rb CHANGED
@@ -1,4 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'active_support/core_ext/array'
4
+
1
5
  module ActiveRecord::Import
6
+ class ValueSetTooLargeError < StandardError
7
+ attr_reader :size
8
+ def initialize(msg = "Value set exceeds max size", size = 0)
9
+ @size = size
10
+ super(msg)
11
+ end
12
+ end
13
+
2
14
  class ValueSetsBytesParser
3
15
  attr_reader :reserved_bytes, :max_bytes, :values
4
16
 
@@ -8,8 +20,8 @@ module ActiveRecord::Import
8
20
 
9
21
  def initialize(values, options)
10
22
  @values = values
11
- @reserved_bytes = options[:reserved_bytes]
12
- @max_bytes = options[:max_bytes]
23
+ @reserved_bytes = options[:reserved_bytes] || 0
24
+ @max_bytes = options.fetch(:max_bytes) { default_max_bytes }
13
25
  end
14
26
 
15
27
  def parse
@@ -18,6 +30,12 @@ module ActiveRecord::Import
18
30
  current_size = 0
19
31
  values.each_with_index do |val, i|
20
32
  comma_bytes = arr.size
33
+ insert_size = reserved_bytes + val.bytesize
34
+
35
+ if insert_size > max_bytes
36
+ raise ValueSetTooLargeError.new("#{insert_size} bytes exceeds the max allowed for an insert [#{@max_bytes}]", insert_size)
37
+ end
38
+
21
39
  bytes_thus_far = reserved_bytes + current_size + val.bytesize + comma_bytes
22
40
  if bytes_thus_far <= max_bytes
23
41
  current_size += val.bytesize
@@ -32,7 +50,15 @@ module ActiveRecord::Import
32
50
  value_sets << arr if i == (values.size - 1)
33
51
  end
34
52
 
35
- [*value_sets]
53
+ value_sets
54
+ end
55
+
56
+ private
57
+
58
+ def default_max_bytes
59
+ values_in_bytes = values.sum(&:bytesize)
60
+ comma_separated_bytes = values.size - 1
61
+ reserved_bytes + values_in_bytes + comma_separated_bytes
36
62
  end
37
63
  end
38
64
 
data/lib/activerecord-import/version.rb CHANGED
@@ -1,5 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module ActiveRecord
2
4
  module Import
3
- VERSION = "0.14.1".freeze
5
+ VERSION = "1.4.1"
4
6
  end
5
7
  end
data/lib/activerecord-import.rb CHANGED
@@ -1,19 +1,8 @@
1
- # rubocop:disable Style/FileName
1
+ # rubocop:disable Naming/FileName
2
+ # frozen_string_literal: true
2
3
 
3
- ActiveSupport.on_load(:active_record) do
4
- class ActiveRecord::Base
5
- class << self
6
- def establish_connection_with_activerecord_import(*args)
7
- conn = establish_connection_without_activerecord_import(*args)
8
- if !ActiveRecord.const_defined?(:Import) || !ActiveRecord::Import.respond_to?(:load_from_connection_pool)
9
- require "activerecord-import/base"
10
- end
4
+ require "active_support/lazy_load_hooks"
11
5
 
12
- ActiveRecord::Import.load_from_connection_pool connection_pool
13
- conn
14
- end
15
- alias establish_connection_without_activerecord_import establish_connection
16
- alias establish_connection establish_connection_with_activerecord_import
17
- end
18
- end
6
+ ActiveSupport.on_load(:active_record) do
7
+ require "activerecord-import/base"
19
8
  end
data/test/adapters/jdbcmysql.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "jdbcmysql"
data/test/adapters/jdbcpostgresql.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "jdbcpostgresql"
data/test/adapters/jdbcsqlite3.rb ADDED
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ ENV["ARE_DB"] = "jdbcsqlite3"
data/test/adapters/makara_postgis.rb ADDED
@@ -0,0 +1,3 @@
1
+ # frozen_string_literal: true
2
+
3
+ ENV["ARE_DB"] = "postgis"
data/test/adapters/mysql2.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "mysql2"
data/test/adapters/mysql2_makara.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "mysql2_makara"
data/test/adapters/mysql2spatial.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "mysql2spatial"
data/test/adapters/postgis.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "postgis"
data/test/adapters/postgresql.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "postgresql"
data/test/adapters/postgresql_makara.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "postgresql"
data/test/adapters/seamless_database_pool.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "seamless_database_pool"
data/test/adapters/spatialite.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "spatialite"
data/test/adapters/sqlite3.rb CHANGED
@@ -1 +1,3 @@
1
+ # frozen_string_literal: true
2
+
1
3
  ENV["ARE_DB"] = "sqlite3"
data/test/{travis → github}/database.yml RENAMED
@@ -1,7 +1,8 @@
1
1
  common: &common
2
2
  username: root
3
- password:
3
+ password: root
4
4
  encoding: utf8
5
+ collation: utf8_general_ci
5
6
  host: localhost
6
7
  database: activerecord_import_test
7
8
 
@@ -15,6 +16,10 @@ jdbcmysql: &mysql2
15
16
  <<: *common
16
17
  adapter: jdbcmysql
17
18
 
19
+ jdbcsqlite3: &sqlite3
20
+ <<: *common
21
+ adapter: jdbcsqlite3
22
+
18
23
  mysql2: &mysql2
19
24
  <<: *common
20
25
  adapter: mysql2
@@ -33,6 +38,7 @@ oracle:
33
38
  postgresql: &postgresql
34
39
  <<: *common
35
40
  username: postgres
41
+ password: postgres
36
42
  adapter: postgresql
37
43
  min_messages: warning
38
44