bulk_update 1.1.4 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 67b504f96d894fd77d2298b021c3176c7ba46fc0
4
+ data.tar.gz: 5eb6122b5aed465c5765a5f3438a6d9e84e829b2
5
+ SHA512:
6
+ metadata.gz: 42e704fca218625c5f77d3c479e6b678fb42cf9f6e4e1e1d6b0a498125cbdae7cef9ee9a6bbdd51393f988b20cb15ee72b4d73df5a156210626c3f98ece93469
7
+ data.tar.gz: 7aeb01c91beea5431af17d81795ee3329c1760ee83a0e5562a281b18a99421f46472ca3e7b265e928df89d9e4943103cf90c127756eae62a0ef81c25f772d14c
data/README.md CHANGED
@@ -1,4 +1,5 @@
1
1
  # BulkUpdate
2
+ [![Code Climate](https://codeclimate.com/github/InWork/bulk_update.png)](https://codeclimate.com/github/InWork/bulk_update)
2
3
 
3
4
  Updates a large amount of Records in a highliy efficient way.
4
5
  Enhances Active Record with a method for bulk inserts and a method for bulk updates. Both methods are used for inserting or updating large amount of Records.
@@ -2,7 +2,7 @@ module BulkUpdate
2
2
  module ActiveRecordInflections
3
3
  #
4
4
  # Clone the database structure of a table
5
- def clone_table args = {}
5
+ def clone_table(args = {})
6
6
  if args[:to]
7
7
  case ActiveRecord::Base.connection_config[:adapter]
8
8
  when 'sqlite3'
@@ -17,7 +17,7 @@ module BulkUpdate
17
17
  end
18
18
 
19
19
 
20
- def insert_str element
20
+ def insert_str(element)
21
21
  if element.class == Fixnum || element.class == Float
22
22
  element
23
23
  elsif element.class == NilClass
@@ -34,15 +34,15 @@ module BulkUpdate
34
34
 
35
35
  #
36
36
  # Bulk insert records
37
- def bulk_insert columns, values, args = {}
37
+ def bulk_insert(columns, values, args = {})
38
38
  # Limit inserts
39
39
  max_records_per_insert = args[:max_records_per_insert] || 100
40
- table = args[:into] || table_name
41
- columns = columns.clone
40
+ table = args[:into] || table_name
41
+ columns = columns.clone
42
42
 
43
43
  # Add timestamp
44
- timestamp = Time.now.to_s(:db)
45
- add_timestamp = false
44
+ timestamp = Time.now.to_s(:db)
45
+ add_timestamp = false
46
46
  add_updated_at = false
47
47
  unless columns.map(&:to_sym).include?(:created_at)
48
48
  columns << :created_at
@@ -84,14 +84,14 @@ module BulkUpdate
84
84
  #
85
85
  # Create, update and delete Records according to a set of new values through ActiveRecord but optimized for performance by
86
86
  # finding all diferences by SQL.
87
- def bulk_update columns, values, args = {}
88
- temp_table = "#{table_name}_temp_table_#{$$}"
89
- key = args[:key] || args[:keys] || 'id'
90
- condition = args[:condition]
87
+ def bulk_update(columns, values, args = {})
88
+ temp_table = "#{table_name}_temp_table_#{$$}"
89
+ key = args[:key] || args[:keys] || 'id'
90
+ condition = args[:condition]
91
91
  exclude_fields = args[:exclude_fields]
92
- insert = args[:insert].nil? ? true : args[:insert]
93
- update = args[:update].nil? ? true : args[:update]
94
- remove = args[:remove].nil? ? true : args[:remove]
92
+ insert = args[:insert].nil? ? true : args[:insert]
93
+ update = args[:update].nil? ? true : args[:update]
94
+ remove = args[:remove].nil? ? true : args[:remove]
95
95
 
96
96
  # Clone temp-table and load it
97
97
  clone_table to: temp_table
@@ -115,23 +115,30 @@ module BulkUpdate
115
115
  #
116
116
  # Exclude List
117
117
  def default_exclude
118
- ['id', 'version', 'created_at', 'updated_at']
118
+ @default_exclude ||= ['id', 'created_at', 'updated_at']
119
+ end
120
+
121
+
122
+ #
123
+ # Exclude List
124
+ def default_exclude=(excludes)
125
+ @default_exclude = excludes
119
126
  end
120
127
 
121
128
 
122
129
  #
123
130
  # Compare Table of args[:model] with its temporary table args[:compare_with] and return all new records as a Array of Hashes
124
- def get_new_records args = {}
125
- model = args[:for] || self
126
- compare_table = args[:compare_with]
127
- keys = args[:on] || 'id'
128
- exclude = args[:exclude_fields] || []
129
- exclude |= default_exclude
131
+ def get_new_records(args = {})
132
+ model = args[:for] || self
133
+ compare_table = args[:compare_with]
134
+ keys = args[:on] || 'id'
135
+ exclude = args[:exclude_fields] || []
136
+ exclude |= default_exclude - %w(created_at updated_at)
130
137
 
131
138
  # Generate conditions for query and sub-query
132
- conditions = []
133
- conditions2 = []
134
- conditions << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
139
+ conditions = []
140
+ conditions2 = []
141
+ conditions << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
135
142
  conditions2 << "#{args[:condition].gsub('--tt--', model.table_name)}" if args[:condition]
136
143
  if keys.class == String || keys.class == Symbol
137
144
  key = keys.to_s
@@ -141,7 +148,7 @@ module BulkUpdate
141
148
  end
142
149
 
143
150
  # Generate and execute SQL-Statement
144
- condition = conditions.join(' AND ')
151
+ condition = conditions.join(' AND ')
145
152
  condition2 = conditions2.join(' AND ')
146
153
  sql = "SELECT * FROM #{compare_table} WHERE #{condition} #{'AND' unless conditions.blank?} #{compare_table}.#{key} NOT IN " +
147
154
  "(SELECT #{key} FROM #{model.table_name} #{'WHERE' unless conditions2.blank?} #{condition2})"
@@ -163,26 +170,26 @@ module BulkUpdate
163
170
  #
164
171
  # Compare Table of args[:model] with its temporary table args[:compare_with] and return all updated records as a Hash of Hashes whose
165
172
  # key is the ID of the changed record
166
- def get_updated_records args = {}
167
- model = args[:for] || self
168
- compare_table = args[:compare_with]
169
- keys = args[:on] || 'id'
170
- exclude = args[:exclude_fields] || []
171
- exclude |= default_exclude
172
- exclude_virtual = args[:exclude_virtual].nil? ? false : args[:exclude_virtual]
173
+ def get_updated_records(args = {})
174
+ model = args[:for] || self
175
+ compare_table = args[:compare_with]
176
+ keys = args[:on] || 'id'
177
+ exclude = args[:exclude_fields] || []
178
+ exclude |= default_exclude
179
+ exclude_virtual = args[:exclude_virtual].nil? ? false : args[:exclude_virtual]
173
180
 
174
181
  # Generate conditions for query and sub-query
175
- conditions = []
182
+ conditions = []
176
183
  conditions2 = []
177
184
  conditions << "NOT #{model.table_name}.virtual" if exclude_virtual
178
185
  if keys.class == String || keys.class == Symbol
179
- key = keys.to_s
186
+ key = keys.to_s
180
187
  conditions << "#{model.table_name}.#{key} = #{compare_table}.#{key}"
181
- exclude |= [keys.to_s]
188
+ exclude |= [keys.to_s]
182
189
  else
183
- key = keys[0].to_s
190
+ key = keys[0].to_s
184
191
  conditions |= keys.map{|k| "#{model.table_name}.#{k.to_s} = #{compare_table}.#{k.to_s}" }
185
- exclude |= keys.map(&:to_s)
192
+ exclude |= keys.map(&:to_s)
186
193
  end
187
194
  conditions << "#{args[:condition].gsub('--tt--', model.table_name)} AND #{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
188
195
  model.attribute_names.each do |an|
@@ -199,20 +206,20 @@ module BulkUpdate
199
206
  end
200
207
 
201
208
  # Generate and execute SQL-Statement
202
- condition = conditions.join(' AND ')
203
- condition += " AND (#{conditions2.join(' OR ')})" unless conditions2.blank?
204
- compare_columns = attribute_names.select { |e| e != 'id' }.map { |e| "#{compare_table}.#{e}" }.join(', ')
205
- sql = "SELECT #{model.table_name}.id, #{compare_columns} FROM #{model.table_name}, #{compare_table} WHERE #{condition}"
206
- results = ActiveRecord::Base.connection.execute sql
209
+ condition = conditions.join(' AND ')
210
+ condition += " AND (#{conditions2.join(' OR ')})" unless conditions2.blank?
211
+ compare_columns = attribute_names.select { |e| e != 'id' }.map { |e| "#{compare_table}.#{e}" }.join(', ')
212
+ sql = "SELECT #{model.table_name}.id, #{compare_columns} FROM #{model.table_name}, #{compare_table} WHERE #{condition}"
213
+ results = ActiveRecord::Base.connection.execute sql
207
214
 
208
215
  # Generate Hash with id as the key and values as a Hashes of all changed records
209
216
  results_hash = {}
210
- keys_to_log = []
217
+ keys_to_log = []
211
218
  results.each do |attributes|
212
- attributes = attributes2array(attributes)
213
- id = attributes[0]
214
- results_hash[id] = result2hash attributes, exclude
215
- keys_to_log << (args[:debug] ? model.find(id).send(key) : id)
219
+ attributes = attributes2array(attributes)
220
+ id = attributes[0]
221
+ results_hash[id] = result2hash attributes, exclude - %w(updated_at)
222
+ keys_to_log << (args[:debug] ? model.find(id).send(key) : id)
216
223
  end
217
224
  args[:logger].info "Change Records for Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
218
225
 
@@ -222,18 +229,18 @@ module BulkUpdate
222
229
 
223
230
  #
224
231
  # Compare Table of args[:model] with its temporary table args[:compare_with] and return all deleted records as a Array of IDs
225
- def get_deleted_records args = {}
226
- model = args[:for] || self
227
- compare_table = args[:compare_with]
228
- keys = args[:on] || 'id'
232
+ def get_deleted_records(args = {})
233
+ model = args[:for] || self
234
+ compare_table = args[:compare_with]
235
+ keys = args[:on] || 'id'
229
236
  exclude_virtual = args[:exclude_virtual].nil? ? false : args[:exclude_virtual]
230
237
 
231
238
  # Generate conditions for query and sub-query
232
- conditions = []
233
- conditions2 = []
234
- conditions << "NOT #{model.table_name}.virtual" if exclude_virtual
235
- conditions << "#{args[:condition].gsub('--tt--', model.table_name)}" if args[:condition]
236
- conditions2 << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
239
+ conditions = []
240
+ conditions2 = []
241
+ conditions << "NOT #{model.table_name}.virtual" if exclude_virtual
242
+ conditions << "#{args[:condition].gsub('--tt--', model.table_name)}" if args[:condition]
243
+ conditions2 << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
237
244
  if keys.class == String || keys.class == Symbol
238
245
  key = keys.to_s
239
246
  else
@@ -242,19 +249,19 @@ module BulkUpdate
242
249
  end
243
250
 
244
251
  # Generate and execute SQL-Statement
245
- condition = conditions.join(' AND ')
252
+ condition = conditions.join(' AND ')
246
253
  condition2 = conditions2.join(' AND ')
247
- sql = "SELECT id, #{key} FROM #{model.table_name} WHERE #{condition} #{'AND' unless conditions.blank?} #{model.table_name}.#{key} NOT IN " +
248
- "(SELECT #{key} FROM #{compare_table} #{'WHERE' unless conditions2.blank?} #{condition2})"
249
- results = ActiveRecord::Base.connection.execute sql
254
+ sql = "SELECT id, #{key} FROM #{model.table_name} WHERE #{condition} #{'AND' unless conditions.blank?} #{model.table_name}.#{key} NOT IN " +
255
+ "(SELECT #{key} FROM #{compare_table} #{'WHERE' unless conditions2.blank?} #{condition2})"
256
+ results = ActiveRecord::Base.connection.execute sql
250
257
 
251
258
  # Generate Array with ids of all deleted records
252
259
  deleted_records = []
253
- keys_to_log = []
260
+ keys_to_log = []
254
261
  results.each do |attributes|
255
- attributes = attributes2array(attributes)
262
+ attributes = attributes2array(attributes)
256
263
  deleted_records << attributes[0]
257
- keys_to_log << attributes[1]
264
+ keys_to_log << attributes[1]
258
265
  end
259
266
  args[:logger].info "Deleting Records from Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
260
267
  deleted_records
@@ -267,8 +274,8 @@ module BulkUpdate
267
274
  def result2hash(attributes, exclude, attribute_nr = 0)
268
275
  hash = {}
269
276
  attribute_names.each do |an|
270
- hash[an.to_sym] = attributes[attribute_nr] unless exclude.include?(an)
271
- attribute_nr += 1
277
+ hash[an.to_sym] = attributes[attribute_nr] unless exclude.include?(an)
278
+ attribute_nr += 1
272
279
  end
273
280
  hash
274
281
  end
@@ -1,3 +1,3 @@
1
1
  module BulkUpdate
2
- VERSION = "1.1.4"
2
+ VERSION = "1.2.0"
3
3
  end
@@ -7,6 +7,7 @@ describe BulkUpdate do
7
7
  @columns = [:name, :value]
8
8
  @values = [['test1', 'value1'], ['test2', 'value2'], ['test3', 'value3'], ['test4', 'value4']]
9
9
  MyHash.bulk_insert @columns, @values
10
+ @ts = MyHash.first.created_at
10
11
  end
11
12
 
12
13
 
@@ -20,10 +21,43 @@ describe BulkUpdate do
20
21
  MyHash.bulk_update @columns, @values, key: 'name'
21
22
  MyHash.count.should be 4
22
23
  MyHash.where(name: 'test1').first.value.should eq 'value1.1'
24
+ MyHash.where(name: 'test1').first.created_at.to_i.should eq @ts.to_i
25
+ MyHash.where(name: 'test1').first.updated_at.to_i.should eq @ts.to_i
23
26
  MyHash.where(name: 'test2').first.value.should eq 'value2'
27
+ MyHash.where(name: 'test2').first.created_at.to_i.should eq @ts.to_i
28
+ MyHash.where(name: 'test2').first.updated_at.to_i.should eq @ts.to_i
24
29
  MyHash.where(name: 'test3').first.should be nil
25
30
  MyHash.where(name: 'test4').first.value.should eq 'value4.4'
31
+ MyHash.where(name: 'test4').first.created_at.to_i.should eq @ts.to_i
32
+ MyHash.where(name: 'test4').first.updated_at.to_i.should eq @ts.to_i
26
33
  MyHash.where(name: 'test5').first.value.should eq 'value5.5'
34
+ MyHash.where(name: 'test5').first.created_at.to_i.should eq @ts.to_i
35
+ MyHash.where(name: 'test5').first.updated_at.to_i.should eq @ts.to_i
36
+ end
37
+
38
+
39
+ it 'should update the created_at and updated_at timestamps if specified' do
40
+ columns = [:name, :value, :updated_at, :created_at]
41
+ ts_1 = 1.hour.ago
42
+ ts_2 = 2.hour.ago
43
+ ts_1_s = ts_1.to_s(:db)
44
+ ts_2_s = ts_2.to_s(:db)
45
+ values = [['test1', 'value1.1', ts_1_s, ts_2_s], ['test2', 'value2', ts_1_s, ts_2_s], ['test4', 'value4.4', ts_1_s, ts_2_s], ['test5', 'value5.5', ts_1_s, ts_2_s]]
46
+ MyHash.bulk_update columns, values, key: 'name'
47
+ MyHash.count.should be 4
48
+ MyHash.where(name: 'test1').first.value.should eq 'value1.1'
49
+ MyHash.where(name: 'test1').first.created_at.to_i.should eq @ts.to_i
50
+ MyHash.where(name: 'test1').first.updated_at.to_i.should eq ts_1.to_i
51
+ MyHash.where(name: 'test2').first.value.should eq 'value2'
52
+ MyHash.where(name: 'test2').first.created_at.to_i.should eq @ts.to_i
53
+ MyHash.where(name: 'test2').first.updated_at.to_i.should eq @ts.to_i
54
+ MyHash.where(name: 'test3').first.should be nil
55
+ MyHash.where(name: 'test4').first.value.should eq 'value4.4'
56
+ MyHash.where(name: 'test4').first.created_at.to_i.should eq @ts.to_i
57
+ MyHash.where(name: 'test4').first.updated_at.to_i.should eq ts_1.to_i
58
+ MyHash.where(name: 'test5').first.value.should eq 'value5.5'
59
+ MyHash.where(name: 'test5').first.created_at.to_i.should eq ts_2.to_i
60
+ MyHash.where(name: 'test5').first.updated_at.to_i.should eq ts_1.to_i
27
61
  end
28
62
 
29
63
  end
metadata CHANGED
@@ -1,110 +1,97 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bulk_update
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.4
5
- prerelease:
4
+ version: 1.2.0
6
5
  platform: ruby
7
6
  authors:
8
7
  - Philip Kurmann
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-05-02 00:00:00.000000000 Z
11
+ date: 2015-01-26 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: activerecord
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - ">="
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - ">="
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: rspec
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - ">="
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :development
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - ">="
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: sqlite3
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: '0'
54
48
  type: :development
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: '0'
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: pg
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ! '>='
59
+ - - ">="
68
60
  - !ruby/object:Gem::Version
69
61
  version: '0'
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ! '>='
66
+ - - ">="
76
67
  - !ruby/object:Gem::Version
77
68
  version: '0'
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: mysql2
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
- - - ! '>='
73
+ - - ">="
84
74
  - !ruby/object:Gem::Version
85
75
  version: '0'
86
76
  type: :development
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
- - - ! '>='
80
+ - - ">="
92
81
  - !ruby/object:Gem::Version
93
82
  version: '0'
94
83
  - !ruby/object:Gem::Dependency
95
84
  name: pry
96
85
  requirement: !ruby/object:Gem::Requirement
97
- none: false
98
86
  requirements:
99
- - - ! '>='
87
+ - - ">="
100
88
  - !ruby/object:Gem::Version
101
89
  version: '0'
102
90
  type: :development
103
91
  prerelease: false
104
92
  version_requirements: !ruby/object:Gem::Requirement
105
- none: false
106
93
  requirements:
107
- - - ! '>='
94
+ - - ">="
108
95
  - !ruby/object:Gem::Version
109
96
  version: '0'
110
97
  description: Updates a large amount of Records in a highly efficient way
@@ -114,7 +101,7 @@ executables: []
114
101
  extensions: []
115
102
  extra_rdoc_files: []
116
103
  files:
117
- - .gitignore
104
+ - ".gitignore"
118
105
  - Gemfile
119
106
  - LICENSE
120
107
  - README.md
@@ -130,27 +117,26 @@ files:
130
117
  - spec/support/schema.rb
131
118
  homepage: ''
132
119
  licenses: []
120
+ metadata: {}
133
121
  post_install_message:
134
122
  rdoc_options: []
135
123
  require_paths:
136
124
  - lib
137
125
  required_ruby_version: !ruby/object:Gem::Requirement
138
- none: false
139
126
  requirements:
140
- - - ! '>='
127
+ - - ">="
141
128
  - !ruby/object:Gem::Version
142
129
  version: '0'
143
130
  required_rubygems_version: !ruby/object:Gem::Requirement
144
- none: false
145
131
  requirements:
146
- - - ! '>='
132
+ - - ">="
147
133
  - !ruby/object:Gem::Version
148
134
  version: '0'
149
135
  requirements: []
150
136
  rubyforge_project:
151
- rubygems_version: 1.8.23
137
+ rubygems_version: 2.2.2
152
138
  signing_key:
153
- specification_version: 3
139
+ specification_version: 4
154
140
  summary: Enhances Active Record with a method for bulk inserts and a method for bulk
155
141
  updates. Both merthods are used for inserting or updating large amount of Records.
156
142
  test_files: