data_miner 3.0.0.rc2 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d3203cf46ab37ec16f8d9fdd4e51d379ce09f442
4
+ data.tar.gz: 426543897bff21b3fde15eaac09fb15ff0ae5666
5
+ SHA512:
6
+ metadata.gz: 75e892e73e41479a63f042f6c3b5e1a8513d926b3ea80703ea1fd92e5c8c7097a820b6892d0ca90bffa154abcb2dfd484b2c226e29a95a682d1673e2b1eb12f8
7
+ data.tar.gz: 56adeddbd921d858b310f764712e9313a9d7537a8f1c511aa0d560fc94ca8b4ccc5d9b582744082f5ca3c5ae32820f2ec462f6952bd58500eed6e8c8e8263ae0
data/CHANGELOG CHANGED
@@ -1,45 +1,29 @@
1
- 3.0.0.rc2 / 2013-08-28
1
+ 3.0.0 / 2014-02-04
2
2
 
3
- * Enhancements
4
-
5
- * (Re-)Start logging the beginning, erroring, and ending of steps
6
- * Raise errors if a step fails
7
-
8
- 3.0.0.rc1 / 2013-08-08
9
-
10
- * Enhancements
11
-
12
- * :field_name can be passed as an Array like [ :first_name, :last_name ], which will join them with a space
13
-
14
- 3.0.0.beta / 2013-07-26
15
-
16
- * Enhancements
17
-
18
- * test steps with after: and every: options, referring to row counts in the previous step
19
- * :limit option for import steps - oh yeah
20
- * :random_skip option - goes nicely with :limit if you're debugging
21
-
22
- * Bug fixes
23
-
24
- * Make sure to stringify data_miner(:append) option
25
-
26
- 3.0.0.alpha / 2013-07-24
27
-
28
- * breaking changes
29
-
30
- * :dictionary is now just a plain Hash-like object (responds to []) - and no longer attempts to refresh between runs
31
- * no more unit conversions
32
- * always nullifies blank strings
33
- * always overwrites columns whether there was a non-null value there before or not (no more :overwrite option)
34
- * don't use synthesize option, just pass a block
35
- * synthesized values get the same whitespace compression and stripping as other values
36
- * not tested against the Earth library, which has particular requirements and makes testing too complicated
37
- * doesn't keep DataMiner::Run or DataMiner::Run::ColumnStatistic records
38
- * no more :matcher option
39
-
40
- * enhancements
3
+ * Breaking changes
41
4
 
42
- * if you have a postgres hstore column called "foo", you can do store 'foo.bar'
5
+ * (alpha) :dictionary is now just a plain Hash-like object (responds to []) - and no longer attempts to refresh between runs
6
+ * (alpha) no more unit conversions
7
+ * (alpha) always nullifies blank strings
8
+ * (alpha) always overwrites columns whether there was a non-null value there before or not (no more :overwrite option)
9
+ * (alpha) don't use synthesize option, just pass a block
10
+ * (alpha) synthesized values get the same whitespace compression and stripping as other values
11
+ * (alpha) not tested against the Earth library, which has particular requirements and makes testing too complicated
12
+ * (alpha) doesn't keep DataMiner::Run or DataMiner::Run::ColumnStatistic records
13
+ * (alpha) no more :matcher option
14
+
15
+ * Enhancements
16
+
17
+ * :date_format option takes format for Date.strptime
18
+ * :ignore_error options recovers from some value conversion errors
19
+ * (rc2) (Re-)Start logging the beginning, erroring, and ending of steps
20
+ * (rc2) Raise errors if a step fails
21
+ * (rc1) :field_name can be passed as an Array like [ :first_name, :last_name ], which will join them with a space
22
+ * (beta) test steps with after: and every: options, referring to row counts in the previous step
23
+ * (beta) :limit option for import steps - oh yeah
24
+ * (beta) :random_skip option - goes nicely with :limit if you're debugging
25
+ * (alpha) if you have a postgres hstore column called "foo", you can do store 'foo.bar'
26
+ * add MIT license
43
27
 
44
28
  2.5.2 / 2013-07-05
45
29
 
data/LICENSE CHANGED
@@ -1,4 +1,4 @@
1
- Copyright (c) 2013 Seamus Abshere
1
+ Copyright (c) 2014 Seamus Abshere
2
2
 
3
3
  Permission is hereby granted, free of charge, to any person obtaining
4
4
  a copy of this software and associated documentation files (the
@@ -10,6 +10,8 @@ Gem::Specification.new do |s|
10
10
  s.summary = %{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.}
11
11
  s.description = %q{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. Uses Upsert internally for speed.}
12
12
 
13
+ s.license = 'MIT'
14
+
13
15
  s.rubyforge_project = "data_miner"
14
16
 
15
17
  s.files = `git ls-files`.split("\n")
@@ -1,5 +1,6 @@
1
1
  require 'singleton'
2
2
  require 'set'
3
+ require 'date'
3
4
  require 'active_support'
4
5
  require 'active_support/version'
5
6
  if ::ActiveSupport::VERSION::MAJOR >= 3
@@ -29,12 +29,15 @@ class DataMiner
29
29
  'upcase',
30
30
  'field_number',
31
31
  'chars',
32
+ 'date_format',
33
+ 'ignore_error',
32
34
  ]
33
35
 
34
36
  DEFAULT_SPLIT_PATTERN = /\s+/
35
37
  DEFAULT_SPLIT_KEEP = 0
36
38
  DEFAULT_DELIMITER = ', '
37
39
  DEFAULT_UPCASE = false
40
+ DEFAULT_IGNORE_ERROR = false
38
41
 
39
42
  # activerecord-3.2.6/lib/active_record/connection_adapters/column.rb
40
43
  TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON', 'yes', 'YES', 'y', 'Y']
@@ -91,6 +94,14 @@ class DataMiner
91
94
  # @return [TrueClass,FalseClass]
92
95
  attr_reader :upcase
93
96
 
97
+ # Date format to pass to Date.strptime
98
+ # @return [String]
99
+ attr_reader :date_format
100
+
101
+ # Ignore value conversion errors - value will be nil.
102
+ # @return [TrueClass, FalseClass]
103
+ attr_reader :ignore_error
104
+
94
105
  # Dictionary for translating.
95
106
  #
96
107
  # You pass a Hash or something that responds to []
@@ -108,9 +119,11 @@ class DataMiner
108
119
  @name = name.to_s
109
120
  @synthesize = blk if block_given?
110
121
  @dictionary = options['dictionary']
122
+ @ignore_error = options.fetch 'ignore_error', DEFAULT_IGNORE_ERROR
111
123
  if @static_boolean = options.has_key?('static')
112
124
  @static = options['static']
113
125
  end
126
+ @date_format = options['date_format']
114
127
  @field_number = options['field_number']
115
128
  @field_name_settings = options['field_name']
116
129
  @delimiter = options.fetch 'delimiter', DEFAULT_DELIMITER
@@ -240,10 +253,19 @@ class DataMiner
240
253
  if sprintf
241
254
  value = sprintf % value.to_f
242
255
  end
256
+ if date_format
257
+ value = Date.strptime value.to_s, date_format
258
+ end
243
259
  if dictionary
244
260
  value = dictionary[value]
245
261
  end
246
262
  value
263
+ rescue
264
+ if ignore_error
265
+ DataMiner.logger.debug { "Error in #{name}: #{$!.message}" }
266
+ else
267
+ raise $!
268
+ end
247
269
  end
248
270
 
249
271
  def hstore?
@@ -1,3 +1,3 @@
1
1
  class DataMiner
2
- VERSION = '3.0.0.rc2'
2
+ VERSION = '3.0.0'
3
3
  end
metadata CHANGED
@@ -1,8 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_miner
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.0.0.rc2
5
- prerelease: 6
4
+ version: 3.0.0
6
5
  platform: ruby
7
6
  authors:
8
7
  - Seamus Abshere
@@ -13,342 +12,300 @@ authors:
13
12
  autorequire:
14
13
  bindir: bin
15
14
  cert_chain: []
16
- date: 2013-08-28 00:00:00.000000000 Z
15
+ date: 2014-02-04 00:00:00.000000000 Z
17
16
  dependencies:
18
17
  - !ruby/object:Gem::Dependency
19
18
  name: activerecord
20
19
  requirement: !ruby/object:Gem::Requirement
21
- none: false
22
20
  requirements:
23
- - - ! '>'
21
+ - - ">"
24
22
  - !ruby/object:Gem::Version
25
23
  version: '3'
26
24
  type: :runtime
27
25
  prerelease: false
28
26
  version_requirements: !ruby/object:Gem::Requirement
29
- none: false
30
27
  requirements:
31
- - - ! '>'
28
+ - - ">"
32
29
  - !ruby/object:Gem::Version
33
30
  version: '3'
34
31
  - !ruby/object:Gem::Dependency
35
32
  name: activesupport
36
33
  requirement: !ruby/object:Gem::Requirement
37
- none: false
38
34
  requirements:
39
- - - ! '>'
35
+ - - ">"
40
36
  - !ruby/object:Gem::Version
41
37
  version: '3'
42
38
  type: :runtime
43
39
  prerelease: false
44
40
  version_requirements: !ruby/object:Gem::Requirement
45
- none: false
46
41
  requirements:
47
- - - ! '>'
42
+ - - ">"
48
43
  - !ruby/object:Gem::Version
49
44
  version: '3'
50
45
  - !ruby/object:Gem::Dependency
51
46
  name: errata
52
47
  requirement: !ruby/object:Gem::Requirement
53
- none: false
54
48
  requirements:
55
- - - ! '>='
49
+ - - ">="
56
50
  - !ruby/object:Gem::Version
57
51
  version: 1.0.1
58
52
  type: :runtime
59
53
  prerelease: false
60
54
  version_requirements: !ruby/object:Gem::Requirement
61
- none: false
62
55
  requirements:
63
- - - ! '>='
56
+ - - ">="
64
57
  - !ruby/object:Gem::Version
65
58
  version: 1.0.1
66
59
  - !ruby/object:Gem::Dependency
67
60
  name: remote_table
68
61
  requirement: !ruby/object:Gem::Requirement
69
- none: false
70
62
  requirements:
71
- - - ! '>='
63
+ - - ">="
72
64
  - !ruby/object:Gem::Version
73
65
  version: 2.0.2
74
66
  type: :runtime
75
67
  prerelease: false
76
68
  version_requirements: !ruby/object:Gem::Requirement
77
- none: false
78
69
  requirements:
79
- - - ! '>='
70
+ - - ">="
80
71
  - !ruby/object:Gem::Version
81
72
  version: 2.0.2
82
73
  - !ruby/object:Gem::Dependency
83
74
  name: upsert
84
75
  requirement: !ruby/object:Gem::Requirement
85
- none: false
86
76
  requirements:
87
- - - ! '>='
77
+ - - ">="
88
78
  - !ruby/object:Gem::Version
89
79
  version: 0.3.1
90
80
  type: :runtime
91
81
  prerelease: false
92
82
  version_requirements: !ruby/object:Gem::Requirement
93
- none: false
94
83
  requirements:
95
- - - ! '>='
84
+ - - ">="
96
85
  - !ruby/object:Gem::Version
97
86
  version: 0.3.1
98
87
  - !ruby/object:Gem::Dependency
99
88
  name: posix-spawn
100
89
  requirement: !ruby/object:Gem::Requirement
101
- none: false
102
90
  requirements:
103
- - - ! '>='
91
+ - - ">="
104
92
  - !ruby/object:Gem::Version
105
93
  version: '0'
106
94
  type: :runtime
107
95
  prerelease: false
108
96
  version_requirements: !ruby/object:Gem::Requirement
109
- none: false
110
97
  requirements:
111
- - - ! '>='
98
+ - - ">="
112
99
  - !ruby/object:Gem::Version
113
100
  version: '0'
114
101
  - !ruby/object:Gem::Dependency
115
102
  name: unix_utils
116
103
  requirement: !ruby/object:Gem::Requirement
117
- none: false
118
104
  requirements:
119
- - - ! '>='
105
+ - - ">="
120
106
  - !ruby/object:Gem::Version
121
107
  version: '0'
122
108
  type: :runtime
123
109
  prerelease: false
124
110
  version_requirements: !ruby/object:Gem::Requirement
125
- none: false
126
111
  requirements:
127
- - - ! '>='
112
+ - - ">="
128
113
  - !ruby/object:Gem::Version
129
114
  version: '0'
130
115
  - !ruby/object:Gem::Dependency
131
116
  name: roo
132
117
  requirement: !ruby/object:Gem::Requirement
133
- none: false
134
118
  requirements:
135
- - - ! '>='
119
+ - - ">="
136
120
  - !ruby/object:Gem::Version
137
121
  version: 1.10.3
138
122
  type: :runtime
139
123
  prerelease: false
140
124
  version_requirements: !ruby/object:Gem::Requirement
141
- none: false
142
125
  requirements:
143
- - - ! '>='
126
+ - - ">="
144
127
  - !ruby/object:Gem::Version
145
128
  version: 1.10.3
146
129
  - !ruby/object:Gem::Dependency
147
130
  name: rspec-expectations
148
131
  requirement: !ruby/object:Gem::Requirement
149
- none: false
150
132
  requirements:
151
- - - ! '>='
133
+ - - ">="
152
134
  - !ruby/object:Gem::Version
153
135
  version: '0'
154
136
  type: :runtime
155
137
  prerelease: false
156
138
  version_requirements: !ruby/object:Gem::Requirement
157
- none: false
158
139
  requirements:
159
- - - ! '>='
140
+ - - ">="
160
141
  - !ruby/object:Gem::Version
161
142
  version: '0'
162
143
  - !ruby/object:Gem::Dependency
163
144
  name: rspec
164
145
  requirement: !ruby/object:Gem::Requirement
165
- none: false
166
146
  requirements:
167
- - - ! '>='
147
+ - - ">="
168
148
  - !ruby/object:Gem::Version
169
149
  version: '0'
170
150
  type: :development
171
151
  prerelease: false
172
152
  version_requirements: !ruby/object:Gem::Requirement
173
- none: false
174
153
  requirements:
175
- - - ! '>='
154
+ - - ">="
176
155
  - !ruby/object:Gem::Version
177
156
  version: '0'
178
157
  - !ruby/object:Gem::Dependency
179
158
  name: pry
180
159
  requirement: !ruby/object:Gem::Requirement
181
- none: false
182
160
  requirements:
183
- - - ! '>='
161
+ - - ">="
184
162
  - !ruby/object:Gem::Version
185
163
  version: '0'
186
164
  type: :development
187
165
  prerelease: false
188
166
  version_requirements: !ruby/object:Gem::Requirement
189
- none: false
190
167
  requirements:
191
- - - ! '>='
168
+ - - ">="
192
169
  - !ruby/object:Gem::Version
193
170
  version: '0'
194
171
  - !ruby/object:Gem::Dependency
195
172
  name: active_record_inline_schema
196
173
  requirement: !ruby/object:Gem::Requirement
197
- none: false
198
174
  requirements:
199
- - - ! '>='
175
+ - - ">="
200
176
  - !ruby/object:Gem::Version
201
177
  version: '0'
202
178
  type: :development
203
179
  prerelease: false
204
180
  version_requirements: !ruby/object:Gem::Requirement
205
- none: false
206
181
  requirements:
207
- - - ! '>='
182
+ - - ">="
208
183
  - !ruby/object:Gem::Version
209
184
  version: '0'
210
185
  - !ruby/object:Gem::Dependency
211
186
  name: fuzzy_match
212
187
  requirement: !ruby/object:Gem::Requirement
213
- none: false
214
188
  requirements:
215
- - - ! '>='
189
+ - - ">="
216
190
  - !ruby/object:Gem::Version
217
191
  version: '0'
218
192
  type: :development
219
193
  prerelease: false
220
194
  version_requirements: !ruby/object:Gem::Requirement
221
- none: false
222
195
  requirements:
223
- - - ! '>='
196
+ - - ">="
224
197
  - !ruby/object:Gem::Version
225
198
  version: '0'
226
199
  - !ruby/object:Gem::Dependency
227
200
  name: minitest
228
201
  requirement: !ruby/object:Gem::Requirement
229
- none: false
230
202
  requirements:
231
- - - ! '>='
203
+ - - ">="
232
204
  - !ruby/object:Gem::Version
233
205
  version: '0'
234
206
  type: :development
235
207
  prerelease: false
236
208
  version_requirements: !ruby/object:Gem::Requirement
237
- none: false
238
209
  requirements:
239
- - - ! '>='
210
+ - - ">="
240
211
  - !ruby/object:Gem::Version
241
212
  version: '0'
242
213
  - !ruby/object:Gem::Dependency
243
214
  name: minitest-reporters
244
215
  requirement: !ruby/object:Gem::Requirement
245
- none: false
246
216
  requirements:
247
- - - ! '>='
217
+ - - ">="
248
218
  - !ruby/object:Gem::Version
249
219
  version: '0'
250
220
  type: :development
251
221
  prerelease: false
252
222
  version_requirements: !ruby/object:Gem::Requirement
253
- none: false
254
223
  requirements:
255
- - - ! '>='
224
+ - - ">="
256
225
  - !ruby/object:Gem::Version
257
226
  version: '0'
258
227
  - !ruby/object:Gem::Dependency
259
228
  name: rake
260
229
  requirement: !ruby/object:Gem::Requirement
261
- none: false
262
230
  requirements:
263
- - - ! '>='
231
+ - - ">="
264
232
  - !ruby/object:Gem::Version
265
233
  version: '0'
266
234
  type: :development
267
235
  prerelease: false
268
236
  version_requirements: !ruby/object:Gem::Requirement
269
- none: false
270
237
  requirements:
271
- - - ! '>='
238
+ - - ">="
272
239
  - !ruby/object:Gem::Version
273
240
  version: '0'
274
241
  - !ruby/object:Gem::Dependency
275
242
  name: yard
276
243
  requirement: !ruby/object:Gem::Requirement
277
- none: false
278
244
  requirements:
279
- - - ! '>='
245
+ - - ">="
280
246
  - !ruby/object:Gem::Version
281
247
  version: '0'
282
248
  type: :development
283
249
  prerelease: false
284
250
  version_requirements: !ruby/object:Gem::Requirement
285
- none: false
286
251
  requirements:
287
- - - ! '>='
252
+ - - ">="
288
253
  - !ruby/object:Gem::Version
289
254
  version: '0'
290
255
  - !ruby/object:Gem::Dependency
291
256
  name: rdiscount
292
257
  requirement: !ruby/object:Gem::Requirement
293
- none: false
294
258
  requirements:
295
- - - ! '>='
259
+ - - ">="
296
260
  - !ruby/object:Gem::Version
297
261
  version: '0'
298
262
  type: :development
299
263
  prerelease: false
300
264
  version_requirements: !ruby/object:Gem::Requirement
301
- none: false
302
265
  requirements:
303
- - - ! '>='
266
+ - - ">="
304
267
  - !ruby/object:Gem::Version
305
268
  version: '0'
306
269
  - !ruby/object:Gem::Dependency
307
270
  name: sqlite3
308
271
  requirement: !ruby/object:Gem::Requirement
309
- none: false
310
272
  requirements:
311
- - - ! '>='
273
+ - - ">="
312
274
  - !ruby/object:Gem::Version
313
275
  version: '0'
314
276
  type: :development
315
277
  prerelease: false
316
278
  version_requirements: !ruby/object:Gem::Requirement
317
- none: false
318
279
  requirements:
319
- - - ! '>='
280
+ - - ">="
320
281
  - !ruby/object:Gem::Version
321
282
  version: '0'
322
283
  - !ruby/object:Gem::Dependency
323
284
  name: mysql2
324
285
  requirement: !ruby/object:Gem::Requirement
325
- none: false
326
286
  requirements:
327
- - - ! '>='
287
+ - - ">="
328
288
  - !ruby/object:Gem::Version
329
289
  version: '0'
330
290
  type: :development
331
291
  prerelease: false
332
292
  version_requirements: !ruby/object:Gem::Requirement
333
- none: false
334
293
  requirements:
335
- - - ! '>='
294
+ - - ">="
336
295
  - !ruby/object:Gem::Version
337
296
  version: '0'
338
297
  - !ruby/object:Gem::Dependency
339
298
  name: pg
340
299
  requirement: !ruby/object:Gem::Requirement
341
- none: false
342
300
  requirements:
343
- - - ! '>='
301
+ - - ">="
344
302
  - !ruby/object:Gem::Version
345
303
  version: '0'
346
304
  type: :development
347
305
  prerelease: false
348
306
  version_requirements: !ruby/object:Gem::Requirement
349
- none: false
350
307
  requirements:
351
- - - ! '>='
308
+ - - ">="
352
309
  - !ruby/object:Gem::Version
353
310
  version: '0'
354
311
  description: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
@@ -364,9 +321,9 @@ executables: []
364
321
  extensions: []
365
322
  extra_rdoc_files: []
366
323
  files:
367
- - .gitignore
368
- - .rspec
369
- - .yardopts
324
+ - ".gitignore"
325
+ - ".rspec"
326
+ - ".yardopts"
370
327
  - CHANGELOG
371
328
  - Gemfile
372
329
  - LICENSE
@@ -402,28 +359,28 @@ files:
402
359
  - test/support/pets_funny.csv
403
360
  - test/test_data_miner.rb
404
361
  homepage: https://github.com/seamusabshere/data_miner
405
- licenses: []
362
+ licenses:
363
+ - MIT
364
+ metadata: {}
406
365
  post_install_message:
407
366
  rdoc_options: []
408
367
  require_paths:
409
368
  - lib
410
369
  required_ruby_version: !ruby/object:Gem::Requirement
411
- none: false
412
370
  requirements:
413
- - - ! '>='
371
+ - - ">="
414
372
  - !ruby/object:Gem::Version
415
373
  version: '0'
416
374
  required_rubygems_version: !ruby/object:Gem::Requirement
417
- none: false
418
375
  requirements:
419
- - - ! '>'
376
+ - - ">="
420
377
  - !ruby/object:Gem::Version
421
- version: 1.3.1
378
+ version: '0'
422
379
  requirements: []
423
380
  rubyforge_project: data_miner
424
- rubygems_version: 1.8.25
381
+ rubygems_version: 2.2.1
425
382
  signing_key:
426
- specification_version: 3
383
+ specification_version: 4
427
384
  summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
428
385
  XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
429
386
  test_files:
@@ -445,4 +402,3 @@ test_files:
445
402
  - test/support/pets.csv
446
403
  - test/support/pets_funny.csv
447
404
  - test/test_data_miner.rb
448
- has_rdoc: