data_miner 0.5.7 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- require 'test_helper'
1
+ require 'helper'
2
2
 
3
- class DataMiner::VerifyTest < Test::Unit::TestCase
3
+ class TestVerify < Test::Unit::TestCase
4
4
  context '#run' do
5
5
  setup do
6
6
  @run = Object.new
@@ -14,13 +14,13 @@ class DataMiner::VerifyTest < Test::Unit::TestCase
14
14
  @verify.check = lambda do
15
15
  assert false
16
16
  end
17
- assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
17
+ assert_raise(DataMiner::VerificationFailed) { @verify.run @run }
18
18
  end
19
19
  should 'raise an exception if the result of the verification block is false' do
20
20
  @verify.check = lambda do
21
21
  false
22
22
  end
23
- assert_raise(DataMiner::Verify::VerificationFailed) { @verify.run @run }
23
+ assert_raise(DataMiner::VerificationFailed) { @verify.run @run }
24
24
  end
25
25
  should 'return true if the verification block succeeds' do
26
26
  assert @verify.run(@run)
metadata CHANGED
@@ -1,12 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_miner
3
3
  version: !ruby/object:Gem::Version
4
+ hash: 23
4
5
  prerelease: false
5
6
  segments:
7
+ - 1
6
8
  - 0
7
- - 5
8
- - 7
9
- version: 0.5.7
9
+ - 0
10
+ version: 1.0.0
10
11
  platform: ruby
11
12
  authors:
12
13
  - Seamus Abshere
@@ -16,7 +17,7 @@ autorequire:
16
17
  bindir: bin
17
18
  cert_chain: []
18
19
 
19
- date: 2010-11-16 00:00:00 -05:00
20
+ date: 2011-01-25 00:00:00 -06:00
20
21
  default_executable:
21
22
  dependencies:
22
23
  - !ruby/object:Gem::Dependency
@@ -25,13 +26,14 @@ dependencies:
25
26
  requirement: &id001 !ruby/object:Gem::Requirement
26
27
  none: false
27
28
  requirements:
28
- - - ~>
29
+ - - ">="
29
30
  - !ruby/object:Gem::Version
31
+ hash: 19
30
32
  segments:
33
+ - 1
31
34
  - 0
32
35
  - 2
33
- - 32
34
- version: 0.2.32
36
+ version: 1.0.2
35
37
  type: :runtime
36
38
  version_requirements: *id001
37
39
  - !ruby/object:Gem::Dependency
@@ -42,6 +44,7 @@ dependencies:
42
44
  requirements:
43
45
  - - ">="
44
46
  - !ruby/object:Gem::Version
47
+ hash: 23
45
48
  segments:
46
49
  - 0
47
50
  - 0
@@ -57,6 +60,7 @@ dependencies:
57
60
  requirements:
58
61
  - - ">="
59
62
  - !ruby/object:Gem::Version
63
+ hash: 11
60
64
  segments:
61
65
  - 2
62
66
  - 3
@@ -72,6 +76,7 @@ dependencies:
72
76
  requirements:
73
77
  - - ">="
74
78
  - !ruby/object:Gem::Version
79
+ hash: 11
75
80
  segments:
76
81
  - 2
77
82
  - 3
@@ -79,191 +84,164 @@ dependencies:
79
84
  version: 2.3.4
80
85
  type: :runtime
81
86
  version_requirements: *id004
82
- - !ruby/object:Gem::Dependency
83
- name: andand
84
- prerelease: false
85
- requirement: &id005 !ruby/object:Gem::Requirement
86
- none: false
87
- requirements:
88
- - - ">="
89
- - !ruby/object:Gem::Version
90
- segments:
91
- - 1
92
- - 3
93
- - 1
94
- version: 1.3.1
95
- type: :runtime
96
- version_requirements: *id005
97
87
  - !ruby/object:Gem::Dependency
98
88
  name: conversions
99
89
  prerelease: false
100
- requirement: &id006 !ruby/object:Gem::Requirement
90
+ requirement: &id005 !ruby/object:Gem::Requirement
101
91
  none: false
102
92
  requirements:
103
93
  - - ">="
104
94
  - !ruby/object:Gem::Version
95
+ hash: 15
105
96
  segments:
106
97
  - 1
107
98
  - 4
108
99
  - 4
109
100
  version: 1.4.4
110
101
  type: :runtime
111
- version_requirements: *id006
102
+ version_requirements: *id005
112
103
  - !ruby/object:Gem::Dependency
113
104
  name: blockenspiel
114
105
  prerelease: false
115
- requirement: &id007 !ruby/object:Gem::Requirement
106
+ requirement: &id006 !ruby/object:Gem::Requirement
116
107
  none: false
117
108
  requirements:
118
- - - "="
109
+ - - ">="
119
110
  - !ruby/object:Gem::Version
111
+ hash: 23
120
112
  segments:
121
113
  - 0
122
114
  - 3
123
115
  - 2
124
116
  version: 0.3.2
125
117
  type: :runtime
126
- version_requirements: *id007
127
- - !ruby/object:Gem::Dependency
128
- name: log4r
129
- prerelease: false
130
- requirement: &id008 !ruby/object:Gem::Requirement
131
- none: false
132
- requirements:
133
- - - ">="
134
- - !ruby/object:Gem::Version
135
- segments:
136
- - 1
137
- - 1
138
- - 7
139
- version: 1.1.7
140
- type: :runtime
141
- version_requirements: *id008
142
- - !ruby/object:Gem::Dependency
143
- name: errata
144
- prerelease: false
145
- requirement: &id009 !ruby/object:Gem::Requirement
146
- none: false
147
- requirements:
148
- - - ">="
149
- - !ruby/object:Gem::Version
150
- segments:
151
- - 0
152
- - 2
153
- - 1
154
- version: 0.2.1
155
- type: :runtime
156
- version_requirements: *id009
118
+ version_requirements: *id006
157
119
  - !ruby/object:Gem::Dependency
158
120
  name: taps
159
121
  prerelease: false
160
- requirement: &id010 !ruby/object:Gem::Requirement
122
+ requirement: &id007 !ruby/object:Gem::Requirement
161
123
  none: false
162
124
  requirements:
163
125
  - - ">="
164
126
  - !ruby/object:Gem::Version
127
+ hash: 5
165
128
  segments:
166
129
  - 0
167
130
  - 3
168
131
  - 11
169
132
  version: 0.3.11
170
133
  type: :runtime
171
- version_requirements: *id010
134
+ version_requirements: *id007
172
135
  - !ruby/object:Gem::Dependency
173
136
  name: loose_tight_dictionary
174
137
  prerelease: false
175
- requirement: &id011 !ruby/object:Gem::Requirement
138
+ requirement: &id008 !ruby/object:Gem::Requirement
176
139
  none: false
177
140
  requirements:
178
141
  - - ">="
179
142
  - !ruby/object:Gem::Version
143
+ hash: 21
180
144
  segments:
181
145
  - 0
182
146
  - 0
183
147
  - 5
184
148
  version: 0.0.5
185
149
  type: :development
186
- version_requirements: *id011
150
+ version_requirements: *id008
187
151
  - !ruby/object:Gem::Dependency
188
- name: jeweler
152
+ name: errata
189
153
  prerelease: false
190
- requirement: &id012 !ruby/object:Gem::Requirement
154
+ requirement: &id009 !ruby/object:Gem::Requirement
191
155
  none: false
192
156
  requirements:
193
157
  - - ">="
194
158
  - !ruby/object:Gem::Version
159
+ hash: 21
195
160
  segments:
161
+ - 1
196
162
  - 0
197
- version: "0"
163
+ - 1
164
+ version: 1.0.1
198
165
  type: :development
199
- version_requirements: *id012
166
+ version_requirements: *id009
200
167
  - !ruby/object:Gem::Dependency
201
- name: rcov
168
+ name: test-unit
202
169
  prerelease: false
203
- requirement: &id013 !ruby/object:Gem::Requirement
170
+ requirement: &id010 !ruby/object:Gem::Requirement
204
171
  none: false
205
172
  requirements:
206
173
  - - ">="
207
174
  - !ruby/object:Gem::Version
175
+ hash: 3
208
176
  segments:
209
177
  - 0
210
178
  version: "0"
211
179
  type: :development
212
- version_requirements: *id013
180
+ version_requirements: *id010
213
181
  - !ruby/object:Gem::Dependency
214
- name: rake
182
+ name: shoulda
215
183
  prerelease: false
216
- requirement: &id014 !ruby/object:Gem::Requirement
184
+ requirement: &id011 !ruby/object:Gem::Requirement
217
185
  none: false
218
186
  requirements:
219
187
  - - ">="
220
188
  - !ruby/object:Gem::Version
189
+ hash: 3
221
190
  segments:
222
191
  - 0
223
192
  version: "0"
224
193
  type: :development
225
- version_requirements: *id014
194
+ version_requirements: *id011
226
195
  - !ruby/object:Gem::Dependency
227
- name: shoulda
196
+ name: mysql
228
197
  prerelease: false
229
- requirement: &id015 !ruby/object:Gem::Requirement
198
+ requirement: &id012 !ruby/object:Gem::Requirement
230
199
  none: false
231
200
  requirements:
232
201
  - - ">="
233
202
  - !ruby/object:Gem::Version
203
+ hash: 3
234
204
  segments:
235
205
  - 0
236
206
  version: "0"
237
207
  type: :development
238
- version_requirements: *id015
208
+ version_requirements: *id012
239
209
  - !ruby/object:Gem::Dependency
240
- name: mysql
210
+ name: ruby-debug
241
211
  prerelease: false
242
- requirement: &id016 !ruby/object:Gem::Requirement
212
+ requirement: &id013 !ruby/object:Gem::Requirement
243
213
  none: false
244
214
  requirements:
245
215
  - - ">="
246
216
  - !ruby/object:Gem::Version
217
+ hash: 3
247
218
  segments:
248
219
  - 0
249
220
  version: "0"
250
221
  type: :development
251
- version_requirements: *id016
252
- description: Mine remote data into your ActiveRecord models. You can also perform associations and convert units.
253
- email: seamus@abshere.net
222
+ version_requirements: *id013
223
+ description: Mine remote data into your ActiveRecord models. You can also convert units.
224
+ email:
225
+ - seamus@abshere.net
254
226
  executables: []
255
227
 
256
228
  extensions: []
257
229
 
258
- extra_rdoc_files:
259
- - LICENSE
260
- - README.rdoc
230
+ extra_rdoc_files: []
231
+
261
232
  files:
233
+ - .document
234
+ - .gitignore
235
+ - CHANGELOG
236
+ - Gemfile
262
237
  - LICENSE
263
238
  - README.rdoc
239
+ - Rakefile
240
+ - data_miner.gemspec
264
241
  - lib/data_miner.rb
242
+ - lib/data_miner/active_record_extensions.rb
265
243
  - lib/data_miner/attribute.rb
266
- - lib/data_miner/base.rb
244
+ - lib/data_miner/config.rb
267
245
  - lib/data_miner/dictionary.rb
268
246
  - lib/data_miner/import.rb
269
247
  - lib/data_miner/process.rb
@@ -271,24 +249,23 @@ files:
271
249
  - lib/data_miner/schema.rb
272
250
  - lib/data_miner/tap.rb
273
251
  - lib/data_miner/verify.rb
274
- - test/data_miner/attribute_test.rb
275
- - test/data_miner/verify_test.rb
252
+ - lib/data_miner/version.rb
253
+ - test/helper.rb
254
+ - test/support/aircraft.rb
276
255
  - test/support/airport.rb
277
- - test/support/country.rb
278
256
  - test/support/automobile_fuel_type.rb
279
- - test/support/aircraft.rb
280
257
  - test/support/automobile_variant.rb
281
- - test/data_miner_test.rb
282
- - test/test_helper.rb
258
+ - test/support/country.rb
259
+ - test/test_attribute.rb
260
+ - test/test_old_syntax.rb
261
+ - test/test_verify.rb
283
262
  has_rdoc: true
284
- homepage: http://github.com/seamusabshere/data_miner
263
+ homepage: https://github.com/seamusabshere/data_miner
285
264
  licenses: []
286
265
 
287
266
  post_install_message:
288
- rdoc_options:
289
- - --charset=UTF-8
290
- - --line-numbers
291
- - --inline-source
267
+ rdoc_options: []
268
+
292
269
  require_paths:
293
270
  - lib
294
271
  required_ruby_version: !ruby/object:Gem::Requirement
@@ -296,6 +273,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
296
273
  requirements:
297
274
  - - ">="
298
275
  - !ruby/object:Gem::Version
276
+ hash: 3
299
277
  segments:
300
278
  - 0
301
279
  version: "0"
@@ -304,23 +282,24 @@ required_rubygems_version: !ruby/object:Gem::Requirement
304
282
  requirements:
305
283
  - - ">="
306
284
  - !ruby/object:Gem::Version
285
+ hash: 3
307
286
  segments:
308
287
  - 0
309
288
  version: "0"
310
289
  requirements: []
311
290
 
312
- rubyforge_project:
291
+ rubyforge_project: data_miner
313
292
  rubygems_version: 1.3.7
314
293
  signing_key:
315
294
  specification_version: 3
316
295
  summary: Mine remote data into your ActiveRecord models.
317
296
  test_files:
318
- - test/data_miner/attribute_test.rb
319
- - test/data_miner/verify_test.rb
297
+ - test/helper.rb
298
+ - test/support/aircraft.rb
320
299
  - test/support/airport.rb
321
- - test/support/country.rb
322
300
  - test/support/automobile_fuel_type.rb
323
- - test/support/aircraft.rb
324
301
  - test/support/automobile_variant.rb
325
- - test/data_miner_test.rb
326
- - test/test_helper.rb
302
+ - test/support/country.rb
303
+ - test/test_attribute.rb
304
+ - test/test_old_syntax.rb
305
+ - test/test_verify.rb
@@ -1,204 +0,0 @@
1
- module DataMiner
2
- class Base
3
- include Blockenspiel::DSL
4
-
5
- attr_accessor :resource, :steps, :step_counter, :attributes
6
-
7
- def initialize(resource)
8
- @steps = Array.new
9
- @resource = resource
10
- @step_counter = 0
11
- @attributes = HashWithIndifferentAccess.new
12
- end
13
-
14
- def schema(create_table_options = {}, &block)
15
- step = DataMiner::Schema.new self, step_counter, create_table_options
16
- Blockenspiel.invoke block, step
17
- steps << step
18
- self.step_counter += 1
19
- end
20
-
21
- def process(method_name_or_block_description, &block)
22
- steps << DataMiner::Process.new(self, step_counter, method_name_or_block_description, &block)
23
- self.step_counter += 1
24
- end
25
-
26
- def tap(description, source, options = {})
27
- steps << DataMiner::Tap.new(self, step_counter, description, source, options)
28
- self.step_counter += 1
29
- end
30
-
31
- def import(*args, &block)
32
- if args.length == 1
33
- description = '(no description)'
34
- else
35
- description = args.first
36
- end
37
- options = args.last
38
-
39
- step = DataMiner::Import.new self, step_counter, description, options
40
- Blockenspiel.invoke block, step
41
- steps << step
42
- self.step_counter += 1
43
- end
44
-
45
- def verify(*args, &block)
46
- description = args.shift or '(no description)'
47
-
48
- step = DataMiner::Verify.new self, step_counter, description, block
49
- steps << step
50
- self.step_counter += 1
51
- end
52
-
53
- # Mine data for this class.
54
- def run(options = {})
55
- options.symbolize_keys!
56
-
57
- return if DataMiner::Base.call_stack.include? resource.name
58
- DataMiner::Base.call_stack.push resource.name
59
-
60
- finished = false
61
- skipped = false
62
- if DataMiner::Run.table_exists?
63
- run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name, :killed => true
64
- else
65
- run = nil
66
- DataMiner.log_info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
67
- end
68
- resource.delete_all if options[:from_scratch]
69
- begin
70
- steps.each do |step|
71
- step.run run
72
- resource.reset_column_information
73
- end
74
- finished = true
75
- rescue DataMiner::Finish
76
- finished = true
77
- rescue DataMiner::Skip
78
- skipped = true
79
- rescue DataMiner::Verify::VerificationFailed
80
- skipped = true
81
- ensure
82
- if DataMiner::Run.table_exists?
83
- run.update_attributes! :terminated_at => Time.now, :finished => finished, :skipped => skipped, :killed => false
84
- end
85
- DataMiner::Base.call_stack.clear if DataMiner::Base.call_stack.first == resource.name and !options[:preserve_call_stack_between_runs]
86
- end
87
- nil
88
- end
89
-
90
- def import_steps
91
- steps.select { |step| step.is_a? Import }
92
- end
93
-
94
- def before_invoke
95
-
96
- end
97
-
98
- def after_invoke
99
- return unless resource.table_exists?
100
- make_sure_unit_definitions_make_sense
101
- suggest_missing_column_migrations
102
- end
103
-
104
- COMPLETE_UNIT_DEFINITIONS = [
105
- [:units],
106
- [:from_units, :to_units],
107
- [:units_field_name],
108
- [:units_field_name, :to_units],
109
- [:units_field_number],
110
- [:units_field_number, :to_units]
111
- ]
112
-
113
- def make_sure_unit_definitions_make_sense
114
- import_steps.each do |step|
115
- step.attributes.each do |_, attribute|
116
- if attribute.options.any? { |k, _| k.to_s =~ /unit/ } and COMPLETE_UNIT_DEFINITIONS.none? { |complete_definition| complete_definition.all? { |required_option| attribute.options[required_option].present? } }
117
- DataMiner.log_or_raise %{
118
-
119
- ================================
120
-
121
- You don't have a valid unit definition for #{resource.name}##{attribute.name}.
122
-
123
- You supplied #{attribute.options.keys.select { |k, _| k.to_s =~ /unit/ }.map(&:to_sym).inspect }.
124
-
125
- You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence}".
126
-
127
- ================================
128
- }
129
- end
130
- end
131
- end
132
- end
133
-
134
- def suggest_missing_column_migrations
135
- missing_columns = Array.new
136
-
137
- import_steps.each do |step|
138
- step.attributes.each do |_, attribute|
139
- DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.end_with? '_units'
140
- unless resource.column_names.include? attribute.name
141
- missing_columns << attribute.name
142
- end
143
- if attribute.wants_units? and !resource.column_names.include?(units_column = "#{attribute.name}_units")
144
- missing_columns << units_column
145
- end
146
- end
147
- end
148
- missing_columns.uniq!
149
- if missing_columns.any?
150
- DataMiner.log_debug %{
151
-
152
- ================================
153
-
154
- On #{resource}, it looks like you're missing some columns...
155
-
156
- Please run this...
157
-
158
- ./script/generate migration AddMissingColumnsTo#{resource.name}
159
-
160
- and **replace** the resulting file with this:
161
-
162
- class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
163
- def self.up
164
- #{missing_columns.map { |column_name| " add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
165
- end
166
-
167
- def self.down
168
- #{missing_columns.map { |column_name| " remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
169
- end
170
- end
171
-
172
- On the other hand, if you're working directly with create_table, this might be helpful:
173
-
174
- #{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
175
-
176
- ================================
177
- }
178
- end
179
- end
180
-
181
- cattr_accessor :resource_names
182
- self.resource_names = Array.new
183
-
184
- cattr_accessor :call_stack
185
- self.call_stack = Array.new
186
- class << self
187
- # Mine data. Defaults to all resource_names touched by DataMiner.
188
- #
189
- # Options
190
- # * <tt>:resource_names</tt>: array of resource (class) names to mine
191
- def run(options = {})
192
- options.symbolize_keys!
193
-
194
- resource_names.each do |resource_name|
195
- if options[:resource_names].blank? or options[:resource_names].include?(resource_name)
196
- resource_name.constantize.data_miner_base.run options
197
- end
198
- end
199
- ensure
200
- RemoteTable.cleanup
201
- end
202
- end
203
- end
204
- end