data_miner 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/data_miner.gemspec +1 -0
- data/lib/data_miner/active_record_extensions.rb +24 -14
- data/lib/data_miner/config.rb +3 -21
- data/lib/data_miner/process.rb +7 -6
- data/lib/data_miner/version.rb +1 -1
- data/test/test_data_miner_verify.rb +3 -3
- metadata +127 -124
data/data_miner.gemspec
CHANGED
@@ -32,6 +32,7 @@ Gem::Specification.new do |s|
|
|
32
32
|
s.add_development_dependency 'test-unit'
|
33
33
|
s.add_development_dependency 'shoulda'
|
34
34
|
s.add_development_dependency 'mysql'
|
35
|
+
s.add_development_dependency 'rake'
|
35
36
|
# if RUBY_VERSION >= '1.9'
|
36
37
|
# s.add_development_dependency 'ruby-debug19'
|
37
38
|
# else
|
@@ -3,25 +3,35 @@ require 'blockenspiel'
|
|
3
3
|
|
4
4
|
class DataMiner
|
5
5
|
module ActiveRecordExtensions
|
6
|
+
def data_miner_config
|
7
|
+
@data_miner_config ||= ::DataMiner::Config.new self
|
8
|
+
end
|
9
|
+
|
10
|
+
def data_miner_config=(config)
|
11
|
+
@data_miner_config = config
|
12
|
+
end
|
13
|
+
|
14
|
+
def data_miner_runs
|
15
|
+
::DataMiner::Run.scoped :conditions => { :resource_name => name }
|
16
|
+
end
|
17
|
+
|
18
|
+
def run_data_miner!(options = {})
|
19
|
+
data_miner_config.run options
|
20
|
+
end
|
21
|
+
|
22
|
+
def run_data_miner_on_parent_associations!
|
23
|
+
reflect_on_all_associations(:belongs_to).each do |assoc|
|
24
|
+
next if assoc.options[:polymorphic]
|
25
|
+
assoc.klass.run_data_miner!
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
6
29
|
def data_miner(options = {}, &blk)
|
7
30
|
::DataMiner.instance.start_logging
|
8
31
|
|
9
32
|
::DataMiner.instance.resource_names.push self.name unless ::DataMiner.instance.resource_names.include? self.name
|
10
33
|
|
11
|
-
|
12
|
-
class_eval do
|
13
|
-
cattr_accessor :data_miner_config
|
14
|
-
def self.data_miner_runs
|
15
|
-
::DataMiner::Run.scoped :conditions => { :resource_name => name }
|
16
|
-
end
|
17
|
-
def self.run_data_miner!(options = {})
|
18
|
-
data_miner_config.run options
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
if options[:append]
|
23
|
-
self.data_miner_config ||= ::DataMiner::Config.new self
|
24
|
-
else
|
34
|
+
unless options[:append]
|
25
35
|
self.data_miner_config = ::DataMiner::Config.new self
|
26
36
|
end
|
27
37
|
|
data/lib/data_miner/config.rb
CHANGED
@@ -19,8 +19,8 @@ class DataMiner
|
|
19
19
|
# @attributes ||= {}
|
20
20
|
# end
|
21
21
|
|
22
|
-
def process(
|
23
|
-
step = Process.new self,
|
22
|
+
def process(method_id_or_description, &blk)
|
23
|
+
step = Process.new self, method_id_or_description, &blk
|
24
24
|
steps.push step
|
25
25
|
end
|
26
26
|
|
@@ -152,25 +152,7 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
|
|
152
152
|
|
153
153
|
On #{resource}, it looks like you're missing some columns...
|
154
154
|
|
155
|
-
|
156
|
-
|
157
|
-
./script/generate migration AddMissingColumnsTo#{resource.name}
|
158
|
-
|
159
|
-
and **replace** the resulting file with this:
|
160
|
-
|
161
|
-
class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
|
162
|
-
def self.up
|
163
|
-
#{missing_columns.map { |column_name| " add_column :#{resource.table_name}, :#{column_name}, :#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
|
164
|
-
end
|
165
|
-
|
166
|
-
def self.down
|
167
|
-
#{missing_columns.map { |column_name| " remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
On the other hand, if you're working directly with create_table, this might be helpful:
|
172
|
-
|
173
|
-
#{missing_columns.map { |column_name| "t.#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
|
155
|
+
#{missing_columns.map { |column_name| "#{column_name.end_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
|
174
156
|
|
175
157
|
================================
|
176
158
|
}
|
data/lib/data_miner/process.rb
CHANGED
@@ -2,18 +2,19 @@ class DataMiner
|
|
2
2
|
class Process
|
3
3
|
attr_reader :config
|
4
4
|
attr_reader :method_id
|
5
|
-
attr_reader :
|
5
|
+
attr_reader :description
|
6
6
|
attr_reader :blk
|
7
7
|
|
8
|
-
alias :
|
8
|
+
alias :block_description :description
|
9
9
|
|
10
|
-
def initialize(config,
|
10
|
+
def initialize(config, method_id_or_description, &blk)
|
11
11
|
@config = config
|
12
12
|
if block_given?
|
13
|
-
@
|
13
|
+
@description = method_id_or_description
|
14
14
|
@blk = blk
|
15
15
|
else
|
16
|
-
@
|
16
|
+
@description = method_id_or_description
|
17
|
+
@method_id = method_id_or_description
|
17
18
|
end
|
18
19
|
end
|
19
20
|
|
@@ -22,7 +23,7 @@ class DataMiner
|
|
22
23
|
end
|
23
24
|
|
24
25
|
def inspect
|
25
|
-
%{#<DataMiner::Process(#{resource}) #{
|
26
|
+
%{#<DataMiner::Process(#{resource}) #{description}>}
|
26
27
|
end
|
27
28
|
|
28
29
|
def run
|
data/lib/data_miner/version.rb
CHANGED
@@ -4,17 +4,17 @@ class TestDataMinerVerify < Test::Unit::TestCase
|
|
4
4
|
context '#run' do
|
5
5
|
should 'raise an exception if the verification block fails through exception' do
|
6
6
|
raising_check = lambda { raise "boom" }
|
7
|
-
verify = DataMiner::Verify.new Aircraft.
|
7
|
+
verify = DataMiner::Verify.new Aircraft.data_miner_config, 'verification of engine type', &raising_check
|
8
8
|
assert_raise(DataMiner::VerificationFailed) { verify.run }
|
9
9
|
end
|
10
10
|
should 'raise an exception if the result of the verification block is false' do
|
11
11
|
failing_check = lambda { false }
|
12
|
-
verify = DataMiner::Verify.new Aircraft.
|
12
|
+
verify = DataMiner::Verify.new Aircraft.data_miner_config, 'verification of engine type', &failing_check
|
13
13
|
assert_raise(DataMiner::VerificationFailed) { verify.run }
|
14
14
|
end
|
15
15
|
should 'return true if the verification block succeeds' do
|
16
16
|
passing_check = lambda { true }
|
17
|
-
verify = DataMiner::Verify.new Aircraft.
|
17
|
+
verify = DataMiner::Verify.new Aircraft.data_miner_config, 'verification of engine type', &passing_check
|
18
18
|
assert_nil verify.run
|
19
19
|
end
|
20
20
|
end
|
metadata
CHANGED
@@ -1,173 +1,180 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_miner
|
3
|
-
version: !ruby/object:Gem::Version
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.2.1
|
4
5
|
prerelease:
|
5
|
-
version: 1.2.0
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
7
|
+
authors:
|
8
8
|
- Seamus Abshere
|
9
9
|
- Andy Rossmeissl
|
10
10
|
- Derek Kastner
|
11
11
|
autorequire:
|
12
12
|
bindir: bin
|
13
13
|
cert_chain: []
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
dependencies:
|
18
|
-
- !ruby/object:Gem::Dependency
|
14
|
+
date: 2011-06-16 00:00:00.000000000Z
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
19
17
|
name: remote_table
|
20
|
-
|
21
|
-
requirement: &id001 !ruby/object:Gem::Requirement
|
18
|
+
requirement: &2156849640 !ruby/object:Gem::Requirement
|
22
19
|
none: false
|
23
|
-
requirements:
|
24
|
-
- -
|
25
|
-
- !ruby/object:Gem::Version
|
20
|
+
requirements:
|
21
|
+
- - ! '>='
|
22
|
+
- !ruby/object:Gem::Version
|
26
23
|
version: 1.2.2
|
27
24
|
type: :runtime
|
28
|
-
version_requirements: *id001
|
29
|
-
- !ruby/object:Gem::Dependency
|
30
|
-
name: escape
|
31
25
|
prerelease: false
|
32
|
-
|
26
|
+
version_requirements: *2156849640
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: escape
|
29
|
+
requirement: &2156849020 !ruby/object:Gem::Requirement
|
33
30
|
none: false
|
34
|
-
requirements:
|
35
|
-
- -
|
36
|
-
- !ruby/object:Gem::Version
|
31
|
+
requirements:
|
32
|
+
- - ! '>='
|
33
|
+
- !ruby/object:Gem::Version
|
37
34
|
version: 0.0.4
|
38
35
|
type: :runtime
|
39
|
-
version_requirements: *id002
|
40
|
-
- !ruby/object:Gem::Dependency
|
41
|
-
name: activerecord
|
42
36
|
prerelease: false
|
43
|
-
|
37
|
+
version_requirements: *2156849020
|
38
|
+
- !ruby/object:Gem::Dependency
|
39
|
+
name: activerecord
|
40
|
+
requirement: &2156848340 !ruby/object:Gem::Requirement
|
44
41
|
none: false
|
45
|
-
requirements:
|
46
|
-
- -
|
47
|
-
- !ruby/object:Gem::Version
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
48
45
|
version: 2.3.4
|
49
46
|
type: :runtime
|
50
|
-
version_requirements: *id003
|
51
|
-
- !ruby/object:Gem::Dependency
|
52
|
-
name: activesupport
|
53
47
|
prerelease: false
|
54
|
-
|
48
|
+
version_requirements: *2156848340
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: activesupport
|
51
|
+
requirement: &2156847760 !ruby/object:Gem::Requirement
|
55
52
|
none: false
|
56
|
-
requirements:
|
57
|
-
- -
|
58
|
-
- !ruby/object:Gem::Version
|
53
|
+
requirements:
|
54
|
+
- - ! '>='
|
55
|
+
- !ruby/object:Gem::Version
|
59
56
|
version: 2.3.4
|
60
57
|
type: :runtime
|
61
|
-
version_requirements: *id004
|
62
|
-
- !ruby/object:Gem::Dependency
|
63
|
-
name: conversions
|
64
58
|
prerelease: false
|
65
|
-
|
59
|
+
version_requirements: *2156847760
|
60
|
+
- !ruby/object:Gem::Dependency
|
61
|
+
name: conversions
|
62
|
+
requirement: &2156846840 !ruby/object:Gem::Requirement
|
66
63
|
none: false
|
67
|
-
requirements:
|
68
|
-
- -
|
69
|
-
- !ruby/object:Gem::Version
|
64
|
+
requirements:
|
65
|
+
- - ! '>='
|
66
|
+
- !ruby/object:Gem::Version
|
70
67
|
version: 1.4.4
|
71
68
|
type: :runtime
|
72
|
-
version_requirements: *id005
|
73
|
-
- !ruby/object:Gem::Dependency
|
74
|
-
name: blockenspiel
|
75
69
|
prerelease: false
|
76
|
-
|
70
|
+
version_requirements: *2156846840
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: blockenspiel
|
73
|
+
requirement: &2156846280 !ruby/object:Gem::Requirement
|
77
74
|
none: false
|
78
|
-
requirements:
|
79
|
-
- -
|
80
|
-
- !ruby/object:Gem::Version
|
75
|
+
requirements:
|
76
|
+
- - ! '>='
|
77
|
+
- !ruby/object:Gem::Version
|
81
78
|
version: 0.3.2
|
82
79
|
type: :runtime
|
83
|
-
version_requirements: *id006
|
84
|
-
- !ruby/object:Gem::Dependency
|
85
|
-
name: taps
|
86
80
|
prerelease: false
|
87
|
-
|
81
|
+
version_requirements: *2156846280
|
82
|
+
- !ruby/object:Gem::Dependency
|
83
|
+
name: taps
|
84
|
+
requirement: &2156845760 !ruby/object:Gem::Requirement
|
88
85
|
none: false
|
89
|
-
requirements:
|
90
|
-
- -
|
91
|
-
- !ruby/object:Gem::Version
|
86
|
+
requirements:
|
87
|
+
- - ! '>='
|
88
|
+
- !ruby/object:Gem::Version
|
92
89
|
version: 0.3.11
|
93
90
|
type: :runtime
|
94
|
-
version_requirements: *id007
|
95
|
-
- !ruby/object:Gem::Dependency
|
96
|
-
name: errata
|
97
91
|
prerelease: false
|
98
|
-
|
92
|
+
version_requirements: *2156845760
|
93
|
+
- !ruby/object:Gem::Dependency
|
94
|
+
name: errata
|
95
|
+
requirement: &2156844720 !ruby/object:Gem::Requirement
|
99
96
|
none: false
|
100
|
-
requirements:
|
101
|
-
- -
|
102
|
-
- !ruby/object:Gem::Version
|
97
|
+
requirements:
|
98
|
+
- - ! '>='
|
99
|
+
- !ruby/object:Gem::Version
|
103
100
|
version: 1.0.1
|
104
101
|
type: :runtime
|
105
|
-
version_requirements: *id008
|
106
|
-
- !ruby/object:Gem::Dependency
|
107
|
-
name: create_table
|
108
102
|
prerelease: false
|
109
|
-
|
103
|
+
version_requirements: *2156844720
|
104
|
+
- !ruby/object:Gem::Dependency
|
105
|
+
name: create_table
|
106
|
+
requirement: &2156843920 !ruby/object:Gem::Requirement
|
110
107
|
none: false
|
111
|
-
requirements:
|
112
|
-
- -
|
113
|
-
- !ruby/object:Gem::Version
|
108
|
+
requirements:
|
109
|
+
- - ! '>='
|
110
|
+
- !ruby/object:Gem::Version
|
114
111
|
version: 0.0.2
|
115
112
|
type: :development
|
116
|
-
version_requirements: *id009
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
|
-
name: loose_tight_dictionary
|
119
113
|
prerelease: false
|
120
|
-
|
114
|
+
version_requirements: *2156843920
|
115
|
+
- !ruby/object:Gem::Dependency
|
116
|
+
name: loose_tight_dictionary
|
117
|
+
requirement: &2156842900 !ruby/object:Gem::Requirement
|
121
118
|
none: false
|
122
|
-
requirements:
|
123
|
-
- -
|
124
|
-
- !ruby/object:Gem::Version
|
119
|
+
requirements:
|
120
|
+
- - ! '>='
|
121
|
+
- !ruby/object:Gem::Version
|
125
122
|
version: 0.0.5
|
126
123
|
type: :development
|
127
|
-
version_requirements: *id010
|
128
|
-
- !ruby/object:Gem::Dependency
|
129
|
-
name: test-unit
|
130
124
|
prerelease: false
|
131
|
-
|
125
|
+
version_requirements: *2156842900
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: test-unit
|
128
|
+
requirement: &2156842520 !ruby/object:Gem::Requirement
|
132
129
|
none: false
|
133
|
-
requirements:
|
134
|
-
- -
|
135
|
-
- !ruby/object:Gem::Version
|
136
|
-
version:
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
137
134
|
type: :development
|
138
|
-
version_requirements: *id011
|
139
|
-
- !ruby/object:Gem::Dependency
|
140
|
-
name: shoulda
|
141
135
|
prerelease: false
|
142
|
-
|
136
|
+
version_requirements: *2156842520
|
137
|
+
- !ruby/object:Gem::Dependency
|
138
|
+
name: shoulda
|
139
|
+
requirement: &2156841880 !ruby/object:Gem::Requirement
|
143
140
|
none: false
|
144
|
-
requirements:
|
145
|
-
- -
|
146
|
-
- !ruby/object:Gem::Version
|
147
|
-
version:
|
141
|
+
requirements:
|
142
|
+
- - ! '>='
|
143
|
+
- !ruby/object:Gem::Version
|
144
|
+
version: '0'
|
148
145
|
type: :development
|
149
|
-
|
150
|
-
|
146
|
+
prerelease: false
|
147
|
+
version_requirements: *2156841880
|
148
|
+
- !ruby/object:Gem::Dependency
|
151
149
|
name: mysql
|
150
|
+
requirement: &2156841300 !ruby/object:Gem::Requirement
|
151
|
+
none: false
|
152
|
+
requirements:
|
153
|
+
- - ! '>='
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
type: :development
|
152
157
|
prerelease: false
|
153
|
-
|
158
|
+
version_requirements: *2156841300
|
159
|
+
- !ruby/object:Gem::Dependency
|
160
|
+
name: rake
|
161
|
+
requirement: &2156840680 !ruby/object:Gem::Requirement
|
154
162
|
none: false
|
155
|
-
requirements:
|
156
|
-
- -
|
157
|
-
- !ruby/object:Gem::Version
|
158
|
-
version:
|
163
|
+
requirements:
|
164
|
+
- - ! '>='
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
159
167
|
type: :development
|
160
|
-
|
161
|
-
|
162
|
-
|
168
|
+
prerelease: false
|
169
|
+
version_requirements: *2156840680
|
170
|
+
description: Mine remote data into your ActiveRecord models. You can also convert
|
171
|
+
units.
|
172
|
+
email:
|
163
173
|
- seamus@abshere.net
|
164
174
|
executables: []
|
165
|
-
|
166
175
|
extensions: []
|
167
|
-
|
168
176
|
extra_rdoc_files: []
|
169
|
-
|
170
|
-
files:
|
177
|
+
files:
|
171
178
|
- .document
|
172
179
|
- .gitignore
|
173
180
|
- CHANGELOG
|
@@ -198,35 +205,31 @@ files:
|
|
198
205
|
- test/test_data_miner_process.rb
|
199
206
|
- test/test_data_miner_verify.rb
|
200
207
|
- test/test_old_syntax.rb
|
201
|
-
has_rdoc: true
|
202
208
|
homepage: https://github.com/seamusabshere/data_miner
|
203
209
|
licenses: []
|
204
|
-
|
205
210
|
post_install_message:
|
206
211
|
rdoc_options: []
|
207
|
-
|
208
|
-
require_paths:
|
212
|
+
require_paths:
|
209
213
|
- lib
|
210
|
-
required_ruby_version: !ruby/object:Gem::Requirement
|
214
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
211
215
|
none: false
|
212
|
-
requirements:
|
213
|
-
- -
|
214
|
-
- !ruby/object:Gem::Version
|
215
|
-
version:
|
216
|
-
required_rubygems_version: !ruby/object:Gem::Requirement
|
216
|
+
requirements:
|
217
|
+
- - ! '>='
|
218
|
+
- !ruby/object:Gem::Version
|
219
|
+
version: '0'
|
220
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
217
221
|
none: false
|
218
|
-
requirements:
|
219
|
-
- -
|
220
|
-
- !ruby/object:Gem::Version
|
221
|
-
version:
|
222
|
+
requirements:
|
223
|
+
- - ! '>='
|
224
|
+
- !ruby/object:Gem::Version
|
225
|
+
version: '0'
|
222
226
|
requirements: []
|
223
|
-
|
224
227
|
rubyforge_project: data_miner
|
225
|
-
rubygems_version: 1.
|
228
|
+
rubygems_version: 1.8.5
|
226
229
|
signing_key:
|
227
230
|
specification_version: 3
|
228
231
|
summary: Mine remote data into your ActiveRecord models.
|
229
|
-
test_files:
|
232
|
+
test_files:
|
230
233
|
- test/helper.rb
|
231
234
|
- test/support/aircraft.rb
|
232
235
|
- test/support/airport.rb
|