data_miner 1.3.6 → 1.3.7
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +24 -0
- data/lib/data_miner.rb +13 -14
- data/lib/data_miner/active_record_extensions.rb +1 -3
- data/lib/data_miner/config.rb +1 -1
- data/lib/data_miner/import.rb +3 -3
- data/lib/data_miner/version.rb +1 -1
- metadata +27 -27
data/Rakefile
CHANGED
@@ -23,3 +23,27 @@ begin
|
|
23
23
|
rescue LoadError
|
24
24
|
puts "Rdoc is not available"
|
25
25
|
end
|
26
|
+
|
27
|
+
gemspec = eval(File.read(Dir["*.gemspec"].first))
|
28
|
+
|
29
|
+
desc "Validate the gemspec"
|
30
|
+
task :gemspec do
|
31
|
+
gemspec.validate
|
32
|
+
end
|
33
|
+
|
34
|
+
desc "Build gem locally"
|
35
|
+
task :build => :gemspec do
|
36
|
+
system "gem build #{gemspec.name}.gemspec"
|
37
|
+
FileUtils.mkdir_p "pkg"
|
38
|
+
FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", "pkg"
|
39
|
+
end
|
40
|
+
|
41
|
+
desc "Install gem locally"
|
42
|
+
task :install => :build do
|
43
|
+
system "gem install pkg/#{gemspec.name}-#{gemspec.version}"
|
44
|
+
end
|
45
|
+
|
46
|
+
desc "Clean automatically generated files"
|
47
|
+
task :clean do
|
48
|
+
FileUtils.rm_rf "pkg"
|
49
|
+
end
|
data/lib/data_miner.rb
CHANGED
@@ -49,7 +49,18 @@ class DataMiner
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
-
|
52
|
+
attr_writer :logger
|
53
|
+
def logger
|
54
|
+
return @logger if @logger
|
55
|
+
if defined?(::Rails)
|
56
|
+
@logger = ::Rails.logger
|
57
|
+
elsif defined?(::ActiveRecord) and active_record_logger = ::ActiveRecord::Base.logger
|
58
|
+
@logger = active_record_logger
|
59
|
+
else
|
60
|
+
require 'logger'
|
61
|
+
@logger = ::Logger.new $stderr
|
62
|
+
end
|
63
|
+
end
|
53
64
|
|
54
65
|
def resource_names
|
55
66
|
@resource_names ||= []
|
@@ -58,19 +69,7 @@ class DataMiner
|
|
58
69
|
def call_stack
|
59
70
|
@call_stack ||= []
|
60
71
|
end
|
61
|
-
|
62
|
-
def start_logging
|
63
|
-
if logger.nil?
|
64
|
-
if defined? ::Rails
|
65
|
-
self.logger = ::Rails.logger
|
66
|
-
else
|
67
|
-
require 'logger'
|
68
|
-
self.logger = ::Logger.new $stderr
|
69
|
-
end
|
70
|
-
end
|
71
|
-
::ActiveRecord::Base.logger = logger
|
72
|
-
end
|
73
|
-
|
72
|
+
|
74
73
|
# Mine data. Defaults to all resource_names touched by DataMiner.
|
75
74
|
#
|
76
75
|
# Options
|
@@ -27,9 +27,7 @@ class DataMiner
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def data_miner(options = {}, &blk)
|
30
|
-
::DataMiner.instance.
|
31
|
-
|
32
|
-
::DataMiner.instance.resource_names.push self.name unless ::DataMiner.instance.resource_names.include? self.name
|
30
|
+
::DataMiner.instance.resource_names.push name unless ::DataMiner.instance.resource_names.include?(name)
|
33
31
|
|
34
32
|
unless options[:append]
|
35
33
|
self.data_miner_config = ::DataMiner::Config.new self
|
data/lib/data_miner/config.rb
CHANGED
@@ -60,7 +60,7 @@ class DataMiner
|
|
60
60
|
steps.each do |step|
|
61
61
|
time = ::Benchmark.realtime { step.run }
|
62
62
|
resource.reset_column_information
|
63
|
-
|
63
|
+
DataMiner.logger.info %{Ran #{step.inspect} in #{time.to_i}}
|
64
64
|
end
|
65
65
|
finished = true
|
66
66
|
rescue Finish
|
data/lib/data_miner/import.rb
CHANGED
@@ -17,13 +17,13 @@ class DataMiner
|
|
17
17
|
@options.stringify_keys!
|
18
18
|
# legacy
|
19
19
|
if @options.has_key? 'table'
|
20
|
-
|
20
|
+
DataMiner.logger.warn "'table' is no longer an allowed option, taking the url from it and ignoring the rest"
|
21
21
|
table_instance = @options.delete 'table'
|
22
22
|
@options['url'] = table_instance.url
|
23
23
|
end
|
24
24
|
# legacy
|
25
25
|
if @options.has_key?('errata') and not @options['errata'].is_a?(::Hash)
|
26
|
-
|
26
|
+
DataMiner.logger.warn "'errata' must be a hash of Errata options. taking the URL from the Errata instance you provided and ignoring everything else"
|
27
27
|
errata_instance = @options.delete 'errata'
|
28
28
|
@options['errata'] = { 'url' => errata_instance.options['url'] }
|
29
29
|
end
|
@@ -83,7 +83,7 @@ class DataMiner
|
|
83
83
|
begin
|
84
84
|
record.save!
|
85
85
|
rescue
|
86
|
-
|
86
|
+
DataMiner.logger.warn "[data_miner] Got #{$!.inspect} when trying to save #{row}"
|
87
87
|
end
|
88
88
|
end
|
89
89
|
free
|
data/lib/data_miner/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_miner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -15,7 +15,7 @@ date: 2012-02-10 00:00:00.000000000 Z
|
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: remote_table
|
18
|
-
requirement: &
|
18
|
+
requirement: &2152340720 !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
21
|
- - ! '>='
|
@@ -23,10 +23,10 @@ dependencies:
|
|
23
23
|
version: 1.2.2
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
|
-
version_requirements: *
|
26
|
+
version_requirements: *2152340720
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activerecord
|
29
|
-
requirement: &
|
29
|
+
requirement: &2152340000 !ruby/object:Gem::Requirement
|
30
30
|
none: false
|
31
31
|
requirements:
|
32
32
|
- - ! '>='
|
@@ -34,10 +34,10 @@ dependencies:
|
|
34
34
|
version: 2.3.4
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *2152340000
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: activesupport
|
40
|
-
requirement: &
|
40
|
+
requirement: &2152339320 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
@@ -45,10 +45,10 @@ dependencies:
|
|
45
45
|
version: 2.3.4
|
46
46
|
type: :runtime
|
47
47
|
prerelease: false
|
48
|
-
version_requirements: *
|
48
|
+
version_requirements: *2152339320
|
49
49
|
- !ruby/object:Gem::Dependency
|
50
50
|
name: conversions
|
51
|
-
requirement: &
|
51
|
+
requirement: &2152338720 !ruby/object:Gem::Requirement
|
52
52
|
none: false
|
53
53
|
requirements:
|
54
54
|
- - ! '>='
|
@@ -56,10 +56,10 @@ dependencies:
|
|
56
56
|
version: 1.4.4
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
|
-
version_requirements: *
|
59
|
+
version_requirements: *2152338720
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: blockenspiel
|
62
|
-
requirement: &
|
62
|
+
requirement: &2152338100 !ruby/object:Gem::Requirement
|
63
63
|
none: false
|
64
64
|
requirements:
|
65
65
|
- - ! '>='
|
@@ -67,10 +67,10 @@ dependencies:
|
|
67
67
|
version: 0.3.2
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
|
-
version_requirements: *
|
70
|
+
version_requirements: *2152338100
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
72
|
name: errata
|
73
|
-
requirement: &
|
73
|
+
requirement: &2152337460 !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
76
|
- - ! '>='
|
@@ -78,10 +78,10 @@ dependencies:
|
|
78
78
|
version: 1.0.1
|
79
79
|
type: :runtime
|
80
80
|
prerelease: false
|
81
|
-
version_requirements: *
|
81
|
+
version_requirements: *2152337460
|
82
82
|
- !ruby/object:Gem::Dependency
|
83
83
|
name: posix-spawn
|
84
|
-
requirement: &
|
84
|
+
requirement: &2152337000 !ruby/object:Gem::Requirement
|
85
85
|
none: false
|
86
86
|
requirements:
|
87
87
|
- - ! '>='
|
@@ -89,10 +89,10 @@ dependencies:
|
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
|
-
version_requirements: *
|
92
|
+
version_requirements: *2152337000
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
94
|
name: mini_record-compat
|
95
|
-
requirement: &
|
95
|
+
requirement: &2152336520 !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
98
98
|
- - ! '>='
|
@@ -100,10 +100,10 @@ dependencies:
|
|
100
100
|
version: '0'
|
101
101
|
type: :development
|
102
102
|
prerelease: false
|
103
|
-
version_requirements: *
|
103
|
+
version_requirements: *2152336520
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
name: loose_tight_dictionary
|
106
|
-
requirement: &
|
106
|
+
requirement: &2152335880 !ruby/object:Gem::Requirement
|
107
107
|
none: false
|
108
108
|
requirements:
|
109
109
|
- - ! '>='
|
@@ -111,10 +111,10 @@ dependencies:
|
|
111
111
|
version: 0.0.5
|
112
112
|
type: :development
|
113
113
|
prerelease: false
|
114
|
-
version_requirements: *
|
114
|
+
version_requirements: *2152335880
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: test-unit
|
117
|
-
requirement: &
|
117
|
+
requirement: &2152335360 !ruby/object:Gem::Requirement
|
118
118
|
none: false
|
119
119
|
requirements:
|
120
120
|
- - ! '>='
|
@@ -122,10 +122,10 @@ dependencies:
|
|
122
122
|
version: '0'
|
123
123
|
type: :development
|
124
124
|
prerelease: false
|
125
|
-
version_requirements: *
|
125
|
+
version_requirements: *2152335360
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: shoulda
|
128
|
-
requirement: &
|
128
|
+
requirement: &2152351060 !ruby/object:Gem::Requirement
|
129
129
|
none: false
|
130
130
|
requirements:
|
131
131
|
- - ! '>='
|
@@ -133,10 +133,10 @@ dependencies:
|
|
133
133
|
version: '0'
|
134
134
|
type: :development
|
135
135
|
prerelease: false
|
136
|
-
version_requirements: *
|
136
|
+
version_requirements: *2152351060
|
137
137
|
- !ruby/object:Gem::Dependency
|
138
138
|
name: mysql
|
139
|
-
requirement: &
|
139
|
+
requirement: &2152350500 !ruby/object:Gem::Requirement
|
140
140
|
none: false
|
141
141
|
requirements:
|
142
142
|
- - ! '>='
|
@@ -144,10 +144,10 @@ dependencies:
|
|
144
144
|
version: '0'
|
145
145
|
type: :development
|
146
146
|
prerelease: false
|
147
|
-
version_requirements: *
|
147
|
+
version_requirements: *2152350500
|
148
148
|
- !ruby/object:Gem::Dependency
|
149
149
|
name: rake
|
150
|
-
requirement: &
|
150
|
+
requirement: &2152349960 !ruby/object:Gem::Requirement
|
151
151
|
none: false
|
152
152
|
requirements:
|
153
153
|
- - ! '>='
|
@@ -155,7 +155,7 @@ dependencies:
|
|
155
155
|
version: '0'
|
156
156
|
type: :development
|
157
157
|
prerelease: false
|
158
|
-
version_requirements: *
|
158
|
+
version_requirements: *2152349960
|
159
159
|
description: Mine remote data into your ActiveRecord models. You can also convert
|
160
160
|
units.
|
161
161
|
email:
|