data_miner 1.3.6 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Rakefile +24 -0
- data/lib/data_miner.rb +13 -14
- data/lib/data_miner/active_record_extensions.rb +1 -3
- data/lib/data_miner/config.rb +1 -1
- data/lib/data_miner/import.rb +3 -3
- data/lib/data_miner/version.rb +1 -1
- metadata +27 -27
data/Rakefile
CHANGED
@@ -23,3 +23,27 @@ begin
|
|
23
23
|
rescue LoadError
|
24
24
|
puts "Rdoc is not available"
|
25
25
|
end
|
26
|
+
|
27
|
+
gemspec = eval(File.read(Dir["*.gemspec"].first))
|
28
|
+
|
29
|
+
desc "Validate the gemspec"
|
30
|
+
task :gemspec do
|
31
|
+
gemspec.validate
|
32
|
+
end
|
33
|
+
|
34
|
+
desc "Build gem locally"
|
35
|
+
task :build => :gemspec do
|
36
|
+
system "gem build #{gemspec.name}.gemspec"
|
37
|
+
FileUtils.mkdir_p "pkg"
|
38
|
+
FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", "pkg"
|
39
|
+
end
|
40
|
+
|
41
|
+
desc "Install gem locally"
|
42
|
+
task :install => :build do
|
43
|
+
system "gem install pkg/#{gemspec.name}-#{gemspec.version}"
|
44
|
+
end
|
45
|
+
|
46
|
+
desc "Clean automatically generated files"
|
47
|
+
task :clean do
|
48
|
+
FileUtils.rm_rf "pkg"
|
49
|
+
end
|
data/lib/data_miner.rb
CHANGED
@@ -49,7 +49,18 @@ class DataMiner
|
|
49
49
|
end
|
50
50
|
end
|
51
51
|
|
52
|
-
|
52
|
+
attr_writer :logger
|
53
|
+
def logger
|
54
|
+
return @logger if @logger
|
55
|
+
if defined?(::Rails)
|
56
|
+
@logger = ::Rails.logger
|
57
|
+
elsif defined?(::ActiveRecord) and active_record_logger = ::ActiveRecord::Base.logger
|
58
|
+
@logger = active_record_logger
|
59
|
+
else
|
60
|
+
require 'logger'
|
61
|
+
@logger = ::Logger.new $stderr
|
62
|
+
end
|
63
|
+
end
|
53
64
|
|
54
65
|
def resource_names
|
55
66
|
@resource_names ||= []
|
@@ -58,19 +69,7 @@ class DataMiner
|
|
58
69
|
def call_stack
|
59
70
|
@call_stack ||= []
|
60
71
|
end
|
61
|
-
|
62
|
-
def start_logging
|
63
|
-
if logger.nil?
|
64
|
-
if defined? ::Rails
|
65
|
-
self.logger = ::Rails.logger
|
66
|
-
else
|
67
|
-
require 'logger'
|
68
|
-
self.logger = ::Logger.new $stderr
|
69
|
-
end
|
70
|
-
end
|
71
|
-
::ActiveRecord::Base.logger = logger
|
72
|
-
end
|
73
|
-
|
72
|
+
|
74
73
|
# Mine data. Defaults to all resource_names touched by DataMiner.
|
75
74
|
#
|
76
75
|
# Options
|
@@ -27,9 +27,7 @@ class DataMiner
|
|
27
27
|
end
|
28
28
|
|
29
29
|
def data_miner(options = {}, &blk)
|
30
|
-
::DataMiner.instance.
|
31
|
-
|
32
|
-
::DataMiner.instance.resource_names.push self.name unless ::DataMiner.instance.resource_names.include? self.name
|
30
|
+
::DataMiner.instance.resource_names.push name unless ::DataMiner.instance.resource_names.include?(name)
|
33
31
|
|
34
32
|
unless options[:append]
|
35
33
|
self.data_miner_config = ::DataMiner::Config.new self
|
data/lib/data_miner/config.rb
CHANGED
@@ -60,7 +60,7 @@ class DataMiner
|
|
60
60
|
steps.each do |step|
|
61
61
|
time = ::Benchmark.realtime { step.run }
|
62
62
|
resource.reset_column_information
|
63
|
-
|
63
|
+
DataMiner.logger.info %{Ran #{step.inspect} in #{time.to_i}}
|
64
64
|
end
|
65
65
|
finished = true
|
66
66
|
rescue Finish
|
data/lib/data_miner/import.rb
CHANGED
@@ -17,13 +17,13 @@ class DataMiner
|
|
17
17
|
@options.stringify_keys!
|
18
18
|
# legacy
|
19
19
|
if @options.has_key? 'table'
|
20
|
-
|
20
|
+
DataMiner.logger.warn "'table' is no longer an allowed option, taking the url from it and ignoring the rest"
|
21
21
|
table_instance = @options.delete 'table'
|
22
22
|
@options['url'] = table_instance.url
|
23
23
|
end
|
24
24
|
# legacy
|
25
25
|
if @options.has_key?('errata') and not @options['errata'].is_a?(::Hash)
|
26
|
-
|
26
|
+
DataMiner.logger.warn "'errata' must be a hash of Errata options. taking the URL from the Errata instance you provided and ignoring everything else"
|
27
27
|
errata_instance = @options.delete 'errata'
|
28
28
|
@options['errata'] = { 'url' => errata_instance.options['url'] }
|
29
29
|
end
|
@@ -83,7 +83,7 @@ class DataMiner
|
|
83
83
|
begin
|
84
84
|
record.save!
|
85
85
|
rescue
|
86
|
-
|
86
|
+
DataMiner.logger.warn "[data_miner] Got #{$!.inspect} when trying to save #{row}"
|
87
87
|
end
|
88
88
|
end
|
89
89
|
free
|
data/lib/data_miner/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_miner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.3.
|
4
|
+
version: 1.3.7
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -15,7 +15,7 @@ date: 2012-02-10 00:00:00.000000000 Z
|
|
15
15
|
dependencies:
|
16
16
|
- !ruby/object:Gem::Dependency
|
17
17
|
name: remote_table
|
18
|
-
requirement: &
|
18
|
+
requirement: &2152340720 !ruby/object:Gem::Requirement
|
19
19
|
none: false
|
20
20
|
requirements:
|
21
21
|
- - ! '>='
|
@@ -23,10 +23,10 @@ dependencies:
|
|
23
23
|
version: 1.2.2
|
24
24
|
type: :runtime
|
25
25
|
prerelease: false
|
26
|
-
version_requirements: *
|
26
|
+
version_requirements: *2152340720
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activerecord
|
29
|
-
requirement: &
|
29
|
+
requirement: &2152340000 !ruby/object:Gem::Requirement
|
30
30
|
none: false
|
31
31
|
requirements:
|
32
32
|
- - ! '>='
|
@@ -34,10 +34,10 @@ dependencies:
|
|
34
34
|
version: 2.3.4
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
|
-
version_requirements: *
|
37
|
+
version_requirements: *2152340000
|
38
38
|
- !ruby/object:Gem::Dependency
|
39
39
|
name: activesupport
|
40
|
-
requirement: &
|
40
|
+
requirement: &2152339320 !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
43
|
- - ! '>='
|
@@ -45,10 +45,10 @@ dependencies:
|
|
45
45
|
version: 2.3.4
|
46
46
|
type: :runtime
|
47
47
|
prerelease: false
|
48
|
-
version_requirements: *
|
48
|
+
version_requirements: *2152339320
|
49
49
|
- !ruby/object:Gem::Dependency
|
50
50
|
name: conversions
|
51
|
-
requirement: &
|
51
|
+
requirement: &2152338720 !ruby/object:Gem::Requirement
|
52
52
|
none: false
|
53
53
|
requirements:
|
54
54
|
- - ! '>='
|
@@ -56,10 +56,10 @@ dependencies:
|
|
56
56
|
version: 1.4.4
|
57
57
|
type: :runtime
|
58
58
|
prerelease: false
|
59
|
-
version_requirements: *
|
59
|
+
version_requirements: *2152338720
|
60
60
|
- !ruby/object:Gem::Dependency
|
61
61
|
name: blockenspiel
|
62
|
-
requirement: &
|
62
|
+
requirement: &2152338100 !ruby/object:Gem::Requirement
|
63
63
|
none: false
|
64
64
|
requirements:
|
65
65
|
- - ! '>='
|
@@ -67,10 +67,10 @@ dependencies:
|
|
67
67
|
version: 0.3.2
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
|
-
version_requirements: *
|
70
|
+
version_requirements: *2152338100
|
71
71
|
- !ruby/object:Gem::Dependency
|
72
72
|
name: errata
|
73
|
-
requirement: &
|
73
|
+
requirement: &2152337460 !ruby/object:Gem::Requirement
|
74
74
|
none: false
|
75
75
|
requirements:
|
76
76
|
- - ! '>='
|
@@ -78,10 +78,10 @@ dependencies:
|
|
78
78
|
version: 1.0.1
|
79
79
|
type: :runtime
|
80
80
|
prerelease: false
|
81
|
-
version_requirements: *
|
81
|
+
version_requirements: *2152337460
|
82
82
|
- !ruby/object:Gem::Dependency
|
83
83
|
name: posix-spawn
|
84
|
-
requirement: &
|
84
|
+
requirement: &2152337000 !ruby/object:Gem::Requirement
|
85
85
|
none: false
|
86
86
|
requirements:
|
87
87
|
- - ! '>='
|
@@ -89,10 +89,10 @@ dependencies:
|
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
|
-
version_requirements: *
|
92
|
+
version_requirements: *2152337000
|
93
93
|
- !ruby/object:Gem::Dependency
|
94
94
|
name: mini_record-compat
|
95
|
-
requirement: &
|
95
|
+
requirement: &2152336520 !ruby/object:Gem::Requirement
|
96
96
|
none: false
|
97
97
|
requirements:
|
98
98
|
- - ! '>='
|
@@ -100,10 +100,10 @@ dependencies:
|
|
100
100
|
version: '0'
|
101
101
|
type: :development
|
102
102
|
prerelease: false
|
103
|
-
version_requirements: *
|
103
|
+
version_requirements: *2152336520
|
104
104
|
- !ruby/object:Gem::Dependency
|
105
105
|
name: loose_tight_dictionary
|
106
|
-
requirement: &
|
106
|
+
requirement: &2152335880 !ruby/object:Gem::Requirement
|
107
107
|
none: false
|
108
108
|
requirements:
|
109
109
|
- - ! '>='
|
@@ -111,10 +111,10 @@ dependencies:
|
|
111
111
|
version: 0.0.5
|
112
112
|
type: :development
|
113
113
|
prerelease: false
|
114
|
-
version_requirements: *
|
114
|
+
version_requirements: *2152335880
|
115
115
|
- !ruby/object:Gem::Dependency
|
116
116
|
name: test-unit
|
117
|
-
requirement: &
|
117
|
+
requirement: &2152335360 !ruby/object:Gem::Requirement
|
118
118
|
none: false
|
119
119
|
requirements:
|
120
120
|
- - ! '>='
|
@@ -122,10 +122,10 @@ dependencies:
|
|
122
122
|
version: '0'
|
123
123
|
type: :development
|
124
124
|
prerelease: false
|
125
|
-
version_requirements: *
|
125
|
+
version_requirements: *2152335360
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: shoulda
|
128
|
-
requirement: &
|
128
|
+
requirement: &2152351060 !ruby/object:Gem::Requirement
|
129
129
|
none: false
|
130
130
|
requirements:
|
131
131
|
- - ! '>='
|
@@ -133,10 +133,10 @@ dependencies:
|
|
133
133
|
version: '0'
|
134
134
|
type: :development
|
135
135
|
prerelease: false
|
136
|
-
version_requirements: *
|
136
|
+
version_requirements: *2152351060
|
137
137
|
- !ruby/object:Gem::Dependency
|
138
138
|
name: mysql
|
139
|
-
requirement: &
|
139
|
+
requirement: &2152350500 !ruby/object:Gem::Requirement
|
140
140
|
none: false
|
141
141
|
requirements:
|
142
142
|
- - ! '>='
|
@@ -144,10 +144,10 @@ dependencies:
|
|
144
144
|
version: '0'
|
145
145
|
type: :development
|
146
146
|
prerelease: false
|
147
|
-
version_requirements: *
|
147
|
+
version_requirements: *2152350500
|
148
148
|
- !ruby/object:Gem::Dependency
|
149
149
|
name: rake
|
150
|
-
requirement: &
|
150
|
+
requirement: &2152349960 !ruby/object:Gem::Requirement
|
151
151
|
none: false
|
152
152
|
requirements:
|
153
153
|
- - ! '>='
|
@@ -155,7 +155,7 @@ dependencies:
|
|
155
155
|
version: '0'
|
156
156
|
type: :development
|
157
157
|
prerelease: false
|
158
|
-
version_requirements: *
|
158
|
+
version_requirements: *2152349960
|
159
159
|
description: Mine remote data into your ActiveRecord models. You can also convert
|
160
160
|
units.
|
161
161
|
email:
|