data_miner 0.4.35 → 0.4.36
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/data_miner.gemspec +1 -1
- data/lib/data_miner.rb +2 -17
- data/lib/data_miner/configuration.rb +2 -0
- data/test/data_miner_test.rb +24 -3
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.36
|
data/data_miner.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{data_miner}
|
8
|
-
s.version = "0.4.
|
8
|
+
s.version = "0.4.36"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
|
data/lib/data_miner.rb
CHANGED
@@ -29,7 +29,8 @@ require 'data_miner/run'
|
|
29
29
|
require 'data_miner/schema'
|
30
30
|
|
31
31
|
module DataMiner
|
32
|
-
class MissingHashColumn <
|
32
|
+
class MissingHashColumn < StandardError; end
|
33
|
+
class Stop < StandardError; end
|
33
34
|
|
34
35
|
mattr_accessor :logger
|
35
36
|
|
@@ -96,22 +97,6 @@ Output:
|
|
96
97
|
end
|
97
98
|
|
98
99
|
ActiveRecord::Base.class_eval do
|
99
|
-
# convenience
|
100
|
-
%w{
|
101
|
-
create_table
|
102
|
-
execute
|
103
|
-
add_column
|
104
|
-
rename_column
|
105
|
-
remove_column
|
106
|
-
drop_table
|
107
|
-
}.each do |method_name|
|
108
|
-
eval %{
|
109
|
-
def self.#{method_name}(*args, &block)
|
110
|
-
ActiveRecord::Base.connection.send(:#{method_name}, *args, &block)
|
111
|
-
end
|
112
|
-
}
|
113
|
-
end
|
114
|
-
|
115
100
|
def self.x_data_miner(&block)
|
116
101
|
DataMiner.start_logging
|
117
102
|
|
@@ -63,6 +63,8 @@ module DataMiner
|
|
63
63
|
resource.reset_column_information
|
64
64
|
end
|
65
65
|
finished = true
|
66
|
+
rescue DataMiner::Stop
|
67
|
+
finished = true
|
66
68
|
ensure
|
67
69
|
run.update_attributes! :ended_at => Time.now, :finished => finished if DataMiner::Run.table_exists?
|
68
70
|
DataMiner::Configuration.call_stack.clear if DataMiner::Configuration.call_stack.first == resource.name
|
data/test/data_miner_test.rb
CHANGED
@@ -503,12 +503,12 @@ class CrosscallingCensusRegion < ActiveRecord::Base
|
|
503
503
|
data_miner do
|
504
504
|
process "derive ourselves from the census divisions table (i.e., cross call census divisions)" do
|
505
505
|
CrosscallingCensusDivision.run_data_miner!
|
506
|
-
create_table :crosscalling_census_regions, :options => 'ENGINE=InnoDB default charset=utf8', :id => false, :force => true do |t|
|
506
|
+
connection.create_table :crosscalling_census_regions, :options => 'ENGINE=InnoDB default charset=utf8', :id => false, :force => true do |t|
|
507
507
|
t.column :number, :integer
|
508
508
|
t.column :name, :string
|
509
509
|
end
|
510
|
-
execute 'ALTER TABLE crosscalling_census_regions ADD PRIMARY KEY (number);'
|
511
|
-
execute %{
|
510
|
+
connection.execute 'ALTER TABLE crosscalling_census_regions ADD PRIMARY KEY (number);'
|
511
|
+
connection.execute %{
|
512
512
|
INSERT IGNORE INTO crosscalling_census_regions(number, name)
|
513
513
|
SELECT crosscalling_census_divisions.census_region_number, crosscalling_census_divisions.census_region_name FROM crosscalling_census_divisions
|
514
514
|
}
|
@@ -976,6 +976,10 @@ class Aircraft < ActiveRecord::Base
|
|
976
976
|
|
977
977
|
class Guru
|
978
978
|
# for errata
|
979
|
+
def is_attributed_to_boeing?(row)
|
980
|
+
row['Manufacturer'] =~ /BOEING/i
|
981
|
+
end
|
982
|
+
|
979
983
|
def is_not_attributed_to_aerospatiale?(row)
|
980
984
|
not row['Manufacturer'] =~ /AEROSPATIALE/i
|
981
985
|
end
|
@@ -1105,6 +1109,10 @@ class AutomobileMakeFleetYear < ActiveRecord::Base
|
|
1105
1109
|
integer 'data_miner_touch_count'
|
1106
1110
|
integer 'data_miner_last_run_id'
|
1107
1111
|
end
|
1112
|
+
|
1113
|
+
process "stop if i tell you to" do
|
1114
|
+
raise DataMiner::Stop if $stop_stop
|
1115
|
+
end
|
1108
1116
|
|
1109
1117
|
# CAFE data privately emailed to Andy from Terry Anderson at the DOT/NHTSA
|
1110
1118
|
import :url => 'http://static.brighterplanet.com/science/data/transport/automobiles/make_fleet_years/make_fleet_years.csv',
|
@@ -1151,6 +1159,19 @@ class DataMinerTest < Test::Unit::TestCase
|
|
1151
1159
|
end
|
1152
1160
|
|
1153
1161
|
if ENV['ALL'] == 'true' or ENV['FAST'] == 'true'
|
1162
|
+
should "stop and finish if it gets a DataMiner::Stop" do
|
1163
|
+
AutomobileMakeFleetYear.delete_all
|
1164
|
+
AutomobileMakeFleetYear.data_miner_runs.delete_all
|
1165
|
+
$stop_stop = true
|
1166
|
+
AutomobileMakeFleetYear.run_data_miner!
|
1167
|
+
assert_equal 0, AutomobileMakeFleetYear.count
|
1168
|
+
assert_equal true, (AutomobileMakeFleetYear.data_miner_runs.count > 0)
|
1169
|
+
assert_equal true, AutomobileMakeFleetYear.data_miner_runs.all? { |run| run.finished? }
|
1170
|
+
$stop_stop = false
|
1171
|
+
AutomobileMakeFleetYear.run_data_miner!
|
1172
|
+
assert AutomobileMakeFleetYear.exists?(:name => 'Alfa Romeo IP 1978')
|
1173
|
+
end
|
1174
|
+
|
1154
1175
|
should "eagerly enforce a schema" do
|
1155
1176
|
ActiveRecord::Base.connection.create_table 'census_division_trois', :force => true, :options => 'ENGINE=InnoDB default charset=utf8' do |t|
|
1156
1177
|
t.string 'name'
|