data_miner 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/data_miner.gemspec +1 -1
- data/lib/data_miner.rb +5 -1
- data/lib/data_miner/attribute.rb +1 -1
- data/lib/data_miner/configuration.rb +14 -1
- data/lib/data_miner/import.rb +7 -4
- data/lib/data_miner/process.rb +1 -1
- metadata +2 -2
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.11
|
data/data_miner.gemspec
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = %q{data_miner}
|
8
|
-
s.version = "0.4.
|
8
|
+
s.version = "0.4.11"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Seamus Abshere", "Andy Rossmeissl"]
|
data/lib/data_miner.rb
CHANGED
@@ -46,6 +46,10 @@ module DataMiner
|
|
46
46
|
end
|
47
47
|
end
|
48
48
|
|
49
|
+
def self.log_info(message)
|
50
|
+
logger.info "[data_miner gem] #{message}"
|
51
|
+
end
|
52
|
+
|
49
53
|
def self.run(options = {})
|
50
54
|
DataMiner::Configuration.run options
|
51
55
|
end
|
@@ -59,7 +63,7 @@ ActiveRecord::Base.class_eval do
|
|
59
63
|
def self.x_data_miner(&block)
|
60
64
|
DataMiner.start_logging
|
61
65
|
|
62
|
-
DataMiner.
|
66
|
+
DataMiner.log_info "Skipping data_miner block in #{self.name} because called as x_data_miner"
|
63
67
|
end
|
64
68
|
|
65
69
|
def self.data_miner(&block)
|
data/lib/data_miner/attribute.rb
CHANGED
@@ -69,7 +69,7 @@ module DataMiner
|
|
69
69
|
|
70
70
|
what_it_is = record.send name
|
71
71
|
if what_it_is.nil? and !what_it_should_be.nil?
|
72
|
-
DataMiner.
|
72
|
+
DataMiner.log_info "ActiveRecord didn't like trying to set #{resource}.#{name} = #{what_it_should_be} (it came out as nil)"
|
73
73
|
nil
|
74
74
|
elsif what_it_is == what_it_was
|
75
75
|
false
|
@@ -38,7 +38,7 @@ module DataMiner
|
|
38
38
|
if DataMiner::Run.table_exists?
|
39
39
|
run = DataMiner::Run.create! :started_at => Time.now, :resource_name => resource.name if DataMiner::Run.table_exists?
|
40
40
|
else
|
41
|
-
DataMiner.
|
41
|
+
DataMiner.log_info "Not logging individual runs. Please run DataMiner::Run.create_tables if you want to enable this."
|
42
42
|
end
|
43
43
|
resource.delete_all if options[:from_scratch]
|
44
44
|
begin
|
@@ -95,6 +95,16 @@ You need to supply one of #{COMPLETE_UNIT_DEFINITIONS.map(&:inspect).to_sentence
|
|
95
95
|
|
96
96
|
def suggest_missing_column_migrations
|
97
97
|
missing_columns = Array.new
|
98
|
+
non_essential_missing_columns = Array.new
|
99
|
+
unless resource.column_names.include?('data_miner_touch_count')
|
100
|
+
non_essential_missing_columns << 'data_miner_touch_count'
|
101
|
+
DataMiner.log_info "Not counting how many times a row has been touched by data_miner."
|
102
|
+
end
|
103
|
+
unless resource.column_names.include?('data_miner_last_run_id')
|
104
|
+
non_essential_missing_columns << 'data_miner_last_run_id'
|
105
|
+
DataMiner.log_info "Not recording which run touched a row."
|
106
|
+
end
|
107
|
+
|
98
108
|
import_runnables.each do |runnable|
|
99
109
|
runnable.attributes.each do |_, attribute|
|
100
110
|
DataMiner.log_or_raise "You can't have an attribute column that ends in _units (reserved): #{resource.table_name}.#{attribute.name}" if attribute.name.ends_with? '_units'
|
@@ -123,16 +133,19 @@ and **replace** the resulting file with this:
|
|
123
133
|
class AddMissingColumnsTo#{resource.name} < ActiveRecord::Migration
|
124
134
|
def self.up
|
125
135
|
#{missing_columns.map { |column_name| " add_column :#{resource.table_name}, :#{column_name}, :#{column_name.ends_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' }" }.join("\n") }
|
136
|
+
#{non_essential_missing_columns.map { |column_name| " add_column :#{resource.table_name}, :#{column_name}, :integer #optional" }.join("\n") }
|
126
137
|
end
|
127
138
|
|
128
139
|
def self.down
|
129
140
|
#{missing_columns.map { |column_name| " remove_column :#{resource.table_name}, :#{column_name}" }.join("\n") }
|
141
|
+
#{non_essential_missing_columns.map { |column_name| " remove_column :#{resource.table_name}, :#{column_name} #optional" }.join("\n") }
|
130
142
|
end
|
131
143
|
end
|
132
144
|
|
133
145
|
On the other hand, if you're working directly with create_table, this might be helpful:
|
134
146
|
|
135
147
|
#{missing_columns.map { |column_name| "t.#{column_name.ends_with?('_units') ? 'string' : 'FIXME_WHAT_COLUMN_TYPE_AM_I' } '#{column_name}'" }.join("\n") }
|
148
|
+
#{non_essential_missing_columns.map { |column_name| "t.integer '#{column_name}' #optional" }.join("\n") }
|
136
149
|
|
137
150
|
================================
|
138
151
|
}
|
data/lib/data_miner/import.rb
CHANGED
@@ -40,6 +40,10 @@ module DataMiner
|
|
40
40
|
|
41
41
|
def run(run)
|
42
42
|
begin; ActiveRecord::Base.connection.execute("SET NAMES 'utf8'"); rescue; end
|
43
|
+
|
44
|
+
increment_counter = resource.column_names.include?('data_miner_touch_count')
|
45
|
+
log_run = resource.column_names.include?('data_miner_last_run_id')
|
46
|
+
|
43
47
|
table.each_row do |row|
|
44
48
|
if errata
|
45
49
|
next if errata.rejects?(row)
|
@@ -48,14 +52,13 @@ module DataMiner
|
|
48
52
|
|
49
53
|
record = resource.send "find_or_initialize_by_#{@key}", attributes[@key].value_from_row(row)
|
50
54
|
changes = attributes.map { |_, attr| attr.set_record_from_row record, row }
|
51
|
-
record.data_miner_touch_count ||= 0
|
52
55
|
if changes.any?
|
53
|
-
record.data_miner_touch_count
|
54
|
-
record.data_miner_last_run = run
|
56
|
+
record.increment :data_miner_touch_count if increment_counter
|
57
|
+
record.data_miner_last_run = run if log_run
|
55
58
|
end
|
56
59
|
record.save!
|
57
60
|
end
|
58
|
-
DataMiner.
|
61
|
+
DataMiner.log_info "performed #{inspect}"
|
59
62
|
end
|
60
63
|
end
|
61
64
|
end
|
data/lib/data_miner/process.rb
CHANGED