data_miner 0.2.6 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +1 -0
- data/CHANGELOG +5 -0
- data/README.rdoc +11 -15
- data/Rakefile +7 -2
- data/VERSION +1 -1
- data/data_miner.gemspec +27 -28
- data/lib/data_miner.rb +50 -27
- data/lib/data_miner/attribute.rb +157 -240
- data/lib/data_miner/configuration.rb +58 -55
- data/lib/data_miner/import.rb +57 -0
- data/lib/data_miner/process.rb +21 -0
- data/lib/data_miner/run.rb +7 -0
- data/lib/data_miner/target.rb +7 -0
- data/test/data_miner_test.rb +644 -48
- data/test/test_helper.rb +134 -3
- metadata +29 -23
- data/lib/data_miner/active_record_ext.rb +0 -25
- data/lib/data_miner/attribute_collection.rb +0 -51
- data/lib/data_miner/step.rb +0 -64
- data/lib/data_miner/step/associate.rb +0 -9
- data/lib/data_miner/step/await.rb +0 -35
- data/lib/data_miner/step/callback.rb +0 -22
- data/lib/data_miner/step/derive.rb +0 -9
- data/lib/data_miner/step/import.rb +0 -57
@@ -1,22 +0,0 @@
|
|
1
|
-
module DataMiner
|
2
|
-
class Step
|
3
|
-
class Callback < Step
|
4
|
-
attr_accessor :foreign_step
|
5
|
-
|
6
|
-
def initialize(configuration, foreign_step)
|
7
|
-
@configuration = configuration
|
8
|
-
@foreign_step = foreign_step
|
9
|
-
@number = "(last)"
|
10
|
-
end
|
11
|
-
|
12
|
-
def perform(*args)
|
13
|
-
foreign_step.callback
|
14
|
-
$stderr.puts "performed #{signature}"
|
15
|
-
end
|
16
|
-
|
17
|
-
def signature
|
18
|
-
"#{super} (on behalf of #{foreign_step.signature})"
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
module DataMiner
|
2
|
-
class Step
|
3
|
-
class Import < Step
|
4
|
-
attr_accessor :table, :errata
|
5
|
-
|
6
|
-
def initialize(configuration, number, options = {}, &block)
|
7
|
-
super
|
8
|
-
@errata = Errata.new(:url => options[:errata], :klass => klass) if options[:errata]
|
9
|
-
@table = RemoteTable.new(options.slice(:url, :filename, :post_data, :format, :skip, :cut, :schema, :schema_name, :trap, :select, :reject, :sheet, :delimiter, :headers, :transform, :crop))
|
10
|
-
end
|
11
|
-
|
12
|
-
def signature
|
13
|
-
"#{super} #{options[:url]}"
|
14
|
-
end
|
15
|
-
|
16
|
-
def perform(*args)
|
17
|
-
ActiveRecord::Base.connection.execute("TRUNCATE #{klass.quoted_table_name}") if wants_truncate?
|
18
|
-
table.each_row do |row|
|
19
|
-
if errata
|
20
|
-
next if errata.rejects?(row)
|
21
|
-
errata.correct!(row)
|
22
|
-
end
|
23
|
-
if uses_existing_data?
|
24
|
-
key_values = key_attributes.map { |key_attr| [ key_attr.value_from_row(self, row) ] }
|
25
|
-
record_set = WilliamJamesCartesianProduct.cart_prod(*key_values).map do |combination|
|
26
|
-
next if combination.include?(nil) and !wants_nil_keys?
|
27
|
-
klass.send(dynamic_finder_name, *combination)
|
28
|
-
end.flatten
|
29
|
-
else
|
30
|
-
record_set = klass.new
|
31
|
-
end
|
32
|
-
Array.wrap(record_set).each do |record|
|
33
|
-
affected_attributes.each { |attr| attr.set_record_from_row(self, record, row) }
|
34
|
-
record.save
|
35
|
-
end
|
36
|
-
end
|
37
|
-
$stderr.puts "performed #{signature}"
|
38
|
-
end
|
39
|
-
|
40
|
-
def wants_truncate?
|
41
|
-
options[:truncate] == true or (!(options[:truncate] == false) and !uses_existing_data?)
|
42
|
-
end
|
43
|
-
|
44
|
-
def wants_nil_keys?
|
45
|
-
options[:allow_nil_keys] == true
|
46
|
-
end
|
47
|
-
|
48
|
-
def uses_existing_data?
|
49
|
-
@uses_existing_data ||= attributes.has_keys_for?(self) or attributes.has_conditional_writes_for?(self)
|
50
|
-
end
|
51
|
-
|
52
|
-
def dynamic_finder_name
|
53
|
-
"find_or_initialize_by_#{key_attributes.map(&:name).join('_and_')}".to_sym
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|