seamusabshere-data_miner 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +22 -18
- data/VERSION +1 -1
- data/data_miner.gemspec +1 -1
- data/lib/data_miner/attribute.rb +17 -1
- data/lib/data_miner/configuration.rb +17 -0
- data/lib/data_miner/step.rb +4 -0
- data/lib/data_miner.rb +4 -0
- metadata +2 -3
data/README.rdoc
CHANGED
@@ -8,23 +8,7 @@ Put this in <tt>config/environment.rb</tt>:
|
|
8
8
|
|
9
9
|
config.gem 'seamusabshere-data_miner', :lib => 'data_miner', :source => 'http://gems.github.com'
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
namespace :data_miner do
|
14
|
-
task :mine => :environment do
|
15
|
-
DataMiner.mine :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
|
16
|
-
end
|
17
|
-
end
|
18
|
-
|
19
|
-
You need to specify what order to mine data. For example, in <tt>config/initializers/data_miner_config.rb</tt>:
|
20
|
-
|
21
|
-
DataMiner.enqueue do |queue|
|
22
|
-
queue << Country # class whose data should be mined 1st
|
23
|
-
queue << Airport # class whose data should be mined 2nd
|
24
|
-
# etc
|
25
|
-
end
|
26
|
-
|
27
|
-
You need to define <tt>mine_data</tt> blocks. For example, in <tt>app/models/country.rb</tt>:
|
11
|
+
You need to define <tt>mine_data</tt> blocks in your ActiveRecord models. For example, in <tt>app/models/country.rb</tt>:
|
28
12
|
|
29
13
|
class Country < ActiveRecord::Base
|
30
14
|
mine_data do |step|
|
@@ -37,7 +21,7 @@ You need to define <tt>mine_data</tt> blocks. For example, in <tt>app/models/cou
|
|
37
21
|
end
|
38
22
|
end
|
39
23
|
|
40
|
-
|
24
|
+
...and in <tt>app/models/airport.rb</tt>:
|
41
25
|
|
42
26
|
class Airport < ActiveRecord::Base
|
43
27
|
belongs_to :country
|
@@ -56,6 +40,26 @@ To complete the example, in <tt>app/models/airport.rb</tt>:
|
|
56
40
|
end
|
57
41
|
end
|
58
42
|
|
43
|
+
Put this in <tt>lib/tasks/data_miner_tasks.rake</tt>: (unfortunately I don't know a way to automatically include gem tasks, so you have to do this manually for now)
|
44
|
+
|
45
|
+
namespace :data_miner do
|
46
|
+
task :mine => :environment do
|
47
|
+
DataMiner.mine :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
|
48
|
+
end
|
49
|
+
|
50
|
+
task :map_to_attrs => :environment do
|
51
|
+
DataMiner.map_to_attrs ENV['METHOD'], :class_names => ENV['CLASSES'].to_s.split(/\s*,\s*/).flatten.compact
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
You need to specify what order to mine data. For example, in <tt>config/initializers/data_miner_config.rb</tt>:
|
56
|
+
|
57
|
+
DataMiner.enqueue do |queue|
|
58
|
+
queue << Country # class whose data should be mined 1st
|
59
|
+
queue << Airport # class whose data should be mined 2nd
|
60
|
+
# etc
|
61
|
+
end
|
62
|
+
|
59
63
|
Once you have (1) set up the order of data mining and (2) defined <tt>mine_data</tt> blocks in your classes, you can:
|
60
64
|
|
61
65
|
$ rake data_miner:mine
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.1
|
data/data_miner.gemspec
CHANGED
data/lib/data_miner/attribute.rb
CHANGED
@@ -9,6 +9,22 @@ module DataMiner
|
|
9
9
|
@affected_by_steps = []
|
10
10
|
@key_for_steps = []
|
11
11
|
end
|
12
|
+
|
13
|
+
# polling questions
|
14
|
+
def report_find_or_create(step)
|
15
|
+
"Creates parents: #{klass}##{name} is set with #{reflection_klass(step)}.find_or_create_by_#{foreign_key(step)}" if wants_create?(step)
|
16
|
+
end
|
17
|
+
|
18
|
+
def report_unnatural_order(step)
|
19
|
+
if (
|
20
|
+
(rk = klass.reflect_on_association(weighting_association(step)).andand.klass) or
|
21
|
+
(wants_inline_association? and rk = reflection_klass(step))
|
22
|
+
) and
|
23
|
+
step.configuration.classes.index(rk) > step.configuration.classes.index(klass) and
|
24
|
+
step.options[:awaiting].andand.klass != klass
|
25
|
+
"Unnatural order: #{klass} comes before #{rk}"
|
26
|
+
end
|
27
|
+
end
|
12
28
|
|
13
29
|
def inspect
|
14
30
|
"Attribute(#{klass}.#{name})"
|
@@ -244,7 +260,7 @@ module DataMiner
|
|
244
260
|
def reflection_klass(step)
|
245
261
|
return nil unless reflection
|
246
262
|
if reflection.options[:polymorphic]
|
247
|
-
polymorphic_type(step).constantize
|
263
|
+
polymorphic_type(step).andand.constantize
|
248
264
|
else
|
249
265
|
reflection.klass
|
250
266
|
end
|
@@ -50,6 +50,11 @@ module DataMiner
|
|
50
50
|
def mine(options = {})
|
51
51
|
steps.each { |step| step.perform options }
|
52
52
|
end
|
53
|
+
|
54
|
+
# Map <tt>method</tt> to attributes
|
55
|
+
def map_to_attrs(method)
|
56
|
+
steps.map { |step| step.map_to_attrs(method) }.compact
|
57
|
+
end
|
53
58
|
|
54
59
|
cattr_accessor :classes
|
55
60
|
self.classes = []
|
@@ -65,6 +70,18 @@ module DataMiner
|
|
65
70
|
end
|
66
71
|
end
|
67
72
|
end
|
73
|
+
|
74
|
+
# Map a <tt>method</tt> to attrs. Defaults to all classes touched by DataMiner.
|
75
|
+
#
|
76
|
+
# Options
|
77
|
+
# * <tt>:class_names</tt>: provide an array class names to mine
|
78
|
+
def map_to_attrs(method, options = {})
|
79
|
+
classes.map do |klass|
|
80
|
+
if options[:class_names].blank? or options[:class_names].include?(klass.name)
|
81
|
+
klass.data_mine.map_to_attrs method
|
82
|
+
end
|
83
|
+
end.flatten.compact
|
84
|
+
end
|
68
85
|
|
69
86
|
# Queue up all the ActiveRecord classes that DataMiner should touch.
|
70
87
|
#
|
data/lib/data_miner/step.rb
CHANGED
data/lib/data_miner.rb
CHANGED
@@ -23,6 +23,10 @@ module DataMiner
|
|
23
23
|
def mine(options = {})
|
24
24
|
DataMiner::Configuration.mine options
|
25
25
|
end
|
26
|
+
|
27
|
+
def map_to_attrs(method, options = {})
|
28
|
+
puts DataMiner::Configuration.map_to_attrs(method, options)
|
29
|
+
end
|
26
30
|
|
27
31
|
def enqueue(&block)
|
28
32
|
DataMiner::Configuration.enqueue &block
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: seamusabshere-data_miner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Seamus Abshere
|
@@ -97,7 +97,6 @@ files:
|
|
97
97
|
- test/test_helper.rb
|
98
98
|
has_rdoc: false
|
99
99
|
homepage: http://github.com/seamusabshere/data_miner
|
100
|
-
licenses:
|
101
100
|
post_install_message:
|
102
101
|
rdoc_options:
|
103
102
|
- --charset=UTF-8
|
@@ -120,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
120
119
|
requirements: []
|
121
120
|
|
122
121
|
rubyforge_project:
|
123
|
-
rubygems_version: 1.
|
122
|
+
rubygems_version: 1.2.0
|
124
123
|
signing_key:
|
125
124
|
specification_version: 3
|
126
125
|
summary: Mine remote data into your ActiveRecord models.
|