wonkavision 0.5.11 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/CHANGELOG.rdoc +3 -0
  2. data/lib/wonkavision.rb +28 -1
  3. data/lib/wonkavision/aggregation.rb +21 -0
  4. data/lib/wonkavision/event_coordinator.rb +19 -7
  5. data/lib/wonkavision/extensions/symbol.rb +55 -0
  6. data/lib/wonkavision/facts.rb +27 -0
  7. data/lib/wonkavision/local_job_queue.rb +28 -0
  8. data/lib/wonkavision/message_mapper.rb +2 -2
  9. data/lib/wonkavision/message_mapper/map.rb +60 -8
  10. data/lib/wonkavision/persistence/mongo.rb +95 -0
  11. data/lib/wonkavision/plugins.rb +2 -1
  12. data/lib/wonkavision/plugins/analytics/aggregation.rb +139 -0
  13. data/lib/wonkavision/plugins/analytics/aggregation/aggregation_spec.rb +53 -0
  14. data/lib/wonkavision/plugins/analytics/aggregation/attribute.rb +22 -0
  15. data/lib/wonkavision/plugins/analytics/aggregation/dimension.rb +64 -0
  16. data/lib/wonkavision/plugins/analytics/aggregation/measure.rb +240 -0
  17. data/lib/wonkavision/plugins/analytics/cellset.rb +171 -0
  18. data/lib/wonkavision/plugins/analytics/facts.rb +106 -0
  19. data/lib/wonkavision/plugins/analytics/handlers/apply_aggregation.rb +35 -0
  20. data/lib/wonkavision/plugins/analytics/handlers/split_by_aggregation.rb +60 -0
  21. data/lib/wonkavision/plugins/analytics/member_filter.rb +106 -0
  22. data/lib/wonkavision/plugins/analytics/mongo.rb +6 -0
  23. data/lib/wonkavision/plugins/analytics/persistence/hash_store.rb +59 -0
  24. data/lib/wonkavision/plugins/analytics/persistence/mongo_store.rb +85 -0
  25. data/lib/wonkavision/plugins/analytics/persistence/store.rb +105 -0
  26. data/lib/wonkavision/plugins/analytics/query.rb +76 -0
  27. data/lib/wonkavision/plugins/event_handling.rb +15 -3
  28. data/lib/wonkavision/version.rb +1 -1
  29. data/test/aggregation_spec_test.rb +99 -0
  30. data/test/aggregation_test.rb +170 -0
  31. data/test/analytics/test_aggregation.rb +78 -0
  32. data/test/apply_aggregation_test.rb +92 -0
  33. data/test/attribute_test.rb +26 -0
  34. data/test/cellset_test.rb +200 -0
  35. data/test/dimension_test.rb +186 -0
  36. data/test/facts_test.rb +146 -0
  37. data/test/hash_store_test.rb +112 -0
  38. data/test/log/test.log +96844 -0
  39. data/test/map_test.rb +48 -1
  40. data/test/measure_test.rb +146 -0
  41. data/test/member_filter_test.rb +143 -0
  42. data/test/mongo_store_test.rb +115 -0
  43. data/test/query_test.rb +106 -0
  44. data/test/split_by_aggregation_test.rb +114 -0
  45. data/test/store_test.rb +71 -0
  46. data/test/symbol_test.rb +62 -0
  47. data/test/test_activity_models.rb +1 -1
  48. data/test/test_aggregation.rb +42 -0
  49. data/test/test_data.tuples +100 -0
  50. data/test/test_helper.rb +7 -0
  51. metadata +57 -5
@@ -0,0 +1,170 @@
1
+ require "test_helper"
2
+
3
+ class AggregationTest < ActiveSupport::TestCase
4
+ context "Aggregation" do
5
+ setup do
6
+ @facts = Class.new
7
+ @facts.class_eval do
8
+ include Wonkavision::Facts
9
+ end
10
+
11
+ @agg = Class.new
12
+ @agg.class_eval do
13
+ def self.name; "MyAggregation"; end
14
+ include Wonkavision::Aggregation
15
+ dimension :a, :b, :c
16
+
17
+ dimension :complex do
18
+ key :cpx
19
+ end
20
+
21
+ measure :d
22
+ store :hash_store
23
+ end
24
+ @agg.aggregates @facts
25
+
26
+ end
27
+
28
+ should "configure a specification" do
29
+ assert_not_nil @agg.aggregation_spec
30
+ end
31
+
32
+ should "set the name of the aggregation to the name of the class" do
33
+ assert_equal @agg.name, @agg.aggregation_spec.name
34
+ end
35
+
36
+ should "proxy relevant calls to the specification" do
37
+ assert_equal @agg.dimensions, @agg.aggregation_spec.dimensions
38
+ assert_equal 4, @agg.dimensions.length
39
+ end
40
+
41
+ should "create complex dimensions" do
42
+ assert_equal :cpx, @agg.dimensions[:complex].key
43
+ end
44
+
45
+ should "register itself with the module" do
46
+ assert_equal @agg, Wonkavision::Aggregation.all[@agg.name]
47
+ end
48
+
49
+ should "set the aggregates property" do
50
+ assert_equal @facts, @agg.aggregates
51
+ end
52
+
53
+ should "register itself with its associated Facts class" do
54
+ assert_equal 1, @facts.aggregations.length
55
+ assert_equal @agg, @facts.aggregations[0]
56
+ end
57
+
58
+ should "set the specified storage" do
59
+ assert @agg.store.kind_of?(Wonkavision::Analytics::Persistence::HashStore)
60
+ assert_equal @agg, @agg.store.owner
61
+ end
62
+
63
+ should "manage a list of cached instances keyed by dimension hashes" do
64
+ instance = @agg[{ "a" => { "a"=>:b}}]
65
+ assert_not_nil instance
66
+ assert_equal instance, @agg[{ "a" => { "a"=>:b}}]
67
+ assert_not_equal instance, @agg[{ "a" => { "a"=>:b}, "b" => { "b"=>:c}}]
68
+ end
69
+
70
+ should "store the dimension list with the instance" do
71
+ instance = @agg[{ "a" => { "a"=>:b}}]
72
+ assert_equal( { "a" => { "a"=>:b}}, instance.dimensions )
73
+ end
74
+
75
+ context "#query" do
76
+ should "create a new query" do
77
+ assert @agg.query(:defer=>true).kind_of?(Wonkavision::Analytics::Query)
78
+ end
79
+ should "apply a provided block to the query" do
80
+ assert_equal [:a], @agg.query(:defer=>true){ select :a }.selected_dimensions
81
+ end
82
+ should "raise an error if the query is invalid" do
83
+ assert_raise(RuntimeError) { @agg.query{ select :a, :on => :rows} }
84
+ end
85
+ should "execute the query against the configured store" do
86
+ @agg.store.expects(:execute_query).returns([])
87
+ @agg.query
88
+ end
89
+ should "return a cellset based on the query results" do
90
+ @agg.store.expects(:execute_query).returns([])
91
+ assert @agg.query.kind_of?(Wonkavision::Analytics::CellSet)
92
+ end
93
+ end
94
+
95
+ context "instance methods" do
96
+ setup do
97
+ @instance = @agg[{ "a" => { "a"=>:b}}]
98
+ end
99
+
100
+ context "#dimension_names" do
101
+ should "present dimension names as an array" do
102
+ assert_equal ["a"], @instance.dimension_names
103
+ end
104
+ end
105
+
106
+ context "#dimension_keys" do
107
+ should "present dimension keys as an array" do
108
+ assert_equal [:b], @instance.dimension_keys
109
+ end
110
+ end
111
+
112
+ context "#add" do
113
+ should "call update with the appropriate action" do
114
+ @instance.expects(:update).with({:a=>:b},:add)
115
+ @instance.add({:a=>:b})
116
+ end
117
+ end
118
+
119
+ context "#reject" do
120
+ should "call update with the appropriate action" do
121
+ @instance.expects(:update).with({ :a=>:b}, :reject)
122
+ @instance.reject({:a=>:b})
123
+ end
124
+ end
125
+
126
+ context "#measure_changes_for" do
127
+ setup do
128
+ @added = @instance.send(:measure_changes_for,"a", 1000, "add")
129
+ @rejected = @instance.send(:measure_changes_for,"a", 1000, "reject")
130
+ end
131
+
132
+ should "prepare a hash of measure components" do
133
+ assert_equal 3, @added.length
134
+ end
135
+ should "prepare a count component" do
136
+ assert_equal 1, @added["measures.a.count"]
137
+ end
138
+ should "prepare a sum component" do
139
+ assert_equal 1000, @added["measures.a.sum"]
140
+ end
141
+ should "prepare a sum2 component" do
142
+ assert_equal 1000*1000, @added["measures.a.sum2"]
143
+ end
144
+ should "reverse the sign of the measures when the action is reject" do
145
+ @rejected.values.each { |val| assert val < 0}
146
+ end
147
+ end
148
+
149
+ context "#update" do
150
+ should "prepare aggrgation data and submit it to store.update_aggregation" do
151
+ expected = {
152
+ :dimension_keys => [:b],
153
+ :dimension_names => ["a"],
154
+ :measures => {
155
+ "measures.d.count" => 1,
156
+ "measures.d.sum" => 1000,
157
+ "measures.d.sum2" => 1000*1000 },
158
+ :dimensions => { "a" => { "a"=>:b}}
159
+ }
160
+
161
+ @instance.class.store.expects(:update_aggregation).with(expected)
162
+ @instance.send(:update, { "d" => 1000}, :add)
163
+
164
+ end
165
+
166
+ end
167
+
168
+ end
169
+ end
170
+ end
@@ -0,0 +1,78 @@
1
+ module Rpm
2
+ class AccountSummary
3
+ include Wonkavision::Analytics::Aggregation
4
+
5
+ event 'wv.billing_record.sample_added',
6
+ 'wv.billing_record.sample_retracted'
7
+
8
+ dimension :company_id
9
+ dimension :current_payer_class, :current_payer
10
+ dimension :primary_payer_class, :primary_payer
11
+ dimension :status_category, :status
12
+ dimension :age_category
13
+ dimension :has_credit_balance
14
+
15
+ measure :age_in_days
16
+ measure :write_offs, :charges, :payments, :current_balance
17
+
18
+ aggregate_by :company_id do
19
+ aggregate_by :status_category
20
+ aggregate_by :status
21
+ aggregate_by :primary_payer_class, :has_credit_balance
22
+ aggregate_by :current_payer_class, :has_credit_balance
23
+ end
24
+
25
+ end
26
+ end
27
+
28
+ #wv.billing_record.sample
29
+ {
30
+ action => :add,
31
+ #dimension data
32
+ :company_id => "123",
33
+ :current_payer_class => "commercial",
34
+ :current_payer => "payer_xyz",
35
+ :primary_payer_class => "commercial",
36
+ :primary_payer => "payer_abc",
37
+ :status_category => "category 1",
38
+ :status => :"status 1",
39
+ :age_category => "0-30 days",
40
+ :has_credit_balance => true,
41
+ #measure data
42
+ :age_in_days => 21,
43
+ :write_offs => 123.45,
44
+ :charges => 456.78,
45
+ :payments => 12.34,
46
+ :current_balance => 320.99
47
+ }
48
+ #collection name:
49
+ #wv.rpm.account_summary
50
+ {
51
+ action => :add,
52
+ dimensions => { :status_category => "category 1"},
53
+ measures => {
54
+ :age_in_days => {
55
+ :count => 1,
56
+ :sum => 123,
57
+ :sum2 => 123,
58
+ :mean => 123,
59
+ ...
60
+
61
+ },
62
+ :write_offs => {
63
+ :count => 1,
64
+ :sum => 123,
65
+ ...
66
+ }
67
+ }
68
+ }
69
+
70
+ #wv.rpm.account_summary.samples
71
+ {
72
+ dimensions => { :status_category => "category 1"},
73
+ measures => {
74
+ :age_in_days => [12 => 1, 34 => 10, ...]
75
+ }
76
+ }
77
+
78
+
@@ -0,0 +1,92 @@
1
+ require "test_helper"
2
+
3
+ class ApplyAggregationTest < ActiveSupport::TestCase
4
+ context "ApplyAggregation" do
5
+ setup do
6
+ @agg = Class.new
7
+ @agg.class_eval do
8
+ def self.name; "MyAggregation" end
9
+ include Wonkavision::Aggregation
10
+ dimension :a, :b, :c
11
+ measure :d, :e
12
+ aggregate_by :a, :b
13
+ aggregate_by :a, :b, :c
14
+ store :hash_store
15
+ end
16
+ @handler = Wonkavision::Analytics::ApplyAggregation.new
17
+ end
18
+
19
+ should "initialize with the appropriate namespace" do
20
+ assert_equal Wonkavision.join("wv", "analytics"), @handler.class.event_namespace
21
+ end
22
+
23
+ context "#aggregation_for" do
24
+ should "look up an aggregation for the provided name" do
25
+ assert_equal @agg, @handler.aggregation_for(@agg.name)
26
+ end
27
+ end
28
+
29
+ context "#process_event" do
30
+ should "return false unless all appropriate metadata is present and valid" do
31
+ assert_equal false, @handler.process_event({"aggregation"=>"ack",
32
+ "action"=>"add","measures"=>{},
33
+ "dimensions"=>{}})
34
+
35
+ assert_equal false, @handler.process_event( { "aggregation"=>@agg.name,
36
+ "action"=>"add","measures"=>{}})
37
+
38
+ assert_equal false, @handler.process_event( { "aggregation"=>@agg.name,
39
+ "measures"=>{},"dimensions"=>{}})
40
+ end
41
+
42
+ context "with a valid message" do
43
+ setup do
44
+ @message = {
45
+ "aggregation" => @agg.name,
46
+ "action" => "add",
47
+ "dimensions" => { "a" => { "a" => :a }, "b" =>{ "b" => :b}, "c" => { "c" => :c } },
48
+ "measures" =>{ "d" => 1.0, "e" => 2.0 }
49
+ }
50
+
51
+ end
52
+
53
+ should "instantiate a new aggregation with the dimensions in the message" do
54
+ aggregator = @agg.new(@message["dimensions"])
55
+ @agg.expects(:new).with(@message["dimensions"]).returns(aggregator)
56
+ @handler.process_event(@message)
57
+ end
58
+ should "add measures if the action is add" do
59
+ @agg.any_instance.expects(:add).with(@message["measures"])
60
+ @handler.process_event(@message)
61
+ end
62
+ should "reject measures if the action is reject" do
63
+ @message["action"] = "reject"
64
+ @agg.any_instance.expects(:reject).with(@message["measures"])
65
+ @handler.process_event(@message)
66
+ end
67
+ should "raise an error if the action is anything other than add or reject" do
68
+ @message["action"] = "whateva"
69
+ assert_raise(RuntimeError) { @handler.process_event(@message) }
70
+ end
71
+ should "do nothing if measures is empty" do
72
+ @message["measures"] = { "d" => nil, "e" => nil}
73
+ @agg.any_instance.expects(:add).never
74
+ @handler.process_event(@message)
75
+ end
76
+
77
+
78
+ end
79
+
80
+ end
81
+
82
+
83
+ context "will listen for aggregation updated messages" do
84
+ should "respond to aggregation updated messages" do
85
+ Wonkavision::Analytics::ApplyAggregation.any_instance.expects(:process_event)
86
+ Wonkavision.event_coordinator.receive_event("wv/analytics/aggregation/updated",{ :a=>:b})
87
+ end
88
+ end
89
+
90
+ end
91
+
92
+ end
@@ -0,0 +1,26 @@
1
+ require "test_helper"
2
+
3
+ class AttributeTest < ActiveSupport::TestCase
4
+ context "Attribute" do
5
+ setup do
6
+ @attribute = Wonkavision::Plugins::Aggregation::Attribute.new(:my_attribute,:an_option=>true)
7
+ end
8
+
9
+ should "take its name from the constructor" do
10
+ assert_equal :my_attribute, @attribute.name
11
+ end
12
+
13
+ should "take its options from the constructor" do
14
+ assert_equal( { :an_option => true}, @attribute.options )
15
+ end
16
+
17
+ context "#extract" do
18
+ should "extract a value from a hash based on the name of the attribute" do
19
+ assert_equal "hi", @attribute.extract({ "my_attribute" => "hi"})
20
+ end
21
+
22
+ end
23
+
24
+
25
+ end
26
+ end
@@ -0,0 +1,200 @@
1
+ require "test_helper"
2
+ require File.join $test_dir, "test_aggregation.rb"
3
+
4
+
5
+ class CellSetTest < ActiveSupport::TestCase
6
+ Query = Wonkavision::Analytics::Query
7
+ CellSet = Wonkavision::Analytics::CellSet
8
+
9
+ context "CellSet" do
10
+ setup do
11
+ @aggregation = ::TestAggregation
12
+ test_data = File.join $test_dir, "test_data.tuples"
13
+ @test_data = eval(File.read(test_data))
14
+ @query = Wonkavision::Analytics::Query.new
15
+ @query.select :size, :shape, :on => :columns
16
+ @query.select :color, :on => :rows
17
+ @query.where :dimensions.color.ne => "black"
18
+ @cellset = CellSet.new @aggregation, @query, @test_data
19
+ end
20
+
21
+ context "Public API" do
22
+ context "#initialize" do
23
+ should "initialize axes" do
24
+ assert_equal 2, @cellset.axes.length
25
+ end
26
+
27
+ should "populate dimension members from tuples" do
28
+ @cellset.axes.each do |axis|
29
+ axis.dimensions.each do |dimension|
30
+ assert dimension.members.length > 0
31
+ end
32
+ end
33
+ end
34
+
35
+ should "populate cells from tuples" do
36
+ assert_equal @test_data.length - 2, @cellset.length #2 records filtered out (color=black)
37
+ end
38
+ end
39
+ context "#[]" do
40
+ should "locate a cell based on its coordinates, specified in query order" do
41
+ cell = @cellset[:large, :square, :red]
42
+ assert_not_nil cell
43
+ assert_equal ["large", "square", "red"], cell.key
44
+ assert_equal 10, cell.cost.count
45
+ end
46
+ end
47
+ context "#length" do
48
+ should "return the number of total tuples in the set" do
49
+ assert_equal @test_data.length - 2, @cellset.length #2 records filtered out (color = black)
50
+ end
51
+ end
52
+ end
53
+ context "Implementation" do
54
+ context "#process_tuples" do
55
+ setup do
56
+ @dims, @cells = @cellset.send(:process_tuples, @aggregation, @query, @test_data)
57
+ end
58
+ context "processed cells" do
59
+ should "contain one entry for each matching tuple" do
60
+ assert_equal @test_data.length - 2, @cells.length #2 records are black, and filtered
61
+ end
62
+ should "be keyed by a query-ordered array of dimension keys" do
63
+ test_key = @cells.keys.find { |key|key - ["red", "square", "large"] == []}
64
+ assert_equal ["large", "square", "red"], test_key
65
+ end
66
+ end
67
+ context "processed dimension members" do
68
+ should "contain one entry for each dimension" do
69
+ assert_equal 3, @dims.length
70
+ @query.selected_dimensions.each { |dim| assert @dims.keys.include?(dim.to_s)}
71
+ end
72
+ should "provide a hash of members for the dimensions" do
73
+ test_dim = @dims["color"]
74
+ %w(red green yellow white).each do |mem_key| #black is filtered out
75
+ assert test_dim.keys.include?(mem_key)
76
+ end
77
+ end
78
+ should "include the dimension attributes in the member hash" do
79
+ test_dim = @dims["color"]
80
+ assert_equal( { "color" => "red" }, test_dim["red"] )
81
+ end
82
+ end
83
+ end
84
+ context "#key_for" do
85
+ setup do
86
+ @record = {
87
+ "dimension_keys"=>["yellow", "square", "small"],
88
+ "dimension_names"=>["color", "shape", "size"] }
89
+ end
90
+ should "re-order the dimension_keys array to match query order" do
91
+ assert_equal ["small", "square", "yellow"], @cellset.send(:key_for,@query,@record)
92
+ end
93
+ end
94
+ context "Support Classes" do
95
+ context "Axis" do
96
+ setup { @axis = @cellset.columns }
97
+ context "#initialize" do
98
+ should "initialize a Dimension object for each dimension" do
99
+ assert_equal 2, @axis.dimensions.length
100
+ end
101
+ should "order dimensions in query order" do
102
+ assert_equal "size", @axis.dimensions[0].name
103
+ assert_equal "shape", @axis.dimensions[1].name
104
+ end
105
+ end
106
+ end
107
+ context "Dimension" do
108
+ setup { @dimension = @cellset.columns.dimensions[0] }
109
+ context "#initialize" do
110
+ should "should return #name" do
111
+ assert_equal "size", @dimension.name
112
+ end
113
+ should "extract its definition from the aggregation" do
114
+ assert_equal @aggregation.dimensions["size"], @dimension.definition
115
+ end
116
+ should "should contain a sorted list of members" do
117
+ %w(large medium small).each_with_index do |size,idx|
118
+ assert_equal size, @dimension.members[idx].key
119
+ end
120
+ end
121
+ end
122
+ end
123
+ context "Member" do
124
+ setup { @member = @cellset.columns.dimensions[0].members[0]}
125
+ should "maintain a reference to its parent dimension" do
126
+ assert_equal @cellset.columns.dimensions[0], @member.dimension
127
+ end
128
+ should "provide named access to the main dimension attributes" do
129
+ assert_equal "large", @member.caption
130
+ assert_equal "large", @member.key
131
+ assert_equal "large", @member.sort
132
+ end
133
+ should "provide access to the raw attribute hash" do
134
+ assert_equal( { "size" => "large"}, @member.attributes )
135
+ end
136
+ end
137
+ context "Cell" do
138
+ setup { @cell = @cellset[:large, :square, :red] }
139
+ should "provide access to the cell key" do
140
+ assert_equal ["large", "square", "red"], @cell.key
141
+ end
142
+ should "include a hash of measures" do
143
+ %w(cost weight).each { |measure| assert @cell.measures.keys.include?(measure)}
144
+ end
145
+ should "provide named access to each measure" do
146
+ assert_equal @cell.measures["cost"], @cell.cost
147
+ assert_equal @cell.measures["weight"], @cell.weight
148
+ end
149
+ context "#aggregate" do
150
+ setup do
151
+ @cell.aggregate({"cost"=>{ "count"=>1,"sum"=>1,"sum2"=>2},
152
+ "different"=>{ "count"=>2,"sum"=>2,"sum2"=>8}})
153
+
154
+ end
155
+ should "insert any new measures" do
156
+ assert @cell.measures.keys.include?("different")
157
+ end
158
+ should "aggregate data from an existing measure" do
159
+ assert_equal 11, @cell.cost.count
160
+ assert_equal 51, @cell.cost.sum
161
+ assert_equal 252, @cell.cost.sum2
162
+ end
163
+
164
+ end
165
+ end
166
+
167
+ context "Measure" do
168
+ setup { @measure = @cellset[:large, :square, :red].cost }
169
+ should "provide access to its name" do
170
+ assert_equal "cost", @measure.name
171
+ end
172
+ should "provide access to the measure hash" do
173
+ assert_equal( {"count"=>10, "sum"=>50, "sum2"=>250}, @measure.data )
174
+ end
175
+ should "provide named hash to measure values" do
176
+ assert_equal 10, @measure.count
177
+ assert_equal 50, @measure.sum
178
+ assert_equal 250, @measure.sum2
179
+ end
180
+ should "calculate an average" do
181
+ assert_equal 5, @measure.average
182
+ end
183
+ context "#aggregate" do
184
+ setup do
185
+ @measure.aggregate(@measure.data.dup)
186
+ end
187
+ should "add sum, sum2 and count to the existing values" do
188
+ assert_equal 20, @measure.count
189
+ assert_equal 100, @measure.sum
190
+ assert_equal 500, @measure.sum2
191
+ end
192
+
193
+ end
194
+
195
+ end
196
+
197
+ end
198
+ end
199
+ end
200
+ end