wonkavision 0.5.11 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.rdoc +3 -0
- data/lib/wonkavision.rb +28 -1
- data/lib/wonkavision/aggregation.rb +21 -0
- data/lib/wonkavision/event_coordinator.rb +19 -7
- data/lib/wonkavision/extensions/symbol.rb +55 -0
- data/lib/wonkavision/facts.rb +27 -0
- data/lib/wonkavision/local_job_queue.rb +28 -0
- data/lib/wonkavision/message_mapper.rb +2 -2
- data/lib/wonkavision/message_mapper/map.rb +60 -8
- data/lib/wonkavision/persistence/mongo.rb +95 -0
- data/lib/wonkavision/plugins.rb +2 -1
- data/lib/wonkavision/plugins/analytics/aggregation.rb +139 -0
- data/lib/wonkavision/plugins/analytics/aggregation/aggregation_spec.rb +53 -0
- data/lib/wonkavision/plugins/analytics/aggregation/attribute.rb +22 -0
- data/lib/wonkavision/plugins/analytics/aggregation/dimension.rb +64 -0
- data/lib/wonkavision/plugins/analytics/aggregation/measure.rb +240 -0
- data/lib/wonkavision/plugins/analytics/cellset.rb +171 -0
- data/lib/wonkavision/plugins/analytics/facts.rb +106 -0
- data/lib/wonkavision/plugins/analytics/handlers/apply_aggregation.rb +35 -0
- data/lib/wonkavision/plugins/analytics/handlers/split_by_aggregation.rb +60 -0
- data/lib/wonkavision/plugins/analytics/member_filter.rb +106 -0
- data/lib/wonkavision/plugins/analytics/mongo.rb +6 -0
- data/lib/wonkavision/plugins/analytics/persistence/hash_store.rb +59 -0
- data/lib/wonkavision/plugins/analytics/persistence/mongo_store.rb +85 -0
- data/lib/wonkavision/plugins/analytics/persistence/store.rb +105 -0
- data/lib/wonkavision/plugins/analytics/query.rb +76 -0
- data/lib/wonkavision/plugins/event_handling.rb +15 -3
- data/lib/wonkavision/version.rb +1 -1
- data/test/aggregation_spec_test.rb +99 -0
- data/test/aggregation_test.rb +170 -0
- data/test/analytics/test_aggregation.rb +78 -0
- data/test/apply_aggregation_test.rb +92 -0
- data/test/attribute_test.rb +26 -0
- data/test/cellset_test.rb +200 -0
- data/test/dimension_test.rb +186 -0
- data/test/facts_test.rb +146 -0
- data/test/hash_store_test.rb +112 -0
- data/test/log/test.log +96844 -0
- data/test/map_test.rb +48 -1
- data/test/measure_test.rb +146 -0
- data/test/member_filter_test.rb +143 -0
- data/test/mongo_store_test.rb +115 -0
- data/test/query_test.rb +106 -0
- data/test/split_by_aggregation_test.rb +114 -0
- data/test/store_test.rb +71 -0
- data/test/symbol_test.rb +62 -0
- data/test/test_activity_models.rb +1 -1
- data/test/test_aggregation.rb +42 -0
- data/test/test_data.tuples +100 -0
- data/test/test_helper.rb +7 -0
- metadata +57 -5
@@ -0,0 +1,59 @@
|
|
1
|
+
module Wonkavision
|
2
|
+
module Analytics
|
3
|
+
module Persistence
|
4
|
+
class HashStore < Store
|
5
|
+
|
6
|
+
attr_reader :storage
|
7
|
+
def initialize(facts, storage = HashWithIndifferentAccess.new)
|
8
|
+
super(facts)
|
9
|
+
@storage = storage
|
10
|
+
end
|
11
|
+
|
12
|
+
def aggregations
|
13
|
+
@storage[:aggregations] ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
protected
|
17
|
+
#Fact persistence
|
18
|
+
def update_facts_record(record_id, data)
|
19
|
+
previous_facts = @storage[record_id]
|
20
|
+
current_facts = @storage[record_id] = (previous_facts || {}).merge(data)
|
21
|
+
[previous_facts, current_facts]
|
22
|
+
end
|
23
|
+
|
24
|
+
def insert_facts_record(record_id, data)
|
25
|
+
@storage[record_id] = data
|
26
|
+
end
|
27
|
+
|
28
|
+
def delete_facts_record(record_id, data)
|
29
|
+
@storage.delete(record_id)
|
30
|
+
end
|
31
|
+
|
32
|
+
#Aggregation persistence
|
33
|
+
def fetch_tuples(dimension_names = [], filters = [])
|
34
|
+
return aggregations.values if dimension_names.blank?
|
35
|
+
tuples = []
|
36
|
+
aggregations.each_pair do |agg_key, agg|
|
37
|
+
tuples << agg if
|
38
|
+
agg_key[:dimension_names] == dimension_names
|
39
|
+
end
|
40
|
+
tuples
|
41
|
+
end
|
42
|
+
|
43
|
+
def update_tuple(data)
|
44
|
+
key = aggregation_key(data)
|
45
|
+
agg = aggregations[key]
|
46
|
+
if agg
|
47
|
+
data[:measures].keys.each do |measure_key|
|
48
|
+
agg[:measures][measure_key] ||= 0
|
49
|
+
agg[:measures][measure_key] += data[:measures][measure_key]
|
50
|
+
end
|
51
|
+
else
|
52
|
+
aggregations[key] = data.dup
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
module Wonkavision
|
2
|
+
module Analytics
|
3
|
+
module Persistence
|
4
|
+
class MongoStore < Store
|
5
|
+
|
6
|
+
def initialize(facts)
|
7
|
+
super(facts)
|
8
|
+
end
|
9
|
+
|
10
|
+
def facts_collection_name
|
11
|
+
"wv.#{owner.name.underscore.gsub("::",".")}.facts"
|
12
|
+
end
|
13
|
+
|
14
|
+
def facts_collection
|
15
|
+
Wonkavision::Mongo.database[facts_collection_name]
|
16
|
+
end
|
17
|
+
|
18
|
+
def aggregations_collection_name
|
19
|
+
"wv.#{owner.name.underscore.gsub("::",".")}.aggregations"
|
20
|
+
end
|
21
|
+
|
22
|
+
def aggregations_collection
|
23
|
+
Wonkavision::Mongo.database[aggregations_collection_name]
|
24
|
+
end
|
25
|
+
|
26
|
+
def[](document_id)
|
27
|
+
facts_collection.find({ :_id => document_id}).to_a.pop
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
protected
|
32
|
+
#Fact persistence
|
33
|
+
def update_facts_record(record_id, data)
|
34
|
+
query = { :_id => record_id }
|
35
|
+
update = { "$set" => data }
|
36
|
+
previous_facts = facts_collection.find_and_modify :query=>query, :update=>update, :upsert=>true
|
37
|
+
current_facts = (previous_facts || {}).merge(data)
|
38
|
+
remove_mongo_id(previous_facts, current_facts)
|
39
|
+
end
|
40
|
+
|
41
|
+
def insert_facts_record(record_id, data)
|
42
|
+
query = { :_id => record_id }
|
43
|
+
facts_collection.update(query, data.merge(:_id=>record_id), :upsert=>true)
|
44
|
+
data
|
45
|
+
end
|
46
|
+
|
47
|
+
def delete_facts_record(record_id, data)
|
48
|
+
query = { :_id => record_id }
|
49
|
+
remove_mongo_id(facts_collection.find_and_modify(:query=>query, :remove=>true))
|
50
|
+
end
|
51
|
+
|
52
|
+
#Aggregation persistence
|
53
|
+
def fetch_tuples(dimension_names, filters)
|
54
|
+
criteria = dimension_names.blank? ? {} : { :dimension_names => dimension_names }
|
55
|
+
append_filters(criteria,filters)
|
56
|
+
aggregations_collection.find(criteria).to_a
|
57
|
+
end
|
58
|
+
|
59
|
+
def update_tuple(data)
|
60
|
+
aggregations_collection.update( aggregation_key(data),
|
61
|
+
{"$inc" => data[:measures],
|
62
|
+
"$set" => { :dimensions=>data[:dimensions]}},
|
63
|
+
:upsert => true, :safe => true )
|
64
|
+
end
|
65
|
+
|
66
|
+
def remove_mongo_id(*documents)
|
67
|
+
unless owner.respond_to?(:record_id) && owner.record_id.to_s == "_id"
|
68
|
+
documents.each { |doc| doc.delete("_id")}
|
69
|
+
end
|
70
|
+
documents.length > 1 ? documents : documents.pop
|
71
|
+
end
|
72
|
+
|
73
|
+
private
|
74
|
+
def append_filters(criteria,filters)
|
75
|
+
filters.each do |filter|
|
76
|
+
filter_key = "#{filter.member_type}s.#{filter.name}.#{filter.attribute_key(owner)}"
|
77
|
+
criteria[filter_key] = filter.operator == :eq ? filter.value :
|
78
|
+
{ "$#{filter.operator}" => filter.value}
|
79
|
+
filter.applied!
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
module Wonkavision
|
2
|
+
module Analytics
|
3
|
+
module Persistence
|
4
|
+
class Store
|
5
|
+
|
6
|
+
def self.[](store_name)
|
7
|
+
@stores ||= {}
|
8
|
+
@stores[store_name.to_s]
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.[]=(store_name,store)
|
12
|
+
@stores ||= {}
|
13
|
+
@stores[store_name.to_s] = store
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.inherited(store)
|
17
|
+
store_name = store.name.split("::").pop.underscore
|
18
|
+
self[store_name] = store
|
19
|
+
end
|
20
|
+
|
21
|
+
attr_reader :owner
|
22
|
+
def initialize(owner)
|
23
|
+
@owner = owner
|
24
|
+
end
|
25
|
+
|
26
|
+
#Facts persistence support
|
27
|
+
#
|
28
|
+
# returns a two element array, the first element
|
29
|
+
# containing the prior state of the facts record,
|
30
|
+
# the second element containing the current state
|
31
|
+
# of the facts record
|
32
|
+
def update_facts(data)
|
33
|
+
record_id = assert_record_id(data)
|
34
|
+
update_facts_record record_id, data
|
35
|
+
end
|
36
|
+
|
37
|
+
#returns the current value of the facts record
|
38
|
+
def add_facts(data)
|
39
|
+
record_id = assert_record_id(data)
|
40
|
+
insert_facts_record record_id, data
|
41
|
+
end
|
42
|
+
|
43
|
+
#returns the previous value of the facts record
|
44
|
+
def remove_facts(data)
|
45
|
+
record_id = assert_record_id(data)
|
46
|
+
delete_facts_record record_id, data
|
47
|
+
end
|
48
|
+
|
49
|
+
#Aggregations persistence support
|
50
|
+
#
|
51
|
+
# Takes a Wonkavision::Analytics::Query and returns an array of
|
52
|
+
# matching tuples
|
53
|
+
def execute_query(query)
|
54
|
+
dimension_names = query.all_dimensions? ? [] :
|
55
|
+
query.referenced_dimensions.sort{ |a,b| a.to_s <=> b.to_s }
|
56
|
+
|
57
|
+
fetch_tuples(dimension_names, query.filters)
|
58
|
+
|
59
|
+
end
|
60
|
+
|
61
|
+
def update_aggregation(aggregation_data)
|
62
|
+
update_tuple(aggregation_data)
|
63
|
+
end
|
64
|
+
|
65
|
+
protected
|
66
|
+
|
67
|
+
def assert_record_id(data)
|
68
|
+
raise "The storage owner does not implement a 'record_id' method. (#{owner.inspect})" unless owner.respond_to?(:record_id)
|
69
|
+
|
70
|
+
data[owner.record_id.to_s].tap do |id|
|
71
|
+
raise "A record_id is required to update the analytics store" unless id
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def aggregation_key(aggregation_data)
|
76
|
+
{
|
77
|
+
:dimension_keys => aggregation_data[:dimension_keys],
|
78
|
+
:dimension_names => aggregation_data[:dimension_names]
|
79
|
+
}
|
80
|
+
end
|
81
|
+
#Abstract methods
|
82
|
+
def update_facts_record(record_id, data)
|
83
|
+
raise NotImplementedError
|
84
|
+
end
|
85
|
+
|
86
|
+
def insert_facts_record(record_id, data)
|
87
|
+
raise NotImplementedError
|
88
|
+
end
|
89
|
+
|
90
|
+
def fetch_tuples(dimension_names, filters = [])
|
91
|
+
raise NotImplementedError
|
92
|
+
end
|
93
|
+
|
94
|
+
def update_tuple(aggregation_data)
|
95
|
+
raise NotImplementedError
|
96
|
+
end
|
97
|
+
|
98
|
+
def delete_facts_record(record_id, data)
|
99
|
+
raise NotImplementedError
|
100
|
+
end
|
101
|
+
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module Wonkavision
|
2
|
+
module Analytics
|
3
|
+
class Query
|
4
|
+
attr_reader :axes, :filters
|
5
|
+
|
6
|
+
def initialize()
|
7
|
+
@axes = []
|
8
|
+
@slicer = Set.new
|
9
|
+
@filters = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def select(*dimensions)
|
13
|
+
options = dimensions.extract_options!
|
14
|
+
axis = options[:axis] || options[:on]
|
15
|
+
axis_ordinal = self.class.axis_ordinal(axis)
|
16
|
+
@axes[axis_ordinal] = dimensions
|
17
|
+
self
|
18
|
+
end
|
19
|
+
|
20
|
+
def where(criteria_hash = {})
|
21
|
+
criteria_hash.each_pair do |filter,value|
|
22
|
+
member_filter = filter.kind_of?(MemberFilter) ? filter :
|
23
|
+
MemberFilter.new(filter)
|
24
|
+
member_filter.value = value
|
25
|
+
@filters << member_filter
|
26
|
+
@slicer << member_filter.name if member_filter.dimension?
|
27
|
+
end
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def slicer
|
32
|
+
dims = selected_dimensions
|
33
|
+
@slicer.reject{|m|dims.include?(m)}
|
34
|
+
end
|
35
|
+
|
36
|
+
def referenced_dimensions
|
37
|
+
( [] + selected_dimensions + slicer ).compact
|
38
|
+
end
|
39
|
+
|
40
|
+
def selected_dimensions
|
41
|
+
dimensions = []
|
42
|
+
axes.each { |dims|dimensions.concat(dims) unless dims.blank? }
|
43
|
+
dimensions.uniq.compact
|
44
|
+
end
|
45
|
+
|
46
|
+
def all_dimensions?
|
47
|
+
axes.empty?
|
48
|
+
end
|
49
|
+
|
50
|
+
def matches_filter?(aggregation, tuple)
|
51
|
+
return true if all_filters_applied?
|
52
|
+
!( filters.detect{ |filter| !filter.matches(aggregation, tuple) } )
|
53
|
+
end
|
54
|
+
|
55
|
+
def all_filters_applied?
|
56
|
+
@all_filters_applied ||= !(filters.detect{ |filter| !filter.applied? })
|
57
|
+
end
|
58
|
+
|
59
|
+
def validate!
|
60
|
+
axes.each_with_index{|axis,index|raise "Axes must be selected from in consecutive order and contain at least one dimension. Axis #{index} is blank." if axis.blank?}
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.axis_ordinal(axis_def)
|
64
|
+
case axis_def.to_s.strip.downcase.to_s
|
65
|
+
when "columns" then 0
|
66
|
+
when "rows" then 1
|
67
|
+
when "pages" then 2
|
68
|
+
when "chapters" then 3
|
69
|
+
when "sections" then 4
|
70
|
+
else axis_def.to_i
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -69,7 +69,7 @@ module Wonkavision
|
|
69
69
|
def event_context
|
70
70
|
@wonkavision_event_context
|
71
71
|
end
|
72
|
-
|
72
|
+
|
73
73
|
def handle_event
|
74
74
|
ctx = @wonkavision_event_context
|
75
75
|
ctx.data = map(ctx.data,ctx.path)
|
@@ -97,13 +97,25 @@ module Wonkavision
|
|
97
97
|
|
98
98
|
def map?(condition,data,path)
|
99
99
|
return true unless condition && condition.to_s != 'all' && condition.to_s != '*'
|
100
|
-
|
101
|
-
if
|
100
|
+
|
101
|
+
if condition.is_a?(Regexp)
|
102
|
+
path =~ condition
|
103
|
+
elsif condition.is_a?(String)
|
104
|
+
path.downcase == condition.downcase
|
105
|
+
elsif condition.is_a?(Proc)
|
102
106
|
return condition.call if condition.arity <= 0
|
103
107
|
return condition.call(path) if condition.arity == 1
|
104
108
|
return condition.call(path,data)
|
105
109
|
end
|
106
110
|
end
|
111
|
+
|
112
|
+
def broadcast(event_name, event)
|
113
|
+
Wonkavision.event_coordinator.publish(event_name, event)
|
114
|
+
end
|
115
|
+
|
116
|
+
def submit(event_name, event)
|
117
|
+
Wonkavision.event_coordinator.submit_job(event_name, event)
|
118
|
+
end
|
107
119
|
end
|
108
120
|
|
109
121
|
end
|
data/lib/wonkavision/version.rb
CHANGED
@@ -0,0 +1,99 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
|
3
|
+
class AggregationSpecTest < ActiveSupport::TestCase
|
4
|
+
context "AggregationSpec" do
|
5
|
+
setup do
|
6
|
+
@aggregation_spec = Wonkavision::Plugins::Aggregation::AggregationSpec.new("MyAggregation")
|
7
|
+
end
|
8
|
+
|
9
|
+
should "take its name from the constructor" do
|
10
|
+
assert_equal "MyAggregation", @aggregation_spec.name
|
11
|
+
end
|
12
|
+
|
13
|
+
context "#dimension" do
|
14
|
+
setup do
|
15
|
+
@aggregation_spec.dimension :a, :b, :c=>"d"
|
16
|
+
end
|
17
|
+
|
18
|
+
should "add a dimension for each provided value" do
|
19
|
+
assert_equal 2, @aggregation_spec.dimensions.length
|
20
|
+
["a","b"].each { |dim| assert @aggregation_spec.dimensions.keys.include?(dim)}
|
21
|
+
end
|
22
|
+
|
23
|
+
should "store pass the options to each dimension" do
|
24
|
+
@aggregation_spec.dimensions.values.each { |dim| assert_equal({:c=>"d"},dim.options)}
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context "#measure" do
|
29
|
+
setup do
|
30
|
+
@aggregation_spec.measure :c, :d, :e => "f"
|
31
|
+
end
|
32
|
+
|
33
|
+
should "add a measure for each provided value" do
|
34
|
+
assert_equal 2, @aggregation_spec.measures.length
|
35
|
+
end
|
36
|
+
|
37
|
+
should "add the attributes as hash elements" do
|
38
|
+
[:c, :d].each { |measure| assert @aggregation_spec.measures[measure]}
|
39
|
+
end
|
40
|
+
|
41
|
+
should "store the options to each measure" do
|
42
|
+
@aggregation_spec.measures.each_pair { |k,v| assert_equal({ "e" => "f"}, v)}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
context "#aggregate_by" do
|
47
|
+
setup do
|
48
|
+
@aggregation_spec.aggregate_by :a, :b
|
49
|
+
@aggregation_spec.aggregate_by :a, :c
|
50
|
+
@aggregation_spec.aggregate_by [:a, :c, :d]
|
51
|
+
end
|
52
|
+
|
53
|
+
should "append the list of aggregations to the aggregations collection" do
|
54
|
+
assert_equal 3, @aggregation_spec.aggregations.length
|
55
|
+
end
|
56
|
+
|
57
|
+
should "append the list of aggregations as a simple array" do
|
58
|
+
assert_equal [:a, :b], @aggregation_spec.aggregations[0]
|
59
|
+
end
|
60
|
+
|
61
|
+
should "flatten arrays of attributes" do
|
62
|
+
assert_equal [:a, :c, :d], @aggregation_spec.aggregations[-1]
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context "#aggregate_all_combinations" do
|
67
|
+
setup do
|
68
|
+
@aggregation_spec.dimension :a,:b,:c
|
69
|
+
@aggregation_spec.aggregate_all_combinations
|
70
|
+
end
|
71
|
+
should "determine all possible combinations of dimensions" do
|
72
|
+
assert_equal 7, @aggregation_spec.aggregations.length
|
73
|
+
combinations = [["a"], ["b"], ["c"], ["a", "b"], ["a", "c"], ["b", "c"], ["a", "b", "c"]]
|
74
|
+
assert_equal combinations, @aggregation_spec.aggregations
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
context "#filter" do
|
79
|
+
setup do
|
80
|
+
@aggregation_spec.filter { |msg|msg["a"] == "b"}
|
81
|
+
end
|
82
|
+
should "register a filter block with the spec" do
|
83
|
+
assert_not_nil @aggregation_spec.filter
|
84
|
+
end
|
85
|
+
end
|
86
|
+
context "#matches" do
|
87
|
+
setup do
|
88
|
+
@aggregation_spec.filter { |msg| msg["a"] == "b"}
|
89
|
+
end
|
90
|
+
should "return true if a message matches the filter" do
|
91
|
+
assert @aggregation_spec.matches({ "a" => "b"})
|
92
|
+
end
|
93
|
+
should "return false if the message does not match the filter" do
|
94
|
+
assert !@aggregation_spec.matches({ "a" => "B"})
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
end
|