wonkavision 0.5.11 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/CHANGELOG.rdoc +3 -0
  2. data/lib/wonkavision.rb +28 -1
  3. data/lib/wonkavision/aggregation.rb +21 -0
  4. data/lib/wonkavision/event_coordinator.rb +19 -7
  5. data/lib/wonkavision/extensions/symbol.rb +55 -0
  6. data/lib/wonkavision/facts.rb +27 -0
  7. data/lib/wonkavision/local_job_queue.rb +28 -0
  8. data/lib/wonkavision/message_mapper.rb +2 -2
  9. data/lib/wonkavision/message_mapper/map.rb +60 -8
  10. data/lib/wonkavision/persistence/mongo.rb +95 -0
  11. data/lib/wonkavision/plugins.rb +2 -1
  12. data/lib/wonkavision/plugins/analytics/aggregation.rb +139 -0
  13. data/lib/wonkavision/plugins/analytics/aggregation/aggregation_spec.rb +53 -0
  14. data/lib/wonkavision/plugins/analytics/aggregation/attribute.rb +22 -0
  15. data/lib/wonkavision/plugins/analytics/aggregation/dimension.rb +64 -0
  16. data/lib/wonkavision/plugins/analytics/aggregation/measure.rb +240 -0
  17. data/lib/wonkavision/plugins/analytics/cellset.rb +171 -0
  18. data/lib/wonkavision/plugins/analytics/facts.rb +106 -0
  19. data/lib/wonkavision/plugins/analytics/handlers/apply_aggregation.rb +35 -0
  20. data/lib/wonkavision/plugins/analytics/handlers/split_by_aggregation.rb +60 -0
  21. data/lib/wonkavision/plugins/analytics/member_filter.rb +106 -0
  22. data/lib/wonkavision/plugins/analytics/mongo.rb +6 -0
  23. data/lib/wonkavision/plugins/analytics/persistence/hash_store.rb +59 -0
  24. data/lib/wonkavision/plugins/analytics/persistence/mongo_store.rb +85 -0
  25. data/lib/wonkavision/plugins/analytics/persistence/store.rb +105 -0
  26. data/lib/wonkavision/plugins/analytics/query.rb +76 -0
  27. data/lib/wonkavision/plugins/event_handling.rb +15 -3
  28. data/lib/wonkavision/version.rb +1 -1
  29. data/test/aggregation_spec_test.rb +99 -0
  30. data/test/aggregation_test.rb +170 -0
  31. data/test/analytics/test_aggregation.rb +78 -0
  32. data/test/apply_aggregation_test.rb +92 -0
  33. data/test/attribute_test.rb +26 -0
  34. data/test/cellset_test.rb +200 -0
  35. data/test/dimension_test.rb +186 -0
  36. data/test/facts_test.rb +146 -0
  37. data/test/hash_store_test.rb +112 -0
  38. data/test/log/test.log +96844 -0
  39. data/test/map_test.rb +48 -1
  40. data/test/measure_test.rb +146 -0
  41. data/test/member_filter_test.rb +143 -0
  42. data/test/mongo_store_test.rb +115 -0
  43. data/test/query_test.rb +106 -0
  44. data/test/split_by_aggregation_test.rb +114 -0
  45. data/test/store_test.rb +71 -0
  46. data/test/symbol_test.rb +62 -0
  47. data/test/test_activity_models.rb +1 -1
  48. data/test/test_aggregation.rb +42 -0
  49. data/test/test_data.tuples +100 -0
  50. data/test/test_helper.rb +7 -0
  51. metadata +57 -5
@@ -0,0 +1,6 @@
1
+ dir = File.dirname(__FILE__)
2
+ [
3
+ '../../persistence/mongo',
4
+ 'persistence/mongo_store.rb'
5
+
6
+ ].each {|lib|require File.join(dir,lib)}
@@ -0,0 +1,59 @@
1
+ module Wonkavision
2
+ module Analytics
3
+ module Persistence
4
+ class HashStore < Store
5
+
6
+ attr_reader :storage
7
+ def initialize(facts, storage = HashWithIndifferentAccess.new)
8
+ super(facts)
9
+ @storage = storage
10
+ end
11
+
12
+ def aggregations
13
+ @storage[:aggregations] ||= {}
14
+ end
15
+
16
+ protected
17
+ #Fact persistence
18
+ def update_facts_record(record_id, data)
19
+ previous_facts = @storage[record_id]
20
+ current_facts = @storage[record_id] = (previous_facts || {}).merge(data)
21
+ [previous_facts, current_facts]
22
+ end
23
+
24
+ def insert_facts_record(record_id, data)
25
+ @storage[record_id] = data
26
+ end
27
+
28
+ def delete_facts_record(record_id, data)
29
+ @storage.delete(record_id)
30
+ end
31
+
32
+ #Aggregation persistence
33
+ def fetch_tuples(dimension_names = [], filters = [])
34
+ return aggregations.values if dimension_names.blank?
35
+ tuples = []
36
+ aggregations.each_pair do |agg_key, agg|
37
+ tuples << agg if
38
+ agg_key[:dimension_names] == dimension_names
39
+ end
40
+ tuples
41
+ end
42
+
43
+ def update_tuple(data)
44
+ key = aggregation_key(data)
45
+ agg = aggregations[key]
46
+ if agg
47
+ data[:measures].keys.each do |measure_key|
48
+ agg[:measures][measure_key] ||= 0
49
+ agg[:measures][measure_key] += data[:measures][measure_key]
50
+ end
51
+ else
52
+ aggregations[key] = data.dup
53
+ end
54
+ end
55
+
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,85 @@
1
+ module Wonkavision
2
+ module Analytics
3
+ module Persistence
4
+ class MongoStore < Store
5
+
6
+ def initialize(facts)
7
+ super(facts)
8
+ end
9
+
10
+ def facts_collection_name
11
+ "wv.#{owner.name.underscore.gsub("::",".")}.facts"
12
+ end
13
+
14
+ def facts_collection
15
+ Wonkavision::Mongo.database[facts_collection_name]
16
+ end
17
+
18
+ def aggregations_collection_name
19
+ "wv.#{owner.name.underscore.gsub("::",".")}.aggregations"
20
+ end
21
+
22
+ def aggregations_collection
23
+ Wonkavision::Mongo.database[aggregations_collection_name]
24
+ end
25
+
26
+ def[](document_id)
27
+ facts_collection.find({ :_id => document_id}).to_a.pop
28
+ end
29
+
30
+
31
+ protected
32
+ #Fact persistence
33
+ def update_facts_record(record_id, data)
34
+ query = { :_id => record_id }
35
+ update = { "$set" => data }
36
+ previous_facts = facts_collection.find_and_modify :query=>query, :update=>update, :upsert=>true
37
+ current_facts = (previous_facts || {}).merge(data)
38
+ remove_mongo_id(previous_facts, current_facts)
39
+ end
40
+
41
+ def insert_facts_record(record_id, data)
42
+ query = { :_id => record_id }
43
+ facts_collection.update(query, data.merge(:_id=>record_id), :upsert=>true)
44
+ data
45
+ end
46
+
47
+ def delete_facts_record(record_id, data)
48
+ query = { :_id => record_id }
49
+ remove_mongo_id(facts_collection.find_and_modify(:query=>query, :remove=>true))
50
+ end
51
+
52
+ #Aggregation persistence
53
+ def fetch_tuples(dimension_names, filters)
54
+ criteria = dimension_names.blank? ? {} : { :dimension_names => dimension_names }
55
+ append_filters(criteria,filters)
56
+ aggregations_collection.find(criteria).to_a
57
+ end
58
+
59
+ def update_tuple(data)
60
+ aggregations_collection.update( aggregation_key(data),
61
+ {"$inc" => data[:measures],
62
+ "$set" => { :dimensions=>data[:dimensions]}},
63
+ :upsert => true, :safe => true )
64
+ end
65
+
66
+ def remove_mongo_id(*documents)
67
+ unless owner.respond_to?(:record_id) && owner.record_id.to_s == "_id"
68
+ documents.each { |doc| doc.delete("_id")}
69
+ end
70
+ documents.length > 1 ? documents : documents.pop
71
+ end
72
+
73
+ private
74
+ def append_filters(criteria,filters)
75
+ filters.each do |filter|
76
+ filter_key = "#{filter.member_type}s.#{filter.name}.#{filter.attribute_key(owner)}"
77
+ criteria[filter_key] = filter.operator == :eq ? filter.value :
78
+ { "$#{filter.operator}" => filter.value}
79
+ filter.applied!
80
+ end
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
@@ -0,0 +1,105 @@
1
+ module Wonkavision
2
+ module Analytics
3
+ module Persistence
4
+ class Store
5
+
6
+ def self.[](store_name)
7
+ @stores ||= {}
8
+ @stores[store_name.to_s]
9
+ end
10
+
11
+ def self.[]=(store_name,store)
12
+ @stores ||= {}
13
+ @stores[store_name.to_s] = store
14
+ end
15
+
16
+ def self.inherited(store)
17
+ store_name = store.name.split("::").pop.underscore
18
+ self[store_name] = store
19
+ end
20
+
21
+ attr_reader :owner
22
+ def initialize(owner)
23
+ @owner = owner
24
+ end
25
+
26
+ #Facts persistence support
27
+ #
28
+ # returns a two element array, the first element
29
+ # containing the prior state of the facts record,
30
+ # the second element containing the current state
31
+ # of the facts record
32
+ def update_facts(data)
33
+ record_id = assert_record_id(data)
34
+ update_facts_record record_id, data
35
+ end
36
+
37
+ #returns the current value of the facts record
38
+ def add_facts(data)
39
+ record_id = assert_record_id(data)
40
+ insert_facts_record record_id, data
41
+ end
42
+
43
+ #returns the previous value of the facts record
44
+ def remove_facts(data)
45
+ record_id = assert_record_id(data)
46
+ delete_facts_record record_id, data
47
+ end
48
+
49
+ #Aggregations persistence support
50
+ #
51
+ # Takes a Wonkavision::Analytics::Query and returns an array of
52
+ # matching tuples
53
+ def execute_query(query)
54
+ dimension_names = query.all_dimensions? ? [] :
55
+ query.referenced_dimensions.sort{ |a,b| a.to_s <=> b.to_s }
56
+
57
+ fetch_tuples(dimension_names, query.filters)
58
+
59
+ end
60
+
61
+ def update_aggregation(aggregation_data)
62
+ update_tuple(aggregation_data)
63
+ end
64
+
65
+ protected
66
+
67
+ def assert_record_id(data)
68
+ raise "The storage owner does not implement a 'record_id' method. (#{owner.inspect})" unless owner.respond_to?(:record_id)
69
+
70
+ data[owner.record_id.to_s].tap do |id|
71
+ raise "A record_id is required to update the analytics store" unless id
72
+ end
73
+ end
74
+
75
+ def aggregation_key(aggregation_data)
76
+ {
77
+ :dimension_keys => aggregation_data[:dimension_keys],
78
+ :dimension_names => aggregation_data[:dimension_names]
79
+ }
80
+ end
81
+ #Abstract methods
82
+ def update_facts_record(record_id, data)
83
+ raise NotImplementedError
84
+ end
85
+
86
+ def insert_facts_record(record_id, data)
87
+ raise NotImplementedError
88
+ end
89
+
90
+ def fetch_tuples(dimension_names, filters = [])
91
+ raise NotImplementedError
92
+ end
93
+
94
+ def update_tuple(aggregation_data)
95
+ raise NotImplementedError
96
+ end
97
+
98
+ def delete_facts_record(record_id, data)
99
+ raise NotImplementedError
100
+ end
101
+
102
+ end
103
+ end
104
+ end
105
+ end
@@ -0,0 +1,76 @@
1
+ module Wonkavision
2
+ module Analytics
3
+ class Query
4
+ attr_reader :axes, :filters
5
+
6
+ def initialize()
7
+ @axes = []
8
+ @slicer = Set.new
9
+ @filters = []
10
+ end
11
+
12
+ def select(*dimensions)
13
+ options = dimensions.extract_options!
14
+ axis = options[:axis] || options[:on]
15
+ axis_ordinal = self.class.axis_ordinal(axis)
16
+ @axes[axis_ordinal] = dimensions
17
+ self
18
+ end
19
+
20
+ def where(criteria_hash = {})
21
+ criteria_hash.each_pair do |filter,value|
22
+ member_filter = filter.kind_of?(MemberFilter) ? filter :
23
+ MemberFilter.new(filter)
24
+ member_filter.value = value
25
+ @filters << member_filter
26
+ @slicer << member_filter.name if member_filter.dimension?
27
+ end
28
+ self
29
+ end
30
+
31
+ def slicer
32
+ dims = selected_dimensions
33
+ @slicer.reject{|m|dims.include?(m)}
34
+ end
35
+
36
+ def referenced_dimensions
37
+ ( [] + selected_dimensions + slicer ).compact
38
+ end
39
+
40
+ def selected_dimensions
41
+ dimensions = []
42
+ axes.each { |dims|dimensions.concat(dims) unless dims.blank? }
43
+ dimensions.uniq.compact
44
+ end
45
+
46
+ def all_dimensions?
47
+ axes.empty?
48
+ end
49
+
50
+ def matches_filter?(aggregation, tuple)
51
+ return true if all_filters_applied?
52
+ !( filters.detect{ |filter| !filter.matches(aggregation, tuple) } )
53
+ end
54
+
55
+ def all_filters_applied?
56
+ @all_filters_applied ||= !(filters.detect{ |filter| !filter.applied? })
57
+ end
58
+
59
+ def validate!
60
+ axes.each_with_index{|axis,index|raise "Axes must be selected from in consecutive order and contain at least one dimension. Axis #{index} is blank." if axis.blank?}
61
+ end
62
+
63
+ def self.axis_ordinal(axis_def)
64
+ case axis_def.to_s.strip.downcase.to_s
65
+ when "columns" then 0
66
+ when "rows" then 1
67
+ when "pages" then 2
68
+ when "chapters" then 3
69
+ when "sections" then 4
70
+ else axis_def.to_i
71
+ end
72
+ end
73
+
74
+ end
75
+ end
76
+ end
@@ -69,7 +69,7 @@ module Wonkavision
69
69
  def event_context
70
70
  @wonkavision_event_context
71
71
  end
72
-
72
+
73
73
  def handle_event
74
74
  ctx = @wonkavision_event_context
75
75
  ctx.data = map(ctx.data,ctx.path)
@@ -97,13 +97,25 @@ module Wonkavision
97
97
 
98
98
  def map?(condition,data,path)
99
99
  return true unless condition && condition.to_s != 'all' && condition.to_s != '*'
100
- return path =~ condition if condition.is_a?(Regexp)
101
- if (condition.is_a?(Proc))
100
+
101
+ if condition.is_a?(Regexp)
102
+ path =~ condition
103
+ elsif condition.is_a?(String)
104
+ path.downcase == condition.downcase
105
+ elsif condition.is_a?(Proc)
102
106
  return condition.call if condition.arity <= 0
103
107
  return condition.call(path) if condition.arity == 1
104
108
  return condition.call(path,data)
105
109
  end
106
110
  end
111
+
112
+ def broadcast(event_name, event)
113
+ Wonkavision.event_coordinator.publish(event_name, event)
114
+ end
115
+
116
+ def submit(event_name, event)
117
+ Wonkavision.event_coordinator.submit_job(event_name, event)
118
+ end
107
119
  end
108
120
 
109
121
  end
@@ -1,3 +1,3 @@
1
1
  module Wonkavision
2
- VERSION = '0.5.11'
2
+ VERSION = '0.6.0'
3
3
  end
@@ -0,0 +1,99 @@
1
+ require "test_helper"
2
+
3
+ class AggregationSpecTest < ActiveSupport::TestCase
4
+ context "AggregationSpec" do
5
+ setup do
6
+ @aggregation_spec = Wonkavision::Plugins::Aggregation::AggregationSpec.new("MyAggregation")
7
+ end
8
+
9
+ should "take its name from the constructor" do
10
+ assert_equal "MyAggregation", @aggregation_spec.name
11
+ end
12
+
13
+ context "#dimension" do
14
+ setup do
15
+ @aggregation_spec.dimension :a, :b, :c=>"d"
16
+ end
17
+
18
+ should "add a dimension for each provided value" do
19
+ assert_equal 2, @aggregation_spec.dimensions.length
20
+ ["a","b"].each { |dim| assert @aggregation_spec.dimensions.keys.include?(dim)}
21
+ end
22
+
23
+ should "store pass the options to each dimension" do
24
+ @aggregation_spec.dimensions.values.each { |dim| assert_equal({:c=>"d"},dim.options)}
25
+ end
26
+ end
27
+
28
+ context "#measure" do
29
+ setup do
30
+ @aggregation_spec.measure :c, :d, :e => "f"
31
+ end
32
+
33
+ should "add a measure for each provided value" do
34
+ assert_equal 2, @aggregation_spec.measures.length
35
+ end
36
+
37
+ should "add the attributes as hash elements" do
38
+ [:c, :d].each { |measure| assert @aggregation_spec.measures[measure]}
39
+ end
40
+
41
+ should "store the options to each measure" do
42
+ @aggregation_spec.measures.each_pair { |k,v| assert_equal({ "e" => "f"}, v)}
43
+ end
44
+ end
45
+
46
+ context "#aggregate_by" do
47
+ setup do
48
+ @aggregation_spec.aggregate_by :a, :b
49
+ @aggregation_spec.aggregate_by :a, :c
50
+ @aggregation_spec.aggregate_by [:a, :c, :d]
51
+ end
52
+
53
+ should "append the list of aggregations to the aggregations collection" do
54
+ assert_equal 3, @aggregation_spec.aggregations.length
55
+ end
56
+
57
+ should "append the list of aggregations as a simple array" do
58
+ assert_equal [:a, :b], @aggregation_spec.aggregations[0]
59
+ end
60
+
61
+ should "flatten arrays of attributes" do
62
+ assert_equal [:a, :c, :d], @aggregation_spec.aggregations[-1]
63
+ end
64
+ end
65
+
66
+ context "#aggregate_all_combinations" do
67
+ setup do
68
+ @aggregation_spec.dimension :a,:b,:c
69
+ @aggregation_spec.aggregate_all_combinations
70
+ end
71
+ should "determine all possible combinations of dimensions" do
72
+ assert_equal 7, @aggregation_spec.aggregations.length
73
+ combinations = [["a"], ["b"], ["c"], ["a", "b"], ["a", "c"], ["b", "c"], ["a", "b", "c"]]
74
+ assert_equal combinations, @aggregation_spec.aggregations
75
+ end
76
+ end
77
+
78
+ context "#filter" do
79
+ setup do
80
+ @aggregation_spec.filter { |msg|msg["a"] == "b"}
81
+ end
82
+ should "register a filter block with the spec" do
83
+ assert_not_nil @aggregation_spec.filter
84
+ end
85
+ end
86
+ context "#matches" do
87
+ setup do
88
+ @aggregation_spec.filter { |msg| msg["a"] == "b"}
89
+ end
90
+ should "return true if a message matches the filter" do
91
+ assert @aggregation_spec.matches({ "a" => "b"})
92
+ end
93
+ should "return false if the message does not match the filter" do
94
+ assert !@aggregation_spec.matches({ "a" => "B"})
95
+ end
96
+ end
97
+
98
+ end
99
+ end