rhcf-timeseries 1.0.3 → 2.0.0pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43ad07821290e60cc41c8741c44ef2f2e5b7769d
-  data.tar.gz: dae3053f370a61499301acfe5b841156ace2bc3a
+  metadata.gz: 60bb46afd3e19f758a6167bcc079283ad7d821d0
+  data.tar.gz: 132cd23eb7fc694f11a3dfd9e5200add5441c295
 SHA512:
-  metadata.gz: 42b5890b40e7084f54987deac6f52050c65bd01191edc2730346703962489e41468b93700113d96d7f219995374cde592c49957b56a1c22d2d40dfcb6087eea6
-  data.tar.gz: 2671b695bd0edb1d1b8e077d77dde591b3d16f31455c7315024fd0f9495740f2a54bf7e58202f381996d72693d84fb820f4facfea72e0d0dc6f50c817885c766
+  metadata.gz: 281bef05f4f3d6090d0e915b9c19328906cd9ffd12ab9f2ae530b8543481c116ad6c33f5804603dabbb8db2d17de9674818b7d007e4962df1755a427fc2cacff
+  data.tar.gz: d0e4eef0b658335d3856348e053b219014deb3c16c6022b1d2217b630ba4c7f8a95e73dd13f0136a9e8be98ee91a4dfd32d1d4470a5313df592567a8ab9c280d

data/lib/rhcf/timeseries/manager.rb

@@ -50,15 +50,15 @@ module Rhcf
         @connection_to_use || fail("No connection given")
       end
 
-      def store(subject, event_point_hash, moment = Time.now, descend_subject = true, descend_event = true)
+      def store(evt_path, subject_point_hash, moment = Time.now, descend_subject = true, descend_event = true)
         resolutions = resolutions_of(moment)
 
-        descend(subject, descend_subject) do |subject_path|
-          event_point_hash.each do |event, point_value|
-            descend(event, descend_event) do |event_path|
+        descend(evt_path, descend_event) do |event_path|
+          subject_point_hash.each do |subj_path, point_value|
+            descend(subj_path, descend_subject) do |subject_path|
               resolutions.each do |res|
                 resolution_name, resolution_value = *res
-                store_point_value(subject_path, resolution_name, resolution_value, point_value, event_path)
+                store_point_value(event_path, resolution_name, resolution_value, subject_path, point_value)
               end
             end
           end
@@ -98,8 +98,8 @@ module Rhcf
         @strategy.store_point_value(self, subject_path, resolution_name, resolution_value, point_value, event_path)
       end
 
-      def find(subject, from, to = Time.now, filter = nil, limit = 1000)
-        Rhcf::Timeseries::Query.new(subject, from, to, self, filter, limit)
+      def find(evt_filter, from, to = Time.now, subj_filter = nil)
+        Rhcf::Timeseries::Query.new(evt_filter, from, to, self, subj_filter)
       end
 
       def resolution(id)
@@ -112,8 +112,12 @@ module Rhcf
         @_resolutions ||= @resolution_ids.map { |id| resolution(id) }
       end
 
-      def crunch_values(subject, resolution_id, point, filter, limit = 1000)
-        @strategy.crunch_values(self, subject, resolution_id, point, filter, limit)
+      def ranking(evt_filter, resolution_id, points_on_range, subj_filter, limit)
+        @strategy.ranking(self, evt_filter, resolution_id, points_on_range, subj_filter, limit)
+      end
+
+      def crunch_values(evt_filter, resolution_id, point, subj_filter)
+        @strategy.crunch_values(self, evt_filter, resolution_id, point, subj_filter)
       end
     end
   end

data/lib/rhcf/timeseries/query.rb

@@ -1,16 +1,15 @@
 module Rhcf
   module Timeseries
     class Query
-      def initialize(subject, from, to, series, filter = nil, limit = 1000)
+      def initialize(evt_filter, from, to, series, subj_filter = nil)
         from, to = to, from if from > to
 
         @series = series
-        @subject = subject
+        @evt_filter = evt_filter
         @from = from
         @to = to
 
-        @filter = filter
-        @limit = limit
+        @subj_filter = subj_filter
       end
 
       def total(resolution_id=nil)
@@ -25,12 +24,18 @@ module Rhcf
         accumulator
       end
 
+      def ranking(limit, resolution_id = nil)
+        resolution_id ||= better_resolution[:id]
+        points_on_range = point_range(resolution_id)
+        @series.ranking(@evt_filter, resolution_id, points_on_range, @subj_filter, limit)
+      end
+
       def points(resolution_id)
         list =[]
 
         point_range(resolution_id) do |point|
 
-          values = @series.crunch_values(@subject, resolution_id, point, @filter, @limit)
+          values = @series.crunch_values(@evt_filter, resolution_id, point, @subj_filter)
 
           next if values.empty?
           data =  {moment: point, values: values }
@@ -44,14 +49,18 @@ module Rhcf
       end
 
       def point_range(resolution_id)
+        points_on_range = []
         resolution = @series.resolution(resolution_id)
         span = resolution[:span]
         ptr = @from.dup
         while ptr < @to
           point = @series.resolution_value_at(ptr, resolution_id)
-          yield point
+          yield point if block_given?
+          points_on_range << point
           ptr += span.to_i
         end
+
+        points_on_range
       rescue FloatDomainError
         # OK
       end

data/lib/rhcf/timeseries/redis_strategies.rb

@@ -5,16 +5,16 @@ module Rhcf
         'H'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
-        values = hgetall(manager, EVENT_POINT_TOKEN, subject, resolution_id, point)
-        values.reject!{|event, value| !filter.match?(event) } if filter
+      def crunch_values(manager, evt_filter, resolution_id, time_point, subj_filter)
+        values = hgetall(manager, EVENT_POINT_TOKEN, evt_filter, resolution_id, time_point)
+        values.reject!{|event, value| !subj_filter.match?(event) } if subj_filter
         values
       end
 
-      def store_point_value(manager, subject_path, resolution_name, resolution_value, point_value, event_path)
-        key = [manager.prefix, EVENT_POINT_TOKEN ,subject_path, resolution_name, resolution_value].join(NAMESPACE_SEPARATOR)
-        manager.connection_to_use.hincrby(key, event_path, point_value)
-        manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
+      def store_point_value(manager, event_path, resolution_id, resolution_val, subject_path, increment)
+        key = point_prefix(manager, event_path, resolution_id, resolution_val)
+        manager.connection_to_use.hincrby(key, subject_path, increment)
+        manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_id][:ttl])
       end
 
       def hgetall(manager, k,s,r,p)
@@ -23,30 +23,49 @@ module Rhcf
           hash[_k] = value.to_i
         end
       end
+
+      def point_prefix(manager, evt_filter, resolution_id, time_point = nil, subj_path = nil)
+        [manager.prefix, EVENT_POINT_TOKEN, evt_filter, resolution_id, time_point, subj_path].compact.join(NAMESPACE_SEPARATOR)
+      end
+
+      def set_prefix(manager, evt_filter, resolution_id, time_point = nil)
+        [manager.prefix, EVENT_SET_TOKEN, evt_filter, resolution_id, time_point].compact.join(NAMESPACE_SEPARATOR)
+      end
+
+
+
     end
 
     class RedisStringBasedStrategy
+
+      def point_prefix(manager, evt_filter, resolution_id, time_point = nil, subj_path = nil)
+        [manager.prefix, EVENT_POINT_TOKEN, evt_filter, resolution_id, time_point, subj_path].compact.join(NAMESPACE_SEPARATOR)
+      end
+
+      def set_prefix(manager, evt_filter, resolution_id, time_point = nil)
+        [manager.prefix, EVENT_SET_TOKEN, evt_filter, resolution_id, time_point].compact.join(NAMESPACE_SEPARATOR)
+      end
+
+
       def id
         fail 'AbstractStrategy'
       end
 
-      def store_point_value(manager, subject_path, resolution_name, resolution_value, point_value, event_path)
-        store_point_event(manager, resolution_name, resolution_value, subject_path, event_path)
-        key = [manager.prefix, EVENT_POINT_TOKEN ,subject_path, resolution_name, resolution_value, event_path].join(NAMESPACE_SEPARATOR)
-        manager.connection_to_use.incrby(key, point_value)
-        manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
-      end
+      def store_point_value(manager, event_path, resolution_name, resolution_val, subj_path, increment)
+        set_key     = set_prefix(manager, event_path, resolution_name, resolution_val)
+        counter_key = point_prefix(manager, event_path, resolution_name, resolution_val, subj_path)
+
+        manager.connection_to_use.sadd(set_key, subj_path)
+        manager.connection_to_use.incrby(counter_key, increment)
 
-      def store_point_event(manager, resolution_name, resolution_value, subject_path, event_path)
-        key = [manager.prefix, EVENT_SET_TOKEN, resolution_name, resolution_value, subject_path].join(NAMESPACE_SEPARATOR)
-        manager.connection_to_use.sadd(key, event_path)
-        manager.connection_to_use.expire(key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
+        manager.connection_to_use.expire(counter_key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
+        manager.connection_to_use.expire(set_key, DEFAULT_RESOLUTIONS_MAP[resolution_name][:ttl])
       end
 
-      def events_for_subject_on(manager, subject, point, resolution_id, filter)
-        key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
+      def events_for_subject_on(manager, evt_filter, res_point, resolution_id, subj_filter)
+        key = set_prefix(manager, evt_filter, resolution_id, res_point)
         events = manager.connection_to_use.smembers(key)
-        events = events.select{|event| filter.match?(event) } if filter
+        events = events.select{|event| subj_filter.match?(event) } if subj_filter
         events
       end
     end
@@ -56,9 +75,9 @@ module Rhcf
         'M'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
-        events = events_for_subject_on(manager, subject, point, resolution_id, filter)
-        mget(manager, EVENT_POINT_TOKEN, subject, resolution_id, point, events)
+      def crunch_values(manager, evt_filter, resolution_id, time_point, subj_filter)
+        events = events_for_subject_on(manager, evt_filter, time_point, resolution_id, subj_filter)
+        mget(manager, EVENT_POINT_TOKEN, evt_filter, resolution_id, time_point, events)
       end
 
       def mget(manager, k, s, r, p, es)
@@ -76,24 +95,39 @@ module Rhcf
     class RedisMgetLuaStrategy < RedisMgetStrategy
       def id; 'ME'; end
 
-      def events_for_subject_on(manager, subject, point, resolution_id, filter)
-        key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
-        events = if filter
-                   manager.connection_to_use.evalsha(evalsha_for(:smembers_matching),
-                                                     keys: [key], argv: [filter.to_lua_pattern])
+      def ranking(manager, evt_filter, resolution_id, points_on_range, subj_filter, limit)
+        point_prefix = point_prefix(manager, evt_filter, resolution_id)
+        set_prefix = set_prefix(manager, evt_filter, resolution_id)
+
+        manager.connection_to_use.evalsha(evalsha_for(manager, :ranking),
+                                          keys: points_on_range,
+                                          argv: [
+                                            evt_filter,
+                                            subj_filter && subj_filter.to_lua_pattern,
+                                            set_prefix,
+                                            point_prefix,
+                                            limit
+                                          ])
+      end
+
+      def events_for_subject_on(manager, evt_filter, time_point, resolution_id, subj_filter)
+        key = set_prefix(manager, resolution_id, evt_filter, time_point)
+        events = if subj_filter
+                   manager.connection_to_use.evalsha(evalsha_for(manager, :smembers_matching),
+                                                     keys: [key], argv: [subj_filter.to_lua_pattern])
                  else
                    manager.connection_to_use.smembers(key)
                  end
         events
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter, limit = 1000)
-        register_lua_scripts!(manager.connection_to_use)
-        point_prefix = [manager.prefix, EVENT_POINT_TOKEN, subject, resolution_id, point].join(NAMESPACE_SEPARATOR)
-        set_key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
+      def crunch_values(manager, evt_filter, resolution_id, time_point, subj_filter)
+        point_prefix = point_prefix(manager, resolution_id, evt_filter, time_point)
+        set_key = point_prefix(manager, resolution_id, evt_filter, time_point)
 
-        data = manager.connection_to_use.evalsha(evalsha_for(:mget_matching_smembers),
-                                                 keys: [set_key], argv: [point_prefix, filter && filter.to_lua_pattern, limit])
+        data = manager.connection_to_use.evalsha(evalsha_for(manager, :mget_matching_smembers),
+                                                 keys: [set_key],
+                                                 argv: [point_prefix, subj_filter && subj_filter.to_lua_pattern])
 
         return {} if data.nil?
         result = {}
@@ -110,17 +144,63 @@ module Rhcf
         result
       end
 
-      def evalsha_for(sym_os_lua_script)
-        @lua_script_register[sym_os_lua_script] || fail("Script for '#{sym_os_lua_script}' not registered")
+      def evalsha_for(manager, sym_os_lua_script)
+        register_lua_scripts(manager.connection_to_use).fetch(sym_os_lua_script)
       end
 
-      def register_lua_scripts!(connection)
+      def register_lua_scripts(connection)
 
         @lua_script_register ||=
           begin
+            ranking_script = <<-EOF
+              local evt_filter   = ARGV[1]
+              local subj_filter  = ARGV[2]
+              local set_prefix   = ARGV[3]
+              local point_prefix = ARGV[4]
+              local limit        = tonumber(ARGV[5])
+
+              local set_keys = {}
+              for _, time in pairs(KEYS) do
+                table.insert(set_keys, set_prefix .. '|' .. time)
+              end
+
+              local all_subjects = redis.call("SUNION", unpack(set_keys))
+              local filtered_subjects = {}
+
+              if subj_filter then
+                for _, val in pairs(all_subjects) do
+                  if string.match(val, subj_filter) then
+                    table.insert(filtered_subjects, val)
+                  end
+                end
+              else
+                filtered_subjects = all_subjects
+              end
+
+              local counter_tuples = {}
+
+              for _, subject in pairs(filtered_subjects) do
+                local my_counter_keys = {}
+
+                for _, time in pairs(KEYS) do
+                  table.insert(my_counter_keys, point_prefix .. "|" .. time .. "|" .. subject)
+                end
+
+                local counter_total = 0
+                for _, val in pairs(redis.call("MGET", unpack(my_counter_keys))) do
+                  counter_total = counter_total + ( tonumber(val) or 0 )
+                end
+
+                table.insert(counter_tuples, {subject, counter_total} )
+              end
+
+              table.sort(counter_tuples, function(a, b) return b[2] < a[2] end )
+              return counter_tuples
+            EOF
+
             smembers_matching = <<-EOF
               local matches = {}
-              for _, val in ipairs(redis.call('smembers', KEYS[1])) do
+              for _, val in pairs(redis.call('smembers', KEYS[1])) do
                 if string.match(val, ARGV[1]) then
                   table.insert(matches, val)
                 end
@@ -136,23 +216,26 @@ module Rhcf
               local keys = {}
               local keys_to_mget = {}
 
+
               local function log(msg)
-                -- redis.call('publish', 'log', msg)
+                redis.call('publish', 'log', "XDEBUG: " .. msg)
               end
 
               local function mget_in_batches(keys_to_mget)
-                local step = 1024
-                local results = {}
+                local step     = 1024
+                local results  = {}
                 local last_end = 0
-                local partial = {}
+                local partial  = {}
+
 
                 local function mget_batch(ini , fin)
                   log("Getting from " .. ini .. ' to ' .. fin .. ' on a total of ' .. #keys_to_mget)
+
                   partial =  redis.call('MGET', unpack(keys_to_mget, ini, fin))
                   for _, value in pairs(partial) do table.insert(results, value) end
                 end
 
-                for ending = step,  #keys_to_mget, step do
+                for ending = step, #keys_to_mget, step do
                   mget_batch(last_end + 1, ending)
                   last_end = ending
                 end
@@ -166,16 +249,14 @@ module Rhcf
 
               local function sort_and_limit_tuples(subjects, values)
                 local dictionary = {}
-                for i, subject in pairs(subjects) do
+                for i, evt_filter in pairs(subjects) do
                   local value = values[i] or 0
-                  -- redis.call('publish', 'log', subject .. ' += ' .. value)
-                  dictionary[subject] = (dictionary[subject] or 0) + value
+                  dictionary[evt_filter] = (dictionary[evt_filter] or 0) + value
                 end
 
                 local tuples = {}
-                for subject, value in pairs(dictionary) do
-                  -- redis.call('publish', 'log', subject .. ' = ' .. value)
-                  table.insert(tuples, { subject, value } )
+                for evt_filter, value in pairs(dictionary) do
+                  table.insert(tuples, { evt_filter, value } )
                 end
 
                 table.sort(tuples, function(a, b) return b[2] < a[2] end )
@@ -186,16 +267,19 @@ module Rhcf
                 for i, tuple in pairs(tuples) do
                   if #new_subjects >= limit  then break end
 
-                  local subject = tuple[1]
+                  local evt_filter = tuple[1]
                   local value = tuple[2]
 
-                  table.insert(new_subjects, subject)
+                  table.insert(new_subjects, evt_filter)
                   table.insert(new_counts, value)
                 end
 
                 return {new_subjects, new_counts}
               end
 
+              log("SETKEY " .. set_key ) -- #.. " | KEY prefix: " .. key_prefix .. "  FILTER PATTERN: " .. filter_pattern)
+--              log("SETKEY " .. set_key .. " | KEY prefix: " .. key_prefix .. "  FILTER PATTERN: " .. filter_pattern)
+
               for _, val in ipairs(redis.call('smembers', set_key)) do
                 if (filter_pattern and string.match(val, filter_pattern)) or not filter_pattern then
                   table.insert(keys, val)
@@ -205,9 +289,10 @@ module Rhcf
 
               if table.getn(keys) > 0 then
                 local values = mget_in_batches(keys_to_mget)
-                local sorted = sort_and_limit_tuples(keys, values)
-                log ("Values card " .. #values .. " | keys card: " .. #keys)
-                return sorted
+                -- local sorted = sort_and_limit_tuples(keys, values)
+                -- log ("Values card " .. #values .. " | keys card: " .. #keys)
+                -- return sorted
+                return {keys, values}
               else
                 return {{},{}}
               end
@@ -215,7 +300,8 @@ module Rhcf
 
             {
               mget_matching_smembers: connection.script(:load, mget_matching_smembers),
-              smembers_matching: connection.script(:load, smembers_matching)
+              smembers_matching: connection.script(:load, smembers_matching),
+              ranking: connection.script(:load, ranking_script)
             }
           end
       end
@@ -226,18 +312,17 @@ module Rhcf
         'G'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
-        events = events_for_subject_on(manager, subject, point, resolution_id, filter)
+      def crunch_values(manager, evt_filter, resolution_id, time_point, subj_filter, limit = 100)
+        events = events_for_subject_on(manager, evt_filter, time_point, resolution_id, subj_filter)
         values = {}
         events.each do |event|
-          value = get(manager, EVENT_POINT_TOKEN, subject, resolution_id, point, event)
+          value = get(manager, point_prefix(manager, evt_filter, resolution_id, time_point))
           values[event] = value.to_i
         end
         values
       end
 
-      def get(manager, *a_key)
-        a_key = [manager.prefix, a_key].flatten.join(NAMESPACE_SEPARATOR)
+      def get(manager, a_key)
         manager.connection_to_use.get(a_key)
       end
     end

data/lib/rhcf/timeseries/version.rb

@@ -1,5 +1,5 @@
 module Rhcf
   module Timeseries
-    VERSION = "1.0.3"
+    VERSION = "2.0.0pre"
   end
 end

data/spec/lib/rhcf/timeseries/manager_spec.rb

@@ -5,6 +5,26 @@ require 'rhcf/timeseries/manager'
 require 'benchmark'
 require 'stackprof'
 
+
+
+def generate_subjects(n_pages, n_edits)
+  edits = [].tap do  |me|
+    1.upto(n_edits) do |i|
+      me << "edit#{i}"
+    end
+  end
+  [].tap {|me| n_pages.times { me << [edits.sample,SecureRandom.hex].join('/') } }
+end
+
+def generate_pageviews(evts_count, subjects, mintime, maxtime)
+  pts = []
+  evts_count.times do
+    time = Time.at(rand( (mintime.to_i)..(maxtime.to_i)  ))
+    pts << [ subjects.sample ,time]
+  end
+  pts
+end
+
 describe Rhcf::Timeseries::Manager do
   let(:redis){Redis.new}
   subject{Rhcf::Timeseries::Manager.new(connection: redis)}
@@ -58,6 +78,53 @@ describe Rhcf::Timeseries::Manager do
     end
   end
 
+  describe 'ranking' do
+    subject do
+      Rhcf::Timeseries::Manager.new(connection: redis,
+                                    resolutions: [:hour],
+                                    strategy: Rhcf::Timeseries::RedisMgetLuaStrategy)
+    end
+
+    let(:n_evts)  { 3000 }
+    let(:n_pages) { 15 }
+    let(:n_edits) { 3 }
+    let(:start_time) { Time.parse('2015-01-01')}
+    let(:end_time) { Time.parse('2015-01-02')}
+
+    let(:subjects){ generate_subjects(n_pages, n_edits) }
+    let(:points)  { generate_pageviews(n_evts, subjects, start_time, end_time)}
+
+    before do
+      points.each do |subject_and_time|
+        evt_subject, time = *subject_and_time
+        subject.store('pageview', {evt_subject => 1}, time, false, false)
+      end
+    end
+
+    let(:query) { subject.find("pageview", start_time, end_time) }
+
+    let(:top10_forca_bruta) {
+      acc = {}
+      points.each do |item|
+        s,_t = * item
+        acc[s] ||=0
+        acc[s] +=1
+
+      end
+      acc.sort_by{|i| i.last}.reverse[0,10]
+    }
+
+    it do
+      expect(points.count).to eq n_evts
+      expect(query.ranking(10).count).to eq 10
+      expect(query.ranking(10)).to eq top10_forca_bruta
+      expect(query.points(:hour).count).to eq 24
+      expect(redis.keys('*').count).to eq 24 + 24 * n_pages # ao infinito
+    end
+
+
+  end
+
   describe "find and total" do
     let(:start_time){  Time.parse("2000-01-01 00:00:00") }
     before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rhcf-timeseries
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 2.0.0pre
 platform: ruby
 authors:
 - Romeu Fonseca
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-08 00:00:00.000000000 Z
+date: 2015-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -172,9 +172,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2