rhcf-timeseries 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95f23eccdfd639db8cdbbf3bf4dc6441d957411a
-  data.tar.gz: b28a08bed2007266e3fc6fe578fc41e0586f3e2b
+  metadata.gz: 43ad07821290e60cc41c8741c44ef2f2e5b7769d
+  data.tar.gz: dae3053f370a61499301acfe5b841156ace2bc3a
 SHA512:
-  metadata.gz: d39583afe3b37ac668b8f457b9edd72a9fa538eca111761fe89d8e48984c5c0ada796500f05c3e3bc33a5d4c800c165ae642bbc765a61769afaabaa4bf9fc568
-  data.tar.gz: 85f73e0303a7cdc3479296da317a8640c8f4a62e4e4d02e76f162200ebbd54237063e209c435271b2990293fdb9e797043984a565cec543c6e476c6099c3241a
+  metadata.gz: 42b5890b40e7084f54987deac6f52050c65bd01191edc2730346703962489e41468b93700113d96d7f219995374cde592c49957b56a1c22d2d40dfcb6087eea6
+  data.tar.gz: 2671b695bd0edb1d1b8e077d77dde591b3d16f31455c7315024fd0f9495740f2a54bf7e58202f381996d72693d84fb820f4facfea72e0d0dc6f50c817885c766

data/lib/rhcf/timeseries/manager.rb

@@ -98,8 +98,8 @@ module Rhcf
         @strategy.store_point_value(self, subject_path, resolution_name, resolution_value, point_value, event_path)
       end
 
-      def find(subject, from, to = Time.now, filter = nil)
-        Rhcf::Timeseries::Query.new(subject, from, to, self, filter)
+      def find(subject, from, to = Time.now, filter = nil, limit = 1000)
+        Rhcf::Timeseries::Query.new(subject, from, to, self, filter, limit)
       end
 
       def resolution(id)
@@ -112,8 +112,8 @@ module Rhcf
         @_resolutions ||= @resolution_ids.map { |id| resolution(id) }
       end
 
-      def crunch_values(subject, resolution_id, point, filter)
-        @strategy.crunch_values(self, subject, resolution_id, point, filter)
+      def crunch_values(subject, resolution_id, point, filter, limit = 1000)
+        @strategy.crunch_values(self, subject, resolution_id, point, filter, limit)
       end
     end
   end

data/lib/rhcf/timeseries/query.rb

@@ -1,14 +1,16 @@
 module Rhcf
   module Timeseries
     class Query
-      def initialize(subject, from, to, series, filter = nil)
-        fail ArgumentError, "Argument 'from' can not be bigger then 'to'" if from > to
+      def initialize(subject, from, to, series, filter = nil, limit = 1000)
+        from, to = to, from if from > to
+
         @series = series
         @subject = subject
         @from = from
         @to = to
 
         @filter = filter
+        @limit = limit
       end
 
       def total(resolution_id=nil)
@@ -28,7 +30,7 @@ module Rhcf
 
         point_range(resolution_id) do |point|
 
-          values = @series.crunch_values(@subject, resolution_id, point, @filter)
+          values = @series.crunch_values(@subject, resolution_id, point, @filter, @limit)
 
           next if values.empty?
           data =  {moment: point, values: values }

data/lib/rhcf/timeseries/redis_strategies.rb

@@ -5,7 +5,7 @@ module Rhcf
         'H'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter)
+      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
         values = hgetall(manager, EVENT_POINT_TOKEN, subject, resolution_id, point)
         values.reject!{|event, value| !filter.match?(event) } if filter
         values
@@ -56,7 +56,7 @@ module Rhcf
         'M'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter)
+      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
         events = events_for_subject_on(manager, subject, point, resolution_id, filter)
         mget(manager, EVENT_POINT_TOKEN, subject, resolution_id, point, events)
       end
@@ -87,20 +87,25 @@ module Rhcf
         events
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter)
+      def crunch_values(manager, subject, resolution_id, point, filter, limit = 1000)
         register_lua_scripts!(manager.connection_to_use)
         point_prefix = [manager.prefix, EVENT_POINT_TOKEN, subject, resolution_id, point].join(NAMESPACE_SEPARATOR)
         set_key = [manager.prefix, EVENT_SET_TOKEN, resolution_id, point, subject].join(NAMESPACE_SEPARATOR)
 
         data = manager.connection_to_use.evalsha(evalsha_for(:mget_matching_smembers),
-                                                 keys: [set_key], argv: [point_prefix, filter && filter.to_lua_pattern])
+                                                 keys: [set_key], argv: [point_prefix, filter && filter.to_lua_pattern, limit])
 
         return {} if data.nil?
         result = {}
+        begin
         data.first.each_with_index do |evt, idx|
           value = data.last[idx].to_i
           result[evt] = value
         end
+        rescue
+          p $!, $!.message
+          raise
+        end
 
         result
       end
@@ -110,6 +115,7 @@ module Rhcf
       end
 
       def register_lua_scripts!(connection)
+
         @lua_script_register ||=
           begin
             smembers_matching = <<-EOF
@@ -123,13 +129,73 @@ module Rhcf
             EOF
 
             mget_matching_smembers = <<-EOF
-              local matches = {}
               local set_key = KEYS[1]
               local key_prefix = ARGV[1]
               local filter_pattern = ARGV[2]
+              local limit = tonumber(ARGV[3])
               local keys = {}
               local keys_to_mget = {}
 
+              local function log(msg)
+                -- redis.call('publish', 'log', msg)
+              end
+
+              local function mget_in_batches(keys_to_mget)
+                local step = 1024
+                local results = {}
+                local last_end = 0
+                local partial = {}
+
+                local function mget_batch(ini , fin)
+                  log("Getting from " .. ini .. ' to ' .. fin .. ' on a total of ' .. #keys_to_mget)
+                  partial =  redis.call('MGET', unpack(keys_to_mget, ini, fin))
+                  for _, value in pairs(partial) do table.insert(results, value) end
+                end
+
+                for ending = step,  #keys_to_mget, step do
+                  mget_batch(last_end + 1, ending)
+                  last_end = ending
+                end
+
+                if last_end < #keys_to_mget then
+                  mget_batch(last_end + 1, #keys_to_mget)
+                end
+
+                return results;
+              end
+
+              local function sort_and_limit_tuples(subjects, values)
+                local dictionary = {}
+                for i, subject in pairs(subjects) do
+                  local value = values[i] or 0
+                  -- redis.call('publish', 'log', subject .. ' += ' .. value)
+                  dictionary[subject] = (dictionary[subject] or 0) + value
+                end
+
+                local tuples = {}
+                for subject, value in pairs(dictionary) do
+                  -- redis.call('publish', 'log', subject .. ' = ' .. value)
+                  table.insert(tuples, { subject, value } )
+                end
+
+                table.sort(tuples, function(a, b) return b[2] < a[2] end )
+
+                local new_subjects = {}
+                local new_counts = {}
+
+                for i, tuple in pairs(tuples) do
+                  if #new_subjects >= limit  then break end
+
+                  local subject = tuple[1]
+                  local value = tuple[2]
+
+                  table.insert(new_subjects, subject)
+                  table.insert(new_counts, value)
+                end
+
+                return {new_subjects, new_counts}
+              end
+
               for _, val in ipairs(redis.call('smembers', set_key)) do
                 if (filter_pattern and string.match(val, filter_pattern)) or not filter_pattern then
                   table.insert(keys, val)
@@ -138,9 +204,13 @@ module Rhcf
               end
 
               if table.getn(keys) > 0 then
-                return {keys, redis.call('MGET', unpack(keys_to_mget)) }
+                local values = mget_in_batches(keys_to_mget)
+                local sorted = sort_and_limit_tuples(keys, values)
+                log ("Values card " .. #values .. " | keys card: " .. #keys)
+                return sorted
+              else
+                return {{},{}}
               end
-              return {{},{}}
             EOF
 
             {
@@ -156,7 +226,7 @@ module Rhcf
         'G'
       end
 
-      def crunch_values(manager, subject, resolution_id, point, filter)
+      def crunch_values(manager, subject, resolution_id, point, filter, limit = 100)
         events = events_for_subject_on(manager, subject, point, resolution_id, filter)
         values = {}
         events.each do |event|

data/lib/rhcf/timeseries/version.rb

@@ -1,5 +1,5 @@
 module Rhcf
   module Timeseries
-    VERSION = "1.0.2"
+    VERSION = "1.0.3"
   end
 end

data/spec/spec_helper.rb

@@ -1,19 +1,7 @@
-# This file was generated by the `rspec --init` command. Conventionally, all
-# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
-# Require this file using `require "spec_helper"` to ensure that it is only
-# loaded once.
-#
-# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each {|f| require f}
 require 'timecop'
 RSpec.configure do |config|
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
-
-  # Run specs in random order to surface order dependencies. If you find an
-  # order dependency and want to debug it, you can fix the order by providing
-  # the seed, which is printed after each run.
-  #     --seed 1234
   config.order = 'random'
-
 end

data/spec/support/valid_strategy_spec.rb

@@ -1,5 +1,6 @@
 require 'redis'
 require 'rhcf/timeseries/manager'
+
 RSpec.shared_examples 'a valid strategy' do
   let(:redis){Redis.new}
   let(:manager) { Rhcf::Timeseries::Manager.new(connection: redis, strategy: described_class) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rhcf-timeseries
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Romeu Fonseca
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-01 00:00:00.000000000 Z
+date: 2015-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler