pulse_meter_core 0.4.13

data/lib/pulse_meter/sensor/hashed_counter.rb

@@ -0,0 +1,36 @@
+require 'json'
+
+# Static hashed counter to count values by multiple keys
+module PulseMeter
+  module Sensor
+    class HashedCounter < Counter
+
+      # Increments counter value by 1 for given key
+      # @param key [String] key to be incremented
+      def incr(key)
+        event({key => 1})
+      end
+
+      # Returs data stored in counter
+      # @return [Hash]
+      def value
+        redis.
+          hgetall(value_key).
+          inject(Hash.new(0)) {|h, (k, v)| h[k] = v.to_i; h}
+      end
+
+      private
+
+      # Processes events for multiple keys
+      # @param data [Hash] hash where keys represent counter keys
+      #   and values are increments for their keys
+      def process_event(data)
+        data.each_pair do |k, v|
+          command_aggregator.hincrby(value_key, k, v.to_i)
+          command_aggregator.hincrby(value_key, :total, v.to_i)
+        end
+      end
+
+    end
+  end
+end

data/lib/pulse_meter/sensor/hashed_indicator.rb

@@ -0,0 +1,24 @@
+module PulseMeter
+  module Sensor
+    # Static hashed indicator. In fact is is just a named hash with float value
+    class HashedIndicator < Indicator
+
+      # Get indicator values
+      # @return [Fixnum] indicator value or zero unless it was initialized
+      def value
+        redis.
+          hgetall(value_key).
+          inject(Hash.new(0)) {|h, (k, v)| h[k] = v.to_f; h}
+      end
+
+      private
+
+      # Sets indicator values
+      # @param value [Hash] new indicator values
+      def process_event(events)
+        events.each_pair {|name, value| command_aggregator.hset(value_key, name, value.to_f)}
+      end
+
+    end
+  end
+end

data/lib/pulse_meter/sensor/indicator.rb

@@ -0,0 +1,35 @@
+module PulseMeter
+  module Sensor
+    # Static indicator. In fact is is just a named variable with float value
+    class Indicator < Base
+
+      # Cleans up all sensor metadata in Redis
+      def cleanup
+        redis.del(value_key)
+        super
+      end
+
+      # Get indicator value
+      # @return [Fixnum] indicator value or zero unless it was initialized
+      def value
+        val = redis.get(value_key)
+        val.nil? ? 0 : val.to_f
+      end
+
+      # Gets redis key by which counter value is stored
+      # @return [String]
+      def value_key
+        @value_key ||= "pulse_meter:value:#{name}"
+      end
+
+      private
+
+      # Sets indicator value
+      # @param value [Float] new indicator value
+      def process_event(value)
+        command_aggregator.set(value_key, value.to_f)
+      end
+
+    end
+  end
+end

data/lib/pulse_meter/sensor/multi.rb

@@ -0,0 +1,97 @@
+module PulseMeter
+  module Sensor
+    class Multi < Base
+      include PulseMeter::Mixins::Utils
+      include Enumerable
+
+      attr_reader :name
+      attr_reader :factors
+      attr_reader :sensors
+      attr_reader :configuration_options
+
+      # TODO restore in initializer
+
+      def initialize(name, options)
+        @name = name
+        @factors = assert_array!(options, :factors)
+        @sensors = PulseMeter::Sensor::Configuration.new
+        @configuration_options = options[:configuration]
+        raise ArgumentError, "configuration option missing" unless @configuration_options
+      end
+
+      def sensor(name)
+        raise ArgumentError, 'need a block' unless block_given?
+        sensors.sensor(name){|s| yield(s)}
+      end
+
+      def event(factors_hash, value)
+        ensure_valid_factors!(factors_hash)
+
+        each_factors_combination do |combination|
+          factor_values = factor_values_for_combination(combination, factors_hash)
+          get_or_create_sensor(combination, factor_values) do |s|
+            s.event(value)
+          end
+        end
+      end
+
+      def each
+        sensors.each {|s| yield(s)}
+      end
+
+      def sensor_for_factors(factor_names, factor_values)
+        raise ArgumentError, 'need a block' unless block_given?
+        sensor(get_sensor_name(factor_names, factor_values)){|s| yield(s)}
+      end
+
+      protected
+
+      def is_subsensor?(sensor)
+        sensor.name.start_with?(get_sensor_name([], []).to_s)
+      end
+
+      def get_or_create_sensor(factor_names, factor_values)
+        raise ArgumentError, 'need a block' unless block_given?
+        name = get_sensor_name(factor_names, factor_values)
+        unless sensors.has_sensor?(name)
+          sensors.add_sensor(name, configuration_options)
+          dump!(false)
+        end
+        sensor(name) do |s|
+          yield(s)
+        end
+      end
+
+      def ensure_valid_factors!(factors_hash)
+        factors.each do |factor_name|
+          unless factors_hash.has_key?(factor_name)
+            raise ArgumentError, "Value of factor #{factor_name} missing"
+          end
+        end
+      end
+
+      def each_factors_combination
+        each_subset(factors) do |combination|
+          yield(combination)
+        end
+      end
+
+      def factor_values_for_combination(combination, factors_hash)
+        combination.each_with_object([]) do |k, acc|
+          acc << factors_hash[k]
+        end
+      end
+
+      def get_sensor_name(factor_names, factor_values)
+        sensor_name = name.to_s
+        unless factor_names.empty?
+          factor_names.zip(factor_values).each do |n, v|
+            sensor_name << "_#{n}_#{v}"
+          end
+        end
+        sensor_name.to_sym
+      end
+
+    end
+  end
+end

data/lib/pulse_meter/sensor/timeline.rb

@@ -0,0 +1,236 @@
+require 'securerandom'
+
+module PulseMeter
+  module Sensor
+    # @abstract Represents timelined sensor: series of values,
+    #   one value for each consequent time interval.
+    class Timeline < Base
+      include PulseMeter::Mixins::Utils
+      include PulseMeter::Sensor::TimelineReduce
+
+      MAX_TIMESPAN_POINTS = 1000
+
+      # @!attribute [r] interval
+      #   @return [Fixnum] Rotation interval
+      # @!attribute [r] ttl
+      #   @return [Fixnum] How long summarized data will be stored before expiration
+      # @!attribute [r] raw_data_ttl
+      #   @return [Fixnum] How long unsummarized raw data will be stored before expiration
+      # @!attribute [r] reduce_delay
+      #   @return [Fixnum] Delay between end of interval and summarization
+      attr_reader :interval, :ttl, :raw_data_ttl, :reduce_delay
+
+      # Default values for some sensor parameters
+      DEFAULTS = {
+        :raw_data_ttl => 3600,
+        :reduce_delay => 60,
+      }
+      
+      # Initializes sensor with given name and parameters
+      # @param name [String] sensor name
+      # @option options [Fixnum] :interval Rotation interval
+      # @option options [Fixnum] :ttl How long summarized data will be stored before expiration
+      # @option options [Fixnum] :raw_data_ttl How long unsummarized raw data will be stored before expiration
+      # @option options [Fixnum] :reduce_delay Delay between end of interval and summarization
+      def initialize(name, options)
+        @interval = assert_positive_integer!(options, :interval)
+        @ttl = assert_positive_integer!(options, :ttl)
+        @raw_data_ttl = assert_positive_integer!(options, :raw_data_ttl, DEFAULTS[:raw_data_ttl])
+        @reduce_delay = assert_positive_integer!(options, :reduce_delay, DEFAULTS[:reduce_delay])
+        super
+      end
+
+      # Clean up all sensor metadata and data
+      def cleanup
+        keys = redis.keys(raw_data_key('*')) + redis.keys(data_key('*'))
+        multi do
+          keys.each{|key| redis.del(key)}
+        end
+        super
+      end
+
+      # Processes event from the past
+      # @param time [Time] event time
+      # @param value event value
+      def event_at(time, value = nil)
+        multi do
+          interval_id = get_interval_id(time)
+          key = raw_data_key(interval_id)
+          aggregate_event(key, value)
+          command_aggregator.expire(key, raw_data_ttl)
+        end
+        true
+      rescue StandardError => e
+        false
+      end
+
+      # Returts sensor data within some last seconds
+      # @param time_ago [Fixnum] interval length in seconds
+      # @return [Array<SensorData>]
+      # @raise ArgumentError if argumets are not valid time objects
+      def timeline(time_ago)
+        raise ArgumentError unless time_ago.respond_to?(:to_i) && time_ago.to_i > 0
+        now = Time.now
+        timeline_within(now - time_ago.to_i, now)
+      end
+
+      # Returts sensor data within given time
+      # @param from [Time] lower bound
+      # @param till [Time] upper bound
+      # @param skip_optimization [Boolean] must be set to true to skip interval optimization
+      # @return [Array<SensorData>]
+      # @raise ArgumentError if argumets are not valid time objects
+      def timeline_within(from, till, skip_optimization = false)
+        raise ArgumentError unless from.kind_of?(Time) && till.kind_of?(Time)
+        start_time, end_time = from.to_i, till.to_i
+        actual_interval = optimized_interval(start_time, end_time, skip_optimization)
+        start_interval_id = get_interval_id(start_time) + actual_interval
+        ids, values = fetch_reduced_interval_data(start_interval_id, actual_interval, end_time)
+        zip_with_raw_data(ids, values)
+      end
+      
+      # Returns sensor data for given interval making in-memory summarization
+      #   and returns calculated value
+      # @param interval_id [Fixnum]
+      # @return [SensorData]
+      def get_raw_value(interval_id)
+        interval_raw_data_key = raw_data_key(interval_id)
+        if redis.exists(interval_raw_data_key)
+          sensor_data(interval_id, summarize(interval_raw_data_key))
+        else
+          sensor_data(interval_id, nil)
+        end
+      end
+
+      # Drops sensor data within given time
+      # @param from [Time] lower bound
+      # @param till [Time] upper bound
+      # @raise ArgumentError if argumets are not valid time objects
+      def drop_within(from, till)
+        raise ArgumentError unless from.kind_of?(Time) && till.kind_of?(Time)
+        start_time, end_time = from.to_i, till.to_i
+        current_interval_id = get_interval_id(start_time) + interval
+        keys = []
+        while current_interval_id < end_time
+          keys << data_key(current_interval_id)
+          keys << raw_data_key(current_interval_id)
+          current_interval_id += interval
+        end
+        keys.empty? ? 0 : redis.del(*keys)
+      end
+
+      # Returns Redis key by which raw data for current interval is stored
+      def current_raw_data_key
+        raw_data_key(current_interval_id)
+      end
+
+      # Returns Redis key by which raw data for given interval is stored
+      # @param id [Fixnum] interval id
+      def raw_data_key(id)
+        "pulse_meter:raw:#{name}:#{id}"
+      end
+
+      # Returns Redis key by which summarized data for given interval is stored
+      # @param id [Fixnum] interval id
+      def data_key(id)
+        "pulse_meter:data:#{name}:#{id}"
+      end
+
+      # Returns interval id where given time is
+      # @param time [Time]
+      def get_interval_id(time)
+        (time.to_i / interval) * interval
+      end
+
+      # Returns current interval id
+      # @return [Fixnum]
+      def current_interval_id
+        get_interval_id(Time.now)
+      end
+
+      # @abstract Registeres event for current interval identified by key
+      # @param key [Fixnum] interval id
+      # @param value [Object] value to be aggregated
+      def aggregate_event(key, value)
+        # simple
+        redis.set(key, value)
+      end
+
+      # @abstract Summarizes all event within interval to a single value
+      # @param key [Fixnum] interval_id 
+      def summarize(key)
+        # simple
+        redis.get(key)
+      end
+
+      # @abstract Deflates data taken from redis as string preserving nil values
+      # @param value [String] raw data
+      def deflate_safe(value)
+        value.nil? ? nil : deflate(value)
+      rescue
+        nil
+      end
+
+      private
+
+      def deflate(value)
+        # simple
+        value
+      end
+
+      def sensor_data(interval_id, value)
+        value = deflate(value) unless value.nil?
+        SensorData.new(Time.at(interval_id), value)
+      end
+
+      # Processes event
+      # @param value event value
+      def process_event(value = nil)
+        command_aggregator.multi do
+          current_key = current_raw_data_key
+          aggregate_event(current_key, value)
+          command_aggregator.expire(current_key, raw_data_ttl)
+        end
+      end
+
+      # Makes interval optimization so that the requested timespan contains less than MAX_TIMESPAN_POINTS values
+      # @param start_time [Fixnum] unix timestamp of timespan start
+      # @param end_time [Fixnum] unix timestamp of timespan start
+      # @return [Fixnum] optimized interval in seconds.
+      def optimized_interval(start_time, end_time, skip_optimization = false)
+        res_interval = interval
+        return res_interval if skip_optimization
+        timespan = end_time - start_time
+        while timespan / res_interval > MAX_TIMESPAN_POINTS - 1
+          res_interval *= 2
+        end
+        res_interval
+      end
+
+      def fetch_reduced_interval_data(start_interval_id, actual_interval, end_time)
+        keys = []
+        ids = []
+        current_interval_id = start_interval_id
+        while current_interval_id < end_time
+          ids << current_interval_id
+          keys << data_key(current_interval_id)
+          current_interval_id += actual_interval
+        end
+        values = keys.empty? ? [] : redis.mget(*keys)
+        [ids, values]
+      end
+
+      def zip_with_raw_data(ids, values)
+        res = []
+        ids.zip(values) do |(id, val)|
+          res << if val.nil?
+            get_raw_value(id)
+          else
+            sensor_data(id, val)
+          end
+        end
+        res
+      end
+    end
+  end
+end

data/lib/pulse_meter/sensor/timeline_reduce.rb

@@ -0,0 +1,68 @@
+module PulseMeter
+  module Sensor
+    # Methods for reducing raw data to single values
+    module TimelineReduce
+      
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+      
+      MAX_INTERVALS = 100
+
+      # @note Interval id is
+      #   just unixtime of its lower bound. Ruduction is a process
+      #   of 'compressing' all interval's raw data to a single value.
+      #   When reduction is done summarized data is saved to Redis
+      #   separately with expiration time taken from sensor configuration.
+      # @param interval_id [Fixnum] 
+      def reduce(interval_id)
+        interval_raw_data_key = raw_data_key(interval_id)
+        return unless redis.exists(interval_raw_data_key)
+        value = summarize(interval_raw_data_key)
+        interval_data_key = data_key(interval_id)
+        multi do
+          redis.del(interval_raw_data_key)
+          if redis.setnx(interval_data_key, value)
+            redis.expire(interval_data_key, ttl)
+          end
+        end
+      end
+
+      # Reduces data in all raw intervals
+      def reduce_all_raw
+        time = Time.now
+        min_time = time - reduce_delay  - interval
+        max_depth = time - reduce_delay - interval * MAX_INTERVALS
+        ids = collect_ids_to_reduce(time, max_depth, min_time)
+        ids.reverse.each {|id| reduce(id)}
+      end
+
+      def collect_ids_to_reduce(time, time_from, time_to)
+        ids = []
+        while (time > time_from) # go backwards
+          time -= interval
+          interval_id = get_interval_id(time)
+          next if Time.at(interval_id) > time_to
+
+          reduced_key = data_key(interval_id)
+          raw_key = raw_data_key(interval_id)
+          break if redis.exists(reduced_key)
+          ids << interval_id
+        end
+        ids
+      end
+
+      module ClassMethods
+
+        def reduce_all_raw
+          list_objects.each do |sensor|
+            sensor.reduce_all_raw if sensor.respond_to? :reduce_all_raw
+          end
+        end
+
+      end
+
+    end
+  end
+end
+