octocore-mongo 0.0.6

data/lib/octocore-mongo/baseline.rb

@@ -0,0 +1,131 @@
+require 'octocore-mongo/counter/helpers'
+require 'descriptive_statistics'
+
+module Octo
+
+  # The baseline module. This module has methods that support
+  #   a baseline structure.
+  module Baseline
+
+    include Octo::Counter::Helper
+
+    # Define the past duration to look for while calculating
+    # baseline. This must be in days
+    MAX_DURATION = 7
+
+    # Defines the column needed for a baseline
+    def baselineable
+      key :type, Integer
+      key :ts, Time
+      key :uid, String
+
+      key :val, Float
+
+      # Generate the aggregator methods
+      generate_aggregators { |ts, method|
+        type = method_names_type_counter(method)
+        aggregate type, ts
+      }
+    end
+
+    # Defines the class for whom the baseline is applicable
+    def baseline_for(klass)
+      @baseline_for = klass
+    end
+
+    # Finds baseline value of an object
+    # @param [Fixnum] baseline_type One of the valid Baseline Types defined
+    # @param [Object] obj The object for whom baseline value is to be found
+    # @param [Time] ts The timestamp at which baseline is to be found
+    def get_baseline_value(baseline_type, obj, ts = Time.now.ceil)
+      unless Octo::Counter.constants.include?baseline_type
+        raise ArgumentError, 'No such baseline defined'
+      end
+
+      args = {
+          ts: ts,
+          type: Octo::Counter.const_get(baseline_type),
+          uid: obj.unique_id,
+          enterprise_id: obj.enterprise.id
+      }
+      bl = get_cached(args)
+      if bl
+        bl.val
+      else
+        0.01
+      end
+    end
+
+    # Does an aggregation of type for a timestamp
+    # @param [Fixnum] type The counter type for which aggregation
+    #   has to be done
+    # @param [Time] ts The time at which aggregation should happen
+    def aggregate(type, ts)
+      Octo::Enterprise.each do |enterprise|
+        aggregate_baseline enterprise.id, type, ts
+      end
+    end
+
+
+    # Aggregates the baseline for a minute
+    def aggregate_baseline(enterprise_id, type, ts = Time.now.floor)
+      clazz = @baseline_for.constantize
+      _ts = ts
+      start_calc_time = (_ts.to_datetime - MAX_DURATION.day).to_time
+      last_n_days_interval = start_calc_time.ceil.to(_ts, 24.hour)
+      last_n_days_interval.each do |hist|
+        args = {
+            ts: hist,
+            type: type,
+            enterprise_id: enterprise_id
+        }
+        counters = @baseline_for.constantize.send(:where, args)
+        baseline = baseline_from_counters(counters)
+        store_baseline enterprise_id, type, hist, baseline
+      end
+    end
+
+    private
+
+    # Stores the baseline for an enterprise, and type
+    # @param [String] enterprise_id The enterprise ID of enterprise
+    # @param [Fixnum] type The Counter type as baseline type
+    # @param [Time] ts The time stamp of storage
+    # @param [Hash{String => Float}] baseline A hash representing baseline
+    def store_baseline(enterprise_id, type, ts, baseline)
+      return if baseline.nil? or baseline.empty?
+      baseline.each do |uid, val|
+        self.new({
+                     enterprise_id: enterprise_id,
+                     type: type,
+                     ts: ts,
+                     uid: uid,
+                     val: val
+                 }).save!
+      end
+    end
+
+    # Calculates the baseline from counters
+    def baseline_from_counters(counters)
+      baseline = {}
+      uid_groups = counters.group_by { |x| x.uid }
+      uid_groups.each do |uid, counts|
+        baseline[uid] = score_counts(counts)
+      end
+      baseline
+    end
+
+    # Calculates the baseline score from an array of scores
+    # @param [Array<Float>] counts The counts array
+    # @return [Float] The baseline score for counters
+    def score_counts(counts)
+      if counts.count > 0
+        _num = counts.map { |x| x.obp }
+        _num.percentile(90)
+      else
+        0.01
+      end
+    end
+
+  end
+end

data/lib/octocore-mongo/callbacks.rb

@@ -0,0 +1,117 @@
+require 'hooks'
+require 'active_support/concern'
+
+module Octo
+
+  # Central Hooks Module
+  module OctoHooks
+
+    extend ActiveSupport::Concern
+
+    included do
+
+      define_hooks :after_app_init, :after_app_login, :after_app_logout,
+                   :after_page_view, :after_productpage_view, :after_connect,
+                   :after_update_profile, :after_update_push_token, :after_funnel_update
+
+      # # Define the after_app_init hook
+      # after_app_init do |args|
+      #   update_counters args
+      # end
+
+      # # Define the after_app_login hook
+      # after_app_login do |args|
+      #   update_counters args
+      # end
+
+      # # Define the after_app_logout hook
+      # after_app_logout do |args|
+      #   update_counters args
+      # end
+
+      # # Define the after_page_view hook
+      # after_page_view do |args|
+      #   add_session args
+      #   update_counters args
+      # end
+
+      # # Define the after_productpage_view hook
+      # after_productpage_view do |args|
+      #   add_session args
+      #   update_counters args
+      # end
+
+
+    end
+
+    # Add all the post-hook-call methods here. Also, extend the module
+    #   from here.
+    module ClassMethods
+
+      # Updates the counters of various types depending
+      #   on the event.
+      # @param [Hash] opts The options hash
+      def update_counters(opts)
+        if opts.has_key?(:product)
+          Octo::ProductHit.increment_for(opts[:product])
+        end
+        if opts.has_key?(:categories)
+          opts[:categories].each do |cat|
+            Octo::CategoryHit.increment_for(cat)
+          end
+        end
+        if opts.has_key?(:tags)
+          opts[:tags].each do |tag|
+            Octo::TagHit.increment_for(tag)
+          end
+        end
+        if opts.has_key?(:event)
+          Octo::ApiHit.increment_for(opts[:event])
+        end
+      end
+      # Adds user session for page_view and
+      # product_page_view to redis.
+      # It self expires in n seconds from last hit
+      def add_session(opts)
+        if Octo.is_not_flagged?(Octo::Funnel)
+          if opts.has_key?(:type)
+            createRedisShadowKey(opts[:enterprise].id.to_s + '_' + opts[:user].id.to_s,
+                                 opts[:type].to_s,
+                                 Octo.get_config(:session_length))
+          end
+        end
+      end
+
+      # Method was created because when a redis key
+      #  expires you can just catch the key name,
+      #  and not its value. So what we do is
+      #  create a shadow key for the given key name
+      #  and make it expire in the given amt of
+      #  time (seconds).
+      # This helps us in catching the event
+      #  when the (shadow) key expires, after which we
+      #  read the value of the main key and later
+      #  delete the main key
+      # You can change it from rpush to lpush, or set
+      #  whichever you want to use.
+      def createRedisShadowKey(keyname, value, duration)
+        MongoMapper::Document.redis.setex("shadow:" + keyname,duration,"")
+        MongoMapper::Document.redis.rpush(keyname, value)
+      end
+    end
+  end
+
+  # The class responsible for handling callbacks.
+  #   You must never need to make changes here
+  class Callbacks
+    include Hooks
+    include Octo::OctoHooks
+
+    class << self
+      def callback_update(msg)
+        update_counters msg
+      end
+    end
+
+  end
+end

data/lib/octocore-mongo/config.rb

@@ -0,0 +1,39 @@
+module Octo
+  module Config
+
+    def self.included(base)
+      base.include InstanceMethods
+      base.extend ClassMethods
+    end
+
+    module InstanceMethods
+
+    end
+
+    module ClassMethods
+
+      def load_config(opts)
+        curr_config.merge!opts
+      end
+
+      def set_config(key, val)
+        curr_config[key] = val
+      end
+
+      def get_config(key, default = nil)
+        curr_config.fetch(key, default)
+      end
+
+      def curr_config
+        @config = Hash.new({}) unless @config
+        @config
+      end
+    end
+  end
+end
+
+module Octo
+  include Octo::Config
+end
+
+require 'octocore-mongo/featureflag'

data/lib/octocore-mongo/config/config.yml

@@ -0,0 +1 @@
+name: octo_development

data/lib/octocore-mongo/config/search/index/user.yml

@@ -0,0 +1,42 @@
+settings:
+  number_of_shards: 1
+  number_of_replicas: 0
+mappings:
+  user:
+    properties:
+      enterpriseid:
+        type: string
+      userid:
+        type: integer
+      created:
+        type: date
+      updated:
+        type: date
+      last_login:
+        type: date
+      device_id:
+        type: string
+      manufacturer:
+        type: string
+      model:
+        type: string
+      city:
+        type: integer
+      country:
+        type: integer
+      state:
+        type: integer
+      os:
+        type: integer
+      browser:
+        type: integer
+      engagement:
+        type: integer
+      home_location:
+        type: geo_point
+      work_location:
+        type: geo_point
+      persona:
+        type: nested
+      time_slots:
+        type: nested

data/lib/octocore-mongo/counter.rb

@@ -0,0 +1,265 @@
+require 'octocore-mongo/counter/helpers'
+
+module Octo
+  module Counter
+    include Octo::Counter::Helper
+
+    INDEX_KEY_PREFIX = :CounterIndex
+    COUNTER_KEY_PREFIX = :Counter
+
+    SEPARATOR = '_'
+
+    # Define the different types of counters here. As a design decision
+    # you MUST ALWAYS keep the counters that can be created from
+    # subcounters in multiples. So for instance, you can not create
+    # a counter of type TYPE_MINUTE_36 from a counter of type TYPE_MINUTE_15.
+    # If you have to create a counter of type TYPE_MINUTE_36, consider
+    # creating subcounters like TYPE_MINUTE_9 and TYPE_MINUTE_4. Derive
+    # TYPE_MINUTE_9 from TYPE_MINUTE_4 and TYPE_MINUTE_4 from TYPE_MINUTE_1
+    # NOTE:
+    #
+    # IT IS VERY VERY IMPORTANT TO KEEP THE VALUES OF THESE COUNTERS IN
+    # ASCENDING ORDER. because it helps to define a concept of "max_type"
+    TYPE_MINUTE     = 0
+    TYPE_MINUTE_30  = 1
+    TYPE_HOUR       = 2
+    TYPE_HOUR_3     = 3
+    TYPE_HOUR_6     = 4
+    TYPE_HOUR_12    = 5
+    TYPE_DAY        = 6
+    TYPE_DAY_3      = 7
+    TYPE_DAY_6      = 8
+    TYPE_WEEK       = 9
+
+
+    # Define the columns necessary for counter model
+    def countables
+      key :type, Integer
+      key :ts, Time
+      key :uid, String
+
+      key :count, Integer
+
+      generate_aggregators { |ts, method|
+        totype = method_names_type_counter(method)
+        fromtype = get_fromtype_for_totype(totype)
+        aggregate_and_create(fromtype, totype, ts)
+      }
+    end
+
+    # Increments the counter for a model.
+    # @param [Object] obj The model instance for whom counter would be
+    #   incremented
+    def increment_for(obj)
+      # decide the time of event asap
+      ts = Time.now.ceil.to_i
+
+      if obj.class.ancestors.include?MongoMapper::Document
+        args = obj.key_attributes.collect { |k,v|  v.to_s }
+        cache_key = generate_key(ts, obj.class.name, *args)
+
+        val = Cequel::Record.redis.get(cache_key)
+        if val.nil?
+          val = 1
+        else
+          val = val.to_i + 1
+        end
+
+        ttl = (time_window + 1) * 60
+
+        # Update a sharded counter
+        MongoMapper::Document.redis.setex(cache_key, ttl, val)
+
+        # Optionally, update the index
+        index_key = generate_index_key(ts, obj.class.name, *args)
+        index_present = MongoMapper::Document.redis.get(index_key).try(:to_i)
+        if index_present != 1
+          MongoMapper::Document.redis.setex(index_key, ttl, 1)
+        end
+      end
+
+    end
+
+    def aggregate_and_create(from_type, to_type, ts = Time.now.ceil)
+      duration = get_duration_for_counter_type(to_type, ts)
+      aggr = local_count(duration, from_type)
+      sum = local_counter_sum(aggr)
+      update_counters(aggr, sum, to_type, ts)
+      # Post Update Hooks should go here
+      call_completion_hook(to_type, ts)
+    end
+
+    def call_completion_hook(type, ts)
+      # If this counter type has a corresponding trends_klass
+      # it means that the trend for it must be calculated.
+      # So, schedule the trend calculation right after this
+      # is finished
+      if self.instance_variables.include?(:@trends_klass)
+        klass = self.instance_variable_get(:@trends_klass).constantize
+        # make sure it responds to the aggregation method
+        if klass.respond_to?(:aggregate_and_create)
+          klass.send(:aggregate_and_create, type, ts)
+        end
+      end
+    end
+
+    def update_counters(aggr, sum, type, ts)
+      aggr.each do |enterprise_id, uidCounters|
+        uidCounters.each do |uid, count|
+          counter = self.new({
+                                 enterprise_id: enterprise_id,
+                                 uid: uid,
+                                 count: count,
+                                 type: type,
+                                 ts: ts
+                             })
+          if counter.respond_to?(:obp)
+            counter.obp = count.to_f/sum[enterprise_id]
+          end
+          counter.save!
+        end
+      end
+    end
+
+    # Does the counting from DB. Unlike the other counter that uses Redis. Hence
+    #   the name local_count
+    # @param [Time] duration A time/time range object
+    # @param [Fixnum] type The type of counter to look for
+    def local_count(duration, type)
+      aggr = {}
+      Octo::Enterprise.each do |enterprise|
+        args = {
+            enterprise_id: enterprise.id,
+            ts: duration,
+            type: type
+        }
+        aggr[enterprise.id.to_s] = {} unless aggr.has_key?(enterprise.id.to_s)
+        results = where(args)
+        results_group = results.group_by { |x| x.uid }
+        results_group.each do |uid, counters|
+          _sum = counters.inject(0) do |sum, counter|
+            sum + counter.count
+          end
+          aggr[enterprise.id.to_s][uid] = _sum
+        end
+      end
+      aggr
+    end
+
+    def local_counter_sum(aggr)
+      sum = {}
+      aggr.each do |enterprise_id, uidCounters|
+        _sum = uidCounters.values.inject(0) do |s, count|
+          s + count
+        end
+        sum[enterprise_id] = _sum
+      end
+      sum
+    end
+
+    # Aggregates all the counters available. Aggregation of only time specific
+    #   events can be done by passing the `ts` parameter.
+    # @param [Time] ts The time at which aggregation has to be done.
+    # @return [Hash{Fixnum => Hash{ Obj => Fixnum }}] The counts of each object
+    def aggregate(ts = Time.now.floor)
+      ts = ts.to_i
+      aggr = {}
+      # Find all counters from the index
+      index_key = generate_index_key(ts, '*')
+      counters = MongoMapper::Document.redis.keys(index_key)
+      counters.each do |cnt|
+        _tmp = cnt.split(SEPARATOR)
+        _ts = _tmp[2].to_i
+        aggr[_ts] = {} unless aggr.has_key?(_ts)
+
+        clazz = _tmp[3]
+        _clazz = clazz.constantize
+
+        _attrs = _tmp[4.._tmp.length]
+
+        args = {}
+        _clazz.key_column_names.each_with_index do |k, i|
+          args[k] = _attrs[i]
+        end
+
+        obj = _clazz.public_send(:get_cached, args)
+
+        # construct the keys for all counters matching this patter
+        _attrs << '*'
+        counters_search_key = generate_key_prefix(_ts, clazz, _attrs)
+        counter_keys = MongoMapper::Document.redis.keys(counters_search_key)
+        counter_keys.each do |c_key|
+          val = MongoMapper::Document.redis.get(c_key)
+          if val
+            aggr[_ts][obj] = aggr[_ts].fetch(obj, 0) + val.to_i
+          else
+            aggr[_ts][obj] = aggr[_ts].fetch(obj, 0) + 1
+          end
+        end
+      end
+      aggr
+
+    end
+
+    # Aggregates and attempts to store it into the database. This would only
+    #   work if the class that extends Octo::Counter includes from
+    #   Cequel::Record
+    def aggregate!(ts = Time.now.floor)
+      unless self.ancestors.include?MongoMapper::Document
+        raise NoMethodError, "aggregate! not defined for this counter"
+      end
+
+      aggr = aggregate(ts)
+      aggr.each do |_ts, counterVals|
+        counterVals.each do |obj, count|
+          args = gen_args_for_instance obj, count, _ts, TYPE_MINUTE
+          counter = self.new args
+          counter.save!
+        end
+      end
+      call_completion_hook(TYPE_MINUTE, ts)
+    end
+
+    private
+
+    def gen_args_for_instance(obj, count, ts, type)
+      {
+          enterprise: obj.enterprise,
+          uid: obj.unique_id,
+          count: count,
+          type: type,
+          ts: ts
+      }
+    end
+
+    def generate_key(ts, *args)
+      args << rand(counters)
+      [COUNTER_KEY_PREFIX, ts, *args].join(SEPARATOR)
+    end
+
+    def generate_key_prefix(ts, *args)
+      [COUNTER_KEY_PREFIX, ts, *args].join(SEPARATOR)
+    end
+
+    def generate_index_key(ts, *args)
+      [INDEX_KEY_PREFIX, self.name, ts, *args].join(SEPARATOR)
+    end
+
+    def counters
+      if self.constants.include?(:COUNTERS)
+        self.const_get(:COUNTERS)
+      else
+        30
+      end
+    end
+
+    def time_window
+      if self.constants.include?(:TIME_WINDOW)
+        self.const_get(:TIME_WINDOW)
+      else
+        1
+      end
+    end
+
+  end
+end