octocore-mongo 0.0.6

data/lib/octocore-mongo/stats.rb

@@ -0,0 +1,33 @@
+require 'statsd-ruby'
+
+module Octo
+
+  # Instrumentation and Statistical module
+  module Stats
+
+    # Instrument a block identified by its name
+    # @param [Symbol] name The name by which this would be identified
+    def instrument(name)
+      if stats
+        stats.time(name.to_s, &Proc.new)
+      else
+        yield
+      end
+    end
+
+    # Get stats instance
+    def stats
+      if statd_config
+        @statsd = Statsd.new(*statd_config.values) unless @statsd
+        @statsd
+      end
+    end
+
+    private
+
+    # Get stats config from Octo
+    def statd_config
+      Octo.get_config :statsd
+    end
+  end
+end

data/lib/octocore-mongo/trendable.rb

@@ -0,0 +1,88 @@
+require 'octocore-mongo/kldivergence'
+require 'octocore-mongo/counter'
+
+module Octo
+
+  module Trendable
+
+    include Octo::KLDivergence
+
+    # Define the columns necessary for a trendable model
+    def trendables
+      key :divergence, Float
+      key :obp, Float
+    end
+
+    # Define the baseline class for this trend
+    def baseline(klass)
+      @baseline_klass = klass
+    end
+
+    # Define the class for trends
+    def trends_class(klass)
+      @trends_klass = klass
+    end
+
+    # Aggregates and attempts to store it into the database. This would only
+    #   work if the class that extends Octo::Counter includes from
+    #   Cequel::Record
+    def aggregate!(ts = Time.now.floor)
+      unless self.ancestors.include?MongoMapper::Document
+        raise NoMethodError, 'aggregate! not defined for this counter'
+      end
+
+      aggr = aggregate(ts)
+      sum = aggregate_sum(aggr)
+      aggr.each do |_ts, counterVals|
+        counterVals.each do |obj, count|
+          counter = self.new
+          counter.enterprise_id = obj.enterprise.id
+          counter.uid = obj.unique_id
+          counter.count = count
+          counter.type = Octo::Counter::TYPE_MINUTE
+          counter.ts = _ts
+          totalCount = sum[_ts][obj.enterprise_id.to_s].to_f
+          counter.obp = (count * 1.0)/totalCount
+
+          baseline_value = get_baseline_value(:TYPE_MINUTE, obj)
+          counter.divergence = kl_divergence(counter.obp,
+                                             baseline_value)
+          counter.save!
+        end
+      end
+      call_completion_hook(Octo::Counter::TYPE_MINUTE, ts)
+    end
+
+    private
+
+    # Aggregates to find the sum of all counters for an enterprise
+    #   at a time
+    # @param [Hash] aggr The aggregated hash
+    # @return [Hash] The summed up hash
+    def aggregate_sum(aggr)
+      sum = {}
+      aggr.each do |ts, counterVals|
+        sum[ts] = {} unless sum.has_key?ts
+        counterVals.each do |obj, count|
+          if obj.respond_to?(:enterprise_id)
+            eid = obj.public_send(:enterprise_id).to_s
+            sum[ts][eid] = sum[ts].fetch(eid, 0) + count
+          end
+        end
+      end
+      sum
+    end
+
+    # Get the baseline value for an object.
+    # @param [Fixnum] baseline_type The type of baseline to fetch
+    # @param [Object] object The object for which baseline is to
+    #   be fetched
+    def get_baseline_value(baseline_type, object)
+      clazz = @baseline_klass.constantize
+      clazz.public_send(:get_baseline_value, baseline_type, object)
+    end
+
+
+  end
+
+end

data/lib/octocore-mongo/trends.rb

@@ -0,0 +1,158 @@
+require 'octocore-mongo/baseline'
+require 'octocore-mongo/counter'
+
+module Octo
+
+  module Trends
+
+    include Octo::Counter::Helper
+
+    DEFAULT_COUNT = 10
+
+    # Define the columns needed for Trends
+    def trendable
+      key :type, Integer
+      key :ts, Time
+      key :rank, Integer
+
+      key :score, Float
+      key :uid, String
+
+      generate_aggregators { |ts, method|
+        trendtype = method_names_type_counter(method)
+        aggregate_and_create trendtype, ts
+      }
+
+    end
+
+    # Aggregates and creates trends for all the enterprises for a specific
+    #   trend type at a specific timestamp
+    # @param [Fixnum] oftype The type of trend to be calculated
+    # @param [Time] ts The time at which trend needs to be calculated
+    def aggregate_and_create(oftype, ts = Time.now.floor)
+      Octo::Enterprise.each do |enterprise|
+        calculate(enterprise.id, oftype, ts)
+      end
+    end
+
+    # Override the aggregate! defined in counter class as the calculations
+    #   for trending are a little different
+    def aggregate!(ts = Time.now.floor)
+      aggregate_and_create(Octo::Counter::TYPE_MINUTE, ts)
+    end
+
+    # Performs the actual trend calculation
+    # @param [String] enterprise_id The enterprise ID for whom trend needs to be found
+    # @param [Fixnum] trend_type The trend type to be calculates
+    # @param [Time] ts The Timestamp at which trend needs to be calculated.
+    def calculate(enterprise_id, trend_type, ts = Time.now.floor)
+        args = {
+          enterprise_id: enterprise_id,
+          ts: ts,
+          type: trend_type
+        }
+
+        klass = @trend_for.constantize
+        hitsResult = klass.public_send(:where, args)
+        trends = hitsResult.map { |h| counter2trend(h) }
+
+        # group trends as per the time of their happening and rank them on their
+        # score
+        grouped_trends = trends.group_by { |x| x.ts }
+        grouped_trends.each do |_ts, trendlist|
+          sorted_trendlist = trendlist.sort_by { |x| x.score }
+          sorted_trendlist.each_with_index do |trend, index|
+            trend.rank = index
+            trend.type = trend_type
+            trend.save!
+          end
+        end
+    end
+
+    # Define the class for which trends shall be found
+    def trend_for(klass)
+      unless klass.constantize.ancestors.include?MongoMapper::Document
+        raise ArgumentError, "Class #{ klass } does not represent a DB Model"
+      else
+        @trend_for = klass
+      end
+    end
+
+    # Define the class which would be returned while fetching trending objects
+    def trend_class(klass)
+      @trend_class = klass
+    end
+
+    # Gets the trend of a type at a time
+    # @param [String] enterprise_id The ID of enterprise for whom trend to fetch
+    # @param [Fixnum] type The type of trend to fetch
+    # @param [Hash] opts The options to be provided for finding trends
+    def get_trending(enterprise_id, type, opts={})
+      ts = opts.fetch(:ts, Time.now.floor)
+      args = {
+        enterprise_id: enterprise_id,
+        ts: opts.fetch(:ts, Time.now.floor),
+        type: type
+      }
+      res = where(args).limit(opts.fetch(:limit, DEFAULT_COUNT))
+      enterprise = Octo::Enterprise.find_by_id(enterprise_id)
+      if res.count == 0 and enterprise.fakedata?
+        Octo.logger.info 'Beginning to fake data'
+        res = []
+        if ts.class == Range
+          ts_begin = ts.begin
+          ts_end = ts.end
+          ts_begin.to(ts_end, 1.day).each do |_ts|
+            3.times do |rank|
+              items = @trend_class.constantize.send(:where, {enterprise_id: enterprise_id}).first(10)
+              if items.count > 0
+                uid = items.shuffle.pop.unique_id
+                _args = args.merge( ts: _ts, rank: rank, score: rank+1, uid: uid )
+                res << self.new(_args).save!
+              end
+            end
+          end
+        elsif ts.class == Time
+          3.times do |rank|
+            uid = 0
+            items = @trend_class.constantize.send(:where, {enterprise_id: enterprise_id}).first(10)
+            if items.count > 0
+              uid = items.shuffle.pop.unique_id
+              _args = args.merge( rank: rank, score: rank+1, uid: uid )
+              res << self.new(_args).save!
+            end
+          end
+        end
+      end
+      res.map do |r|
+        clazz = @trend_class.constantize
+        clazz.public_send(:recreate_from, r)
+      end
+    end
+
+    private
+
+    # Converts a couunter into a trend
+    # @param [Object] counter A counter object. This object must belong
+    #   to one of the counter types defined in models.
+    # @return [Object] Returns a trend instance corresponding to the counter
+    #   instance
+    def counter2trend(counter)
+      self.new({
+        enterprise: counter.enterprise,
+        score: score(counter.divergence),
+        uid: counter.uid,
+        ts: counter.ts
+      })
+    end
+
+    # For now, just make sure divergence is absolute. This is responsible to
+    #   score a counter on the basis of its divergence
+    # @param [Float] divergence The divergence value
+    # @return [Float] The score
+    def score(divergence)
+      return 0 if divergence.nil?
+      divergence.abs
+    end
+  end
+end

data/lib/octocore-mongo/utils.rb

@@ -0,0 +1,90 @@
+require 'set'
+
+module Octo
+
+  module Utils
+
+    class << self
+
+      # Serialize one record before adding it to the cache. Creates a ruby byte
+      #   stream
+      # @param [Object] record Any object to be serialized
+      def serialize(record)
+        Marshal::dump(record).to_s
+      end
+
+      # Deserialize a data.
+      # @param [String] data A string containing Marshal dump of the object
+      def deserialize(data)
+        Marshal::load(data)
+      end
+    end
+  end
+end
+
+class ::Time
+
+  # Find floor time
+  # @param [Fixnum] height The minutes of height for floor. Defaults to 1
+  def floor(height = 1)
+    if height < 1
+      height = 1
+    end
+    sec = height.to_i * 60
+    Time.at((self.to_i / sec).round * sec)
+  end
+
+  # Find ceil time
+  # @param [Fixnum] height The minutes of height for ceil. Defaults to 1
+  def ceil(height = 1)
+    if height < 1
+      height = 1
+    end
+    sec = height.to_i * 60
+    Time.at((1 + (self.to_i / sec)).round * sec)
+  end
+
+  # Finds the steps between two time.
+  # @param [Time] to The end time
+  # @param [Time] step The step time. Defaults to 15.minute
+  # @return [Array<Time>] An array containint times
+  def to(to, step = 15.minutes)
+    [self].tap { |array| array << array.last + step while array.last < to }
+  end
+
+end
+
+class ::String
+
+  # Create a custom method to convert strings to Slugs
+  def to_slug
+    #strip the string
+    ret = self.strip
+
+    #blow away apostrophes
+    ret.gsub!(/['`]/,'')
+
+    # @ --> at, and & --> and
+    ret.gsub!(/\s*@\s*/, ' at ')
+    ret.gsub!(/\s*&\s*/, ' and ')
+
+    #replace all non alphanumeric, underscore or periods with underscore
+    ret.gsub!(/\s*[^A-Za-z0-9\.\-]\s*/, '_')
+
+    #convert double underscores to single
+    ret.gsub!(/_+/,'_')
+
+    #strip off leading/trailing underscore
+    ret.gsub!(/\A[_\.]+|[_\.]+\z/,'')
+
+    ret
+  end
+
+end
+
+class ::Hash
+  def deep_merge(second)
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
+    self.merge(second.to_h, &merger)
+  end
+end

data/lib/octocore-mongo/version.rb

@@ -0,0 +1,4 @@
+module Octo
+  # The current version of the library
+  VERSION = '0.0.6'
+end

data/octocore-mongo.gemspec

@@ -0,0 +1,50 @@
+require File.expand_path('../lib/octocore-mongo/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'octocore-mongo'
+  s.version     = Octo::VERSION
+
+  s.summary     = 'Octo Enterprise Core Modules for MongoDB'
+  s.description = <<DESC
+Octocore is the core framework of Octomatic Enterprise. It
+contains all the core models, tasks, actions etc for MongoDB
+DESC
+
+  s.authors     = ['Dipankar Sarkar']
+  s.email       = 'dipankar@octo.ai'
+  s.files       = Dir['lib/**/*.rb', 'spec/**/*.rb', '[A-Z]*', 'lib/**/*.yml']
+
+  s.executables << 'fakestream-mongo'
+  s.executables << 'octocore-admin-mongo'
+
+  s.homepage    =
+    'https://github.com/octoai/gem-octocore-mongo'
+  s.license       = 'MIT'
+
+  s.has_rdoc    = true
+  s.extra_rdoc_files = 'README.md'
+
+  s.required_ruby_version = '>= 2.1.5'
+
+  s.add_runtime_dependency 'activemodel-serializers-xml', '~> 1.0.1', '>= 1.0.1'
+  s.add_runtime_dependency 'active_model_serializers', '~> 0.9.5', '>= 0.9.5'
+  s.add_runtime_dependency 'redis', '~> 3.2.2', '>= 3.2.0'
+  s.add_runtime_dependency 'redis-queue', '~> 0.0.4', '>= 0.0.4'
+  s.add_runtime_dependency 'hiredis', '~> 0.6.1', '>= 0.6.0'
+  s.add_runtime_dependency 'rake', '~> 11.1.0', '>= 11.1.0'
+  s.add_runtime_dependency 'resque', '~> 1.26.0', '>= 1.26.0'
+  s.add_runtime_dependency 'resque-scheduler', '~> 4.1.0', '>= 4.1.0'
+  s.add_runtime_dependency 'descriptive_statistics', '~> 2.5.1', '>= 2.5.0'
+  s.add_runtime_dependency 'statsd-ruby', '~> 1.3.0', '>= 1.3.0'
+  s.add_runtime_dependency 'hooks', '~> 0.4.1', '>= 0.4.1'
+  s.add_runtime_dependency 'json', '~> 1.8.1', '>= 1.8.1'
+  s.add_runtime_dependency 'ruby-kafka', '~> 0.3.2', '>= 0.3.2'
+  s.add_runtime_dependency 'mandrill-api', '~> 1.0', '>= 1.0.53'
+  s.add_runtime_dependency 'elasticsearch', '~> 1.0.17', '>= 1.0.17'
+  s.add_runtime_dependency 'faraday', '~> 0.9.2', '>= 0.9.2'
+  s.add_runtime_dependency 'mongo_mapper', '~> 0.13.1'
+  s.add_runtime_dependency 'bson_ext', '~> 1.12', '>= 1.12.5'
+
+  s.add_development_dependency 'rspec', '~> 3.4.0', '>= 3.4.0'
+  s.add_development_dependency 'parallel_tests', '~> 2.5.0', '>= 2.5.0'
+end

data/spec/lib/stats_spec.rb

@@ -0,0 +1,20 @@
+require_relative '../spec_helper'
+
+describe Octo::Stats do
+
+    describe '#instrument' do
+     let(:dummy_class) { Class.new { extend Octo::Stats } }
+
+      it 'yields the given block' do
+        block = Proc.new { puts 'hello world' }
+        name = :hello_world
+
+        expect(dummy_class).to receive(:instrument).
+            with(name, &block).
+            and_yield(&block)
+
+        dummy_class.instrument(name, &block)
+      end
+    end
+
+end