statlysis 0.0.1 → 0.0.2

data/lib/statlysis/cron/top/lastest_visits.rb

@@ -0,0 +1,53 @@
+# encoding: UTF-8
+
+module Statlysis
+  # See tech details at http://mvj3.github.io/2013/01/30/recent-visitors-implement/
+  class LastestVisits < Top
+    attr_accessor :clock
+    attr_accessor :reject_proc
+
+    # *pattern_proc* is a proc to extract user_id or url_prefix to compute the
+    # top visitors from log
+    # *user_id_proc* is a proc to extract user_id from log
+    # *user_info_proc* is a proc to extract visitor informations(like id, name, ...)
+    # *reject_proc* filter visitors
+    def initialize source, opts = {}
+      # set variables
+      cron.reclock opts[:default_time]
+      cron.reject_proc = opts[:reject_proc] || proc {|pattern, user_id| pattern.to_i == user_id.to_i }
+      super
+      Utils.setup_pattern_table_and_model cron.stat_table_name
+      cron
+    end
+
+    def output
+      cron.logs = cron.source.asc(cron.time_column).where(cron.time_column => {"$gte" => cron.clock.current}).limit(1000).to_a
+      return {} if cron.logs.blank?
+      cron.logs.inject({}) do |h, log|
+        pattern = cron.pattern_proc.call(log)
+        if pattern
+          h[pattern] ||= []
+          user_id = cron.user_id_proc.call(log).to_i
+          h[pattern] << user_id if not user_id.zero?
+        end
+        h
+      end
+    end
+
+    def write
+      logger.info "#{Time.now.strftime('%H:%M:%S')} #{cron.stat_model} #{cron.output.inspect}"
+      cron.output.each do |pattern, user_ids|
+        s = cron.stat_model.find_or_create(:pattern => pattern)
+        old_array = (JSON.parse(s.result) rescue []).map {|i| Array(i)[0] }
+        new_user_ids = (old_array + user_ids).reverse.uniq.reverse # ensure the right items will overwrite the left [1,4,5,7,4,3,3,2,1,5].uniq => [1, 4, 5, 7, 3, 2]
+        s.update :result => new_user_ids.reject {|user_id| cron.reject_proc.call(pattern, user_id) rescue false }.map {|user_id| cron.user_info_proc.call(user_id) }.compact[0..cron.result_limit].to_json
+      end
+      cron.clock.update cron.logs.last.try(cron.time_column)
+    end
+
+    def reclock default_time = nil
+      cron.clock = Clock.new cron.stat_table_name, (default_time || cron.clock.current)
+    end
+  end
+
+end

data/lib/statlysis/cron_set.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+require 'statlysis/cron'
+
+module Statlysis
+  class CronSet < Set
+    # filter cron_sets by pattern
+    def [] pattern = nil
+      case pattern
+      when Fixnum, Integer # support array idx access
+        self.to_a[pattern]
+      else
+        CronSet.new(select do |cron_set|
+          cron_set.multiple_dataset.name.to_s.match Regexp.new(pattern.to_s)
+        end)
+      end
+    end
+
+    def last; [-1]; end
+
+    def run
+      map(&:run)
+    end
+  end
+
+end

data/lib/statlysis/dataset.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+
+module Statlysis
+  class Dataset
+  end
+end

data/lib/statlysis/javascript/count.rb

@@ -3,7 +3,7 @@
 module Statlysis
   module Javascript
     class MultiDimensionalCount
-      attr_accessor :map_func, :reduce_func
+      attr_reader :map_func, :reduce_func
 
       def initialize *fields
         fields = :_id if fields.blank?
@@ -17,11 +17,11 @@ module Statlysis
           raise "Please assign symbol, string, or array of them"
         end
 
-        self.map_func = "function() {
+        @map_func = "function() {
           emit (#{emit_key}, {count: 1});
         }"
 
-        self.reduce_func = "function(key, values) {
+        @reduce_func = "function(key, values) {
           var count = 0;
 
           values.forEach(function(v) {

data/lib/statlysis/multiple_dataset.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+
+module Statlysis
+  class MultipleDataset
+    def initialize cron = nil
+      @cron = cron
+      @sources ||= Set.new
+      return self
+    end
+
+    attr_reader :cron, :regexp, :sources
+    def set_regexp regexp
+      case regexp
+      when Regexp
+      when String
+        regexp = Regexp.new(string)
+      else
+        raise "regexp #{regexp} should be a Regexp!" 
+      end
+      @regexp = regexp
+
+      return self
+    end
+
+    def add_source s
+      @sources.add s
+
+      return self
+    end
+
+    def name
+      if @sources.size.zero?
+        Statlysis.logger.warn "Add source to #{self} first!"
+        return nil
+      elsif @sources.size == 1
+        @sources.first.send(Utils.name(@sources.first))
+      else
+        # /multiple_log_2013[0-9]{4}/ => 'multiple_log'
+        regexp.inspect[1..-2].gsub(/\-|\[|\]|\{|\}|[0-9]/, '').sub(/\_+$/, '')
+      end
+    end
+    # Access dataset name, compact with many ORM
+    alias collection_name name # mongoid
+    alias table_name name # activerecord
+
+
+    def first_time
+      t = _resort_source_order.map(&:first).compact.map {|i| i.send(cron.time_column) }.compact.min || DateTime1970
+      t.in_time_zone(cron.time_zone)
+    end
+    def _resort_source_order; resort_source_order if cron; end # lazy load if cron is unassigned
+    def resort_source_order; raise DefaultNotImplementWrongMessage; end
+
+    # select ORM models fron ::Object namespace
+    def _select_orm _module
+      ::Object.constants.map do |c|
+        c.to_s.constantize rescue nil # NameError: uninitialized constant ClassMethods
+      end.compact.select do |c|
+        (c.class === Class) &&
+        c.respond_to?(:included_modules) &&
+        c.included_modules.index(_module)
+      end
+    end
+
+  end
+end
+
+require 'statlysis/multiple_dataset/mongoid'
+require 'statlysis/multiple_dataset/active_record'

data/lib/statlysis/multiple_dataset/active_record.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+require 'active_record'
+
+module Statlysis
+  class ActiveRecordDataset < MultipleDataset
+    attr_reader :arel
+    def arel
+      @arel = @sources.first.where("").arel
+    end
+    # TODO
+    def method_missing
+    end
+
+    def set_regexp regexp
+      super
+
+      # TODO test it
+      activerecord_models = _select_orm(ActiveRecord::Store)
+      activerecord_models.select do |_model|
+        @sources.add _model if _model.table_name.to_s.match(@regexp)
+      end
+
+      _resort_source_order
+
+      return self
+    end
+
+    def resort_source_order; @sources = @sources.map {|s| s.order("#{cron.time_column} ASC") } end
+
+  end
+
+  def ActiveRecord.[] regexp
+  end
+
+end

data/lib/statlysis/multiple_dataset/mongoid.rb

@@ -0,0 +1,54 @@
+# encoding: UTF-8
+
+require 'mongoid'
+
+# http://mongoid.org/en/origin/index.html
+# Origin provides a DSL to mix in to any object to give it the ability to build MongoDB queries easily. It was extracted from Mongoid in an attempt to allow others to leverage the DSL in their own applications without needing a mapper.
+require 'origin'
+
+module Statlysis
+  class MongoidDataset < MultipleDataset
+    # Notice: Origin::Queryable overwrite MongoidDataset#initialize
+    class Query; include Origin::Queryable end
+
+    # delegate mongoid query to @sources
+    # see document at http://rubydoc.info/github/mongoid/origin/Origin/Queryable & http://rubydoc.info/github/mongoid/origin/Origin/Forwardable
+    attr_reader :criteria
+    def method_missing m, *args, &blk
+      @criteria ||= Query.new
+      if (Origin::Selectable.forwardables + Origin::Optional.forwardables).include?(m)
+        @criteria = @criteria.__send__(m, *args, &blk)
+        @sources = @sources.map {|s| s.__send__(m, *args, &blk) }
+        return self # support method chain
+      else
+        super
+      end
+    end
+
+    def set_regexp regexp
+      super
+
+      _collections = Mongoid.default_session.collections.select {|_collection| _collection.name.match(@regexp) }
+      mongoid_models = _select_orm(Mongoid::Document)
+
+      _collections.select do |_collection|
+        _mongoid_model = mongoid_models.detect {|m| m.collection_name === _collection.name }
+        raise "Please define Mongoid model for #{_collection}.collection under ::Object namespace!" if _mongoid_model.nil?
+        mongoid_models.delete _mongoid_model
+        @sources.add _mongoid_model
+      end
+
+      _resort_source_order
+
+      return self
+    end
+
+    def resort_source_order; @sources = @sources.map {|s| s.asc(cron.time_column) } end
+
+  end
+
+  def Mongoid.[] regexp
+    MongoidDataset.new.set_regexp(regexp)
+  end
+
+end

data/lib/statlysis/rake.rb

@@ -1,28 +1,29 @@
 # encoding: UTF-8
 
 require 'rake'
+require 'only_one_rake'
 
 namespace :statlysis do
-  Statlysis::Units.each do |unit|
+  Statlysis::TimeUnits.each do |unit|
     desc "statistical in #{unit}"
     only_one_task "#{unit}_count" => :environment do
-      Statlysis.send("#{unit}_crons").map(&:run)
+      Statlysis.config.send("#{unit}_crons").map(&:run)
     end
   end
 
   desc "realtime process"
   only_one_task :realtime_process => :environment do
-    loop { Statlysis.realtime_crons.map(&:run); sleep 1 }
+    loop { Statlysis.config.realtime_crons.map(&:run); sleep 1 }
   end
 
   desc "similar process"
   only_one_task :similar_process => :environment do
-    Statlysis.similar_crons.map(&:run)
+    Statlysis.config.similar_crons.map(&:run)
   end
 
   desc "hotest process"
   only_one_task :hotest_process => :environment do
-    Statlysis.hotest_crons.map(&:run)
+    Statlysis.config.hotest_crons.map(&:run)
   end
 
 end

data/lib/statlysis/similar.rb

@@ -12,27 +12,27 @@ module Statlysis
 
       # 初始化表和模型
       cron.stat_table_name = [Statlysis.tablename_default_pre, "similar", model_name].compact.join("_")
-      cron.pattern_table_and_model cron.stat_table_name
+      Utils.setup_pattern_table_and_model cron.stat_table_name
 
       cron.id_to_similar_ids = {}
       cron
     end
 
     def process
-      puts "SimilarProcess #{cron.stat_model} at #{DateTime.now}"
+      logger.info "SimilarProcess #{cron.stat_model} at #{DateTime.now}"
       require 'gsl'
       require 'tf-idf-similarity'
 
       # 初始化文档
-      puts "开始取出 cron.id_to_text_hash_proc"
+      logger.info "开始取出 cron.id_to_text_hash_proc"
       @id_to_text_hash = cron.id_to_text_hash_proc.call
 
-      puts "开始把@id_to_text_hash转化为数组"
+      logger.info "开始把@id_to_text_hash转化为数组"
       as = @id_to_text_hash.to_a
 
-      puts "开始把as slice为1200每次"
+      logger.info "开始把as slice为1200每次"
       as.each_slice(1200) do |a|
-      puts "开始跑 #{a.size}  个条目的相似性"
+      logger.info "开始跑 #{a.size}  个条目的相似性"
       cron.corpus = TfIdfSimilarity::Collection.new
       a.each do |id, text|
         cron.corpus << TfIdfSimilarity::Document.new(text, :id => id)
@@ -59,15 +59,15 @@ module Statlysis
           _item_id_to_score[matrix_idx_to_item_id_hash[idx2]] = (num.nan? ? 0.0 : num)
         end
         _item_id_to_score.delete document.id
-        puts "对比文档："
-        puts "#{document.id} # #{summary(document.id)}"
-        puts "相关文档："
+        logger.info "对比文档："
+        logger.info "#{document.id} # #{summary(document.id)}"
+        logger.info "相关文档："
         _item_ids = _item_id_to_score.sort {|a1, b1| b1[1] <=> a1[1] }
         _item_ids[0..9].each do |item_id, score|
-          puts "#{score} #  #{summary(item_id)}"
+          logger.info "#{score} #  #{summary(item_id)}"
         end
         cron.id_to_similar_ids[document.id] = _item_ids[0..99].map(&:first)
-        puts
+        logger.info
       end
 
       # save results to database

data/lib/statlysis/timeseries.rb

@@ -4,21 +4,24 @@ module Statlysis
   module TimeSeries
     # range支持如下三种时间范围格式
     #   20121201 20121221
-    #   DateTime.parse('20121221')
-    #   DateTime.parse('20121201')..DateTime.parse('20121221')
+    #   Time.zone.parse('20121221')
+    #   Time.zone.parse('20121201')..Time.zone.parse('20121221')
     # opts[:unit]支持:hour, :day, :week, :month等时间单位
     # 返回的结果为时间范围内的序列数组
     def self.parse range, opts = {}
-      opts = opts.reverse_merge :unit => :day, :utc => true, :offset => nil
+      # removed :utc => true, no effect.
+      # and so does :offset => nil
+      opts = opts.reverse_merge :unit => :day
       unit = opts[:unit]
+      zone = opts[:zone] || Statlysis.default_time_zone || Time.zone
 
-      range = Range.new(*range.split.map {|i| DateTime.parse(i).to_time_in_current_zone }) if range.is_a?(String)
+      range = Range.new(*range.split.map {|i| Time.zone.parse(i).in_time_zone(zone) }) if range.is_a?(String)
 
       begin_unit = "beginning_of_#{unit}".to_sym
       array = if range.respond_to?(:to_datetime)
-        [range.in_time_zone.send(begin_unit)]
+        [range.in_time_zone(zone).send(begin_unit)]
       elsif range.is_a?(Range)
-        ary = [range.first.in_time_zone, range.last.in_time_zone].map(&begin_unit).uniq
+        ary = [range.first.in_time_zone(zone), range.last.in_time_zone(zone)].map(&begin_unit).uniq
 
         _ary = []
         _ary.push ary[0]
@@ -32,9 +35,9 @@ module Statlysis
         _ary.compact.reject {|i| (i < range.first) && (i >= range.last) }
       end
 
-      array = array.map {|s| s.to_time } if opts[:utc]
-      array = array.map {|i| i + opts[:offset] } if opts[:offset]
-      array.map(&:in_time_zone)
+      # array = array.map {|s| s.to_time } if opts[:utc]
+      # array = array.map {|i| i + opts[:offset] } if opts[:offset]
+      array.map(&:to_datetime)
     end
 
   end

data/lib/statlysis/utils.rb

@@ -0,0 +1,40 @@
+# encoding: UTF-8
+
+module Statlysis
+  module Utils
+    class << self
+      def is_activerecord?(data); data.is_a?(ActiveRecordDataset) || !!((data.respond_to?(:included_modules) ? data.included_modules : []).index(ActiveRecord::Store)) end
+      def is_mongoid?(data); data.is_a?(MongoidDataset) || !!((data.respond_to?(:included_modules) ? data.included_modules : []).index(Mongoid::Document)) end
+      def name(data)
+        return :collection_name if Utils.is_mongoid?(data)
+        return :table_name      if Utils.is_activerecord?(data)
+      end
+
+      def setup_pattern_table_and_model tn
+        # ensure statlysis table
+        tn = tn.pluralize
+        if not Statlysis.sequel.table_exists?(tn)
+          Statlysis.sequel.create_table tn, DefaultTableOpts.merge(:engine => "InnoDB") do
+            primary_key :id
+            String :pattern
+            index  :pattern
+          end
+          Statlysis.sequel.add_column tn, :result, String, :text => true
+        end
+
+        # generate a statlysis kv model
+        str = tn.to_s.singularize.camelize
+        class_eval <<-MODEL, __FILE__, __LINE__ + 1
+          class ::#{str} < Sequel::Model;
+            self.set_dataset :#{tn}
+            def self.[] item_id
+              JSON.parse(find_or_create(:pattern => item_id).result) rescue []
+            end
+          end;
+        MODEL
+        {:table => tn, :model => str.constantize}
+      end
+
+    end
+  end
+end