statlysis 0.0.1 → 0.0.2

data/lib/statlysis/constants.rb

@@ -0,0 +1,10 @@
+# encoding: UTF-8
+
+module Statlysis
+  TimeUnits = %w[hour day week month year]
+  DateTime1970 = Time.zone.parse("19700101").in_time_zone
+
+  DefaultTableOpts = {:charset => "utf8", :collate => "utf8_general_ci", :engine => "MyISAM"}
+
+  DefaultNotImplementWrongMessage = "Not implement yet, please config it by subclass".freeze
+end

data/lib/statlysis/cron.rb

@@ -2,76 +2,74 @@
 
 module Statlysis
   class Cron
-    attr_accessor :source, :time_column, :time_unit
+    attr_reader :multiple_dataset, :source_type, :time_column, :time_unit, :time_zone
     include Common
 
-    DefaultWrongMessage = "not implement yet, please config it by subclass".freeze
-    def initialize source, opts = {}
-      cron.stat_table_name = opts[:stat_table_name] if opts[:stat_table_name]
-      cron.time_column     = opts[:time_column]
-      cron.source          = source
-      cron.time_unit       = opts[:time_unit]
+    def initialize s, opts = {}
+      # setup data type related
+      @source_type = ({Utils.is_activerecord?(s) => :activerecord, Utils.is_mongoid?(s) => :mongoid}.detect {|k, v| k } || {})[1] || :unknown
+
+      @time_column      = opts[:time_column]
+      @time_unit        = opts[:time_unit]
+      @time_zone        = opts[:time_zone] || Statlysis.default_time_zone || Time.zone || Time.now.utc_offset
+
+      # insert source as a dataset
+      @multiple_dataset = (s.is_a?(ActiveRecordDataset) ? s : ActiveRecordDataset.new(cron).add_source(s)) if is_activerecord?
+      @multiple_dataset = (s.is_a?(MongoidDataset) ? s : MongoidDataset.new(cron).add_source(s)) if is_mongoid?
+      @multiple_dataset.instance_variable_set("@cron", cron) if is_orm? && @multiple_dataset.cron.nil?
+
+      @stat_table_name = opts[:stat_table_name] if opts[:stat_table_name]
+
       cron
     end
-    def output; raise DefaultWrongMessage end
-    def setup_stat_table; raise DefaultWrongMessage end
-    def run; raise DefaultWrongMessage end
-
-    # overwrite to lazy load @source
-    def inspect
-      source_inspect = is_mysql? ? cron.source.to_sql : cron.source
-      str = "#<#{cron.class} @source=#{source_inspect} @stat_table_name=#{cron.stat_table_name} @time_column=#{cron.time_column} @stat_table=#{cron.stat_table}"
-      str << " @stat_model=#{cron.stat_model}" if cron.methods.index(:stat_model)
-      str << ">"
-      str
-    end
+    def output; raise DefaultNotImplementWrongMessage end
+    def reoutput; @output = nil; output end
+    def setup_stat_model; raise DefaultNotImplementWrongMessage end
+    def run; raise  DefaultNotImplementWrongMessage end
+    def is_activerecord?; @source_type == :activerecord; end
+    def is_mongoid?; @source_type == :mongoid; end
+    def is_orm?; [:activerecord, :mongoid].include?(@source_type); end
 
+    def _source
+      cron.multiple_dataset.sources.first
+    end
     def source_where_array
       # TODO follow index seq
-      a = cron.source.where("").where_values.map do |equality|
+      a = _source.where("").where_values.map do |equality|
         # use full keyvalue index name
         equality.is_a?(String) ? equality.to_sym : "#{equality.operand1.name}#{equality.operand2}"
-      end if is_mysql?
-      a = cron.source.all.selector.reject {|k, v| k == 't' } if is_mongodb?
-      a.map {|s| s.to_s.split(//).select {|s| s.match(/[a-z0-9]/i) }.join }.sort.map(&:to_sym)
-    end
-
-    def source_name
-      @source_name ||= begin
-        m = :table_name if is_mysql?
-        m = :collection_name if is_mongodb?
-        cron.source.send(m)
-      end
+      end if is_activerecord?
+      a = _source.all.selector.reject {|k, v| k == 't' } if is_mongoid?
+      a.map {|s1| s1.to_s.split(//).select {|s2| s2.match(/[a-z0-9]/i) }.join }.sort.map(&:to_sym)
     end
 
     # automode
     # or
     # specify TIME_RANGE and TIME_UNIT in shell to run
     def time_range
-      return TimeSeries.parse(ENV['TIME_RANGE'], :unit => (ENV['TIME_UNIT'] || 'day')) if ENV['TIME_RANGE']
-      # 选择开始时间。取出统计表的最后时间，和数据表的最先时间对比，哪个最后就选择
+      return TimeSeries.parse(ENV['TIME_RANGE'], :unit => (ENV['TIME_UNIT'] || 'day'), :zone => cron.time_zone) if ENV['TIME_RANGE']
+      # 选择开始时间。取出统计表的最后时间，和数据表的最先时间对比，哪个在后就选择哪个
       begin_day = DateTime.now.beginning_of_day
-      st_timebegin = (a = cron.stat_table.order(:t).where("t >= ?", begin_day.yesterday).first) ? a[:t] : nil
-      cron.stat_table.where("t >= ?", begin_day.tomorrow).delete # 明天的数据没出来肯定统计不了
-      timebegin = (a = cron.source.first) ? a.send(cron.time_column) : (DateTime.now - 1.second)
+      st_timebegin = (a = cron.stat_model.order(:t).where("t >= ?", begin_day.yesterday).first) ? a[:t] : nil
+
+      # TODO support multiple log
+      cron.stat_model.where("t >= ?", begin_day.tomorrow).delete # 明天的数据没出来肯定统计不了
+      timebegin = (multiple_dataset.first_time != DateTime1970) ? multiple_dataset.first_time : (DateTime.now - 1.second)
       timebegin = Time.at(timebegin) if is_time_column_integer?
       timebegin = (st_timebegin > timebegin) ? st_timebegin : timebegin if st_timebegin
 
       timeend = DateTime.now
-      puts "#{cron.source_name}'s range #{timebegin..timeend}"
+      logger.info "#{multiple_dataset.name}'s range #{timebegin..timeend}"
       # 把统计表的最后时间点也包含进去重新计算下
       TimeSeries.parse(timebegin..timeend, :unit => cron.time_unit)
     end
 
     protected
-    def is_mysql?; @_is_mysql ||= modules.grep(/ActiveRecord::Store/).any? end
-    def is_mongodb?; @_is_mongodb ||= modules.grep(/Mongoid::Document/).any? end
-    def modules; @_modules ||= cron.source.included_modules.map(&:to_s) end
 
     # 兼容采用整数类型作时间字段
     def is_time_column_integer?
-      if is_mysql?
-        cron.source.columns_hash[cron.time_column.to_s].type == :integer
+      if is_activerecord?
+        _source.columns_hash[cron.time_column.to_s].type == :integer
       else
         false
       end

data/lib/statlysis/cron/count.rb

@@ -5,89 +5,47 @@ module Statlysis
     def initialize source, opts = {}
       super
       Statlysis.check_set_database
-      cron.setup_stat_table
-      Statlysis.setup_stat_table_and_model cron
+      cron.setup_stat_model
       cron
     end
 
     # 设置数据源，并保存结果入数据库
     def run
-      cron.source          = cron.source.order("#{cron.time_column} ASC") if is_mysql?
-      cron.source          = cron.source.asc(cron.time_column) if is_mongodb?
-
-      (puts("#{cron.source_name} have no result!"); return false) if cron.output.blank?
+      (logger.info("#{cron.multiple_dataset.name} have no result!"); return false) if cron.output.blank?
       # delete first in range
       @output = cron.output
       unless @output.any?
-        puts "没有数据"; return
+        logger.info "没有数据"; return
       end
-      @num_i = 0; @num_add = 999
+      num_i = 0; num_add = 999
       Statlysis.sequel.transaction do
-        cron.stat_table.where("t >= ? AND t <= ?", cron.output[0][:t], cron.output[-1][:t]).delete
-        while !(_a = @output[@num_i..(@num_i+@num_add)]).blank? do
+        cron.stat_model.where("t >= ? AND t <= ?", cron.output[0][:t], cron.output[-1][:t]).delete
+        while !(_a = @output[num_i..(num_i+num_add)]).blank? do
           # batch insert all
-          cron.stat_table.multi_insert _a
-          @num_i += (@num_add + 1)
+          cron.stat_model.multi_insert _a
+          num_i += (num_add + 1)
         end
       end
+
+      return self
     end
 
 
-    def reoutput; @output = nil; output end
     protected
     def unit_range_query time, time_begin = nil
       # time begin and end
-      tb = time # TODO 差八个小时 [.in_time_zone, .localtime, .utc] 对于Rails，计算结果还是一样的。
+      tb = time
       te = (time+1.send(cron.time_unit)-1.second)
       tb, te = tb.to_i, te.to_i if is_time_column_integer?
       tb = time_begin || tb
-      return ["#{cron.time_column} >= ? AND #{cron.time_column} < ?", tb, te] if is_mysql?
-      return {cron.time_column => {"$gte" => tb.utc, "$lt" => te.utc}} if is_mongodb? # .utc  [fix undefined method `__bson_dump__' for Sun, 16 Dec 2012 16:00:00 +0000:DateTime]
+      return ["#{cron.time_column} >= ? AND #{cron.time_column} < ?", tb, te] if is_activerecord?
+      return {cron.time_column => {"$gte" => tb.utc, "$lt" => te.utc}} if is_mongoid? # .utc  [fix undefined method `__bson_dump__' for Sun, 16 Dec 2012 16:00:00 +0000:DateTime]
     end
 
   end
 
-  class Timely < Count
-    def setup_stat_table
-      # TODO migration proc, merge into setup_stat_table_and_model
-      cron.stat_table_name = [cron.class.name.split("::")[-1], cron.source_name, cron.source_where_array.join, cron.time_unit[0]].map {|s| s.to_s.gsub('_','') }.reject {|s| s.blank? }.join('_').downcase
-      raise "mysql only support table_name in 64 characters, the size of '#{cron.stat_table_name}' is #{cron.stat_table_name.to_s.size}. please set cron.stat_table_name when you create a Cron instance" if cron.stat_table_name.to_s.size > 64
-      unless Statlysis.sequel.table_exists?(cron.stat_table_name)
-        Statlysis.sequel.transaction do
-          Statlysis.sequel.create_table cron.stat_table_name, DefaultTableOpts do
-            DateTime :t # alias for :time
-          end
-
-          # TODO Add cron.source_where_array before count_columns
-          count_columns = [:timely_c, :totally_c] # alias for :count
-          count_columns.each {|w| Statlysis.sequel.add_column cron.stat_table_name, w, Integer }
-          index_column_names = [:t] + count_columns
-          index_column_names_name = index_column_names.join("_")
-          index_column_names_name = index_column_names_name[-63..-1] if index_column_names_name.size > 64
-
-          Statlysis.sequel.add_index cron.stat_table_name, index_column_names, :name => index_column_names_name
-        end
-      end
-    end
-
-    def output
-      @output ||= (cron.time_range.map do |time|
-        timely_c  = cron.source.where(unit_range_query(time)).count
-        _t = DateTime.parse("19700101")
-        _t = is_time_column_integer? ? _t.to_i : _t
-        totally_c = cron.source.where(unit_range_query(time, _t)).count
-
-        puts "#{time.in_time_zone} #{cron.source_name} timely_c:#{timely_c} totally_c:#{totally_c}"
-        if timely_c.zero? && totally_c.zero?
-          nil
-        else
-          {:t => time, :timely_c => timely_c, :totally_c => totally_c}
-        end
-      end.compact)
-    end
-  end
+end
 
-  class Dimensions < Count
-  end
 
-end
+require 'statlysis/cron/count/timely'
+require 'statlysis/cron/count/dimensions'

data/lib/statlysis/cron/count/dimensions.rb

@@ -0,0 +1,7 @@
+# encoding: UTF-8
+
+module Statlysis
+  class Dimensions < Count
+  end
+
+end

data/lib/statlysis/cron/count/timely.rb

@@ -0,0 +1,63 @@
+# encoding: UTF-8
+
+module Statlysis
+  class Timely < Count
+    def setup_stat_model
+      cron.stat_table_name = [cron.class.name.split("::")[-1], cron.multiple_dataset.name, cron.source_where_array.join, cron.time_unit[0]].map {|s| s.to_s.gsub('_','') }.reject {|s| s.blank? }.join('_').downcase
+      raise "mysql only support table_name in 64 characters, the size of '#{cron.stat_table_name}' is #{cron.stat_table_name.to_s.size}. please set cron.stat_table_name when you create a Cron instance" if cron.stat_table_name.to_s.size > 64
+
+      if not Statlysis.sequel.table_exists?(cron.stat_table_name)
+        Statlysis.sequel.transaction do
+          Statlysis.sequel.create_table cron.stat_table_name, DefaultTableOpts do
+            DateTime :t # alias for :time
+          end
+
+          # TODO Add cron.source_where_array before count_columns
+          count_columns = [:timely_c, :totally_c] # alias for :count
+          count_columns.each {|w| Statlysis.sequel.add_column cron.stat_table_name, w, Integer }
+          index_column_names = [:t] + count_columns
+          index_column_names_name = index_column_names.join("_")
+          index_column_names_name = index_column_names_name[-63..-1] if index_column_names_name.size > 64
+
+          # Fix there should be uniq index name between tables
+          # `SQLite3::SQLException: index t_timely_c_totally_c already exists (Sequel::DatabaseError)`
+          if not Statlysis.config.is_skip_database_index
+            Statlysis.sequel.add_index cron.stat_table_name, index_column_names, :name => index_column_names_name
+          end
+        end
+      end
+
+      n = cron.stat_table_name.to_s.singularize.camelize
+      cron.stat_model = class_eval <<-MODEL, __FILE__, __LINE__+1
+        class ::#{n} < Sequel::Model;
+          self.set_dataset :#{cron.stat_table_name}
+        end
+        #{n}
+      MODEL
+    end
+
+    def output
+      @output ||= (cron.time_range.map do |time|
+        timely_c = 0
+        totally_c = 0
+        # support multiple data sources
+        _first_source = nil
+        cron.multiple_dataset.sources.each do |s|
+          timely_c  += s.where(unit_range_query(time)).count
+          _t = DateTime1970
+          _t = is_time_column_integer? ? _t.to_i : _t
+          totally_c += s.where(unit_range_query(time, _t)).count
+          _first_source ||= s.where(unit_range_query(time))
+        end
+        logger.info "#{time.in_time_zone(cron.time_zone)} multiple_dataset:#{cron.multiple_dataset.name} _first_source:#{_first_source.inspect} timely_c:#{timely_c} totally_c:#{totally_c}" if ENV['DEBUG']
+
+        if timely_c.zero? && totally_c.zero?
+          nil
+        else
+          {:t => time, :timely_c => timely_c, :totally_c => totally_c}
+        end
+      end.compact)
+    end
+  end
+
+end

data/lib/statlysis/cron/top.rb

@@ -24,15 +24,7 @@ module Statlysis
       cron.write
     end
 
-    def write; raise DefaultWrongMessage end
-
-
-    def self.ensure_statlysis_table_and_model tn
-      Top.new("FakeLogSource", :test => true, :stat_table_name => tn).pattern_table_and_model tn
-    end
-    def ensure_statlysis_table_and_model tn
-      Top.ensure_statlysis_table_and_model tn
-    end
+    def write; raise DefaultNotImplementWrongMessage end
 
     def default_assign_attr key_symbol, opts
       if opts[key_symbol]
@@ -43,58 +35,6 @@ module Statlysis
     end
   end
 
-  # 博客最近用户访问计算实现流程讨论
-  # 问题分两个，一个是后端，一个是前端。对后端来说，用户每次blog/index|show访问都生成访问记录，后端需要进行排重和去掉未登陆用户。如果在该次访问里进行，特别是某个博客突然火了，必然每次访问都产生IO(磁盘或网络，因为多进程要共享信息），所以必定是异步的。
-  # 前端展示考虑到缓存，一般是页面片段缓存，或者ajax载入。
-  # 后端异步如何计算每个blog的最近访客，log.js记录了最近访问，一个后台常驻进程循环对日志表按时间记录来读取blog访问信息，把最近访客信息刷新到blog。相对单次请求全部处理，这里处理次数更少，资源更节约，当然瓶颈也在日志表的索引更新和读取。
-  class LastestVisits < Top
-    attr_accessor :clock
-    attr_accessor :reject_proc
-
-    # *pattern_proc* is a proc to extract user_id or url_prefix to compute the
-    # top visitors from log
-    # *user_id_proc* is a proc to extract user_id from log
-    # *user_info_proc* is a proc to extract visitor informations(like id, name, ...)
-    # *reject_proc* filter visitors
-    def initialize source, opts = {}
-      # set variables
-      cron.reclock opts[:default_time]
-      cron.reject_proc = opts[:reject_proc] || proc {|pattern, user_id| pattern.to_i == user_id.to_i }
-      super
-      cron.pattern_table_and_model cron.stat_table_name
-      cron
-    end
-
-    def output
-      cron.logs = cron.source.asc(cron.time_column).where(cron.time_column => {"$gte" => cron.clock.current}).limit(1000).to_a
-      return {} if cron.logs.blank?
-      cron.logs.inject({}) do |h, log|
-        pattern = cron.pattern_proc.call(log)
-        if pattern
-          h[pattern] ||= []
-          user_id = cron.user_id_proc.call(log).to_i
-          h[pattern] << user_id if not user_id.zero?
-        end
-        h
-      end
-    end
-
-    def write
-      puts "#{Time.now.strftime('%H:%M:%S')} #{cron.stat_model} #{cron.output.inspect}"
-      cron.output.each do |pattern, user_ids|
-        s = cron.stat_model.find_or_create(:pattern => pattern)
-        old_array = (JSON.parse(s.result) rescue []).map {|i| Array(i)[0] }
-        new_user_ids = (old_array + user_ids).reverse.uniq.reverse # ensure the right items will overwrite the left [1,4,5,7,4,3,3,2,1,5].uniq => [1, 4, 5, 7, 3, 2]
-        s.update :result => new_user_ids.reject {|user_id| cron.reject_proc.call(pattern, user_id) rescue false }.map {|user_id| cron.user_info_proc.call(user_id) }.compact[0..cron.result_limit].to_json
-      end
-      cron.clock.update cron.logs.last.try(cron.time_column)
-    end
-
-    def reclock default_time = nil
-      cron.clock = Clock.new cron.stat_table_name, (default_time || cron.clock.current)
-    end
-  end
-
   class SingleKv < Top
     attr_accessor :time_ago, :stat_column_name
 
@@ -102,53 +42,13 @@ module Statlysis
       [:time_ago, :stat_column_name].each {|key_symbol| default_assign_attr key_symbol, opts }
       raise "#{cron.class} only is kv store" if cron.stat_table_name # TODO
       super
-      cron.ensure_statlysis_table_and_model [Statlysis.tablename_default_pre, 'single_kvs'].compact.join("_").freeze
       cron
     end
 
   end
 
-  # 一般最近热门列表通常采用简单对一个字段记录访问数的算法，但是这可能会导致刷量等问题。
-  #
-  # 解决方法为从用户行为中去综合分析，具体流程为：
-  # 从URI中抽取item_id, 从访问日志抽取排重IP和user_id，从like,fav,comment表获取更深的用户行为，把前两者通过一定比例相加得到排行。
-  # 最后用时间降温来避免马太效应，必可动态提升比例以使最近稍微热门的替换掉之前太热门的。
-  #
-  # 线性计算速度很快
-  #
-  class HotestItems < SingleKv
-    attr_accessor :key, :id_to_score_and_time_hash_proc
-    attr_accessor :limit
-
-    def initialize key, id_to_score_and_time_hash_proc
-      cron.key = key
-      cron.id_to_score_and_time_hash_proc = id_to_score_and_time_hash_proc
-      cron.limit = 20
-      super
-      cron
-    end
-
-    def output
-      t = cron.id_to_score_and_time_hash_proc
-      while t.is_a?(Proc) do
-        t = t.call
-      end
-      @id_to_score_and_time_hash = t
-      @id_to_day_hash = @id_to_score_and_time_hash.inject({}) {|h, ab| h[ab[0]] = (((Time.now - ab[1][1]) / (3600*24)).round + 1); h }
-
-      @id_to_timecooldown_hash = @id_to_score_and_time_hash.inject({}) {|h, kv| h[kv[0]] = (kv[1][0] / Math.sqrt(@id_to_day_hash[kv[0]])); h }
-      array = @id_to_timecooldown_hash.sort {|a, b| b[1] <=> a[1] }.map(&:first)
-      {cron.key => array}
-    end
-
-    def write
-      cron.output.each do |key, array|
-        json = array[0..140].to_json
-        StSingleKv.find_or_create(:pattern => key).update :result => json
-        StSingleKvHistory.find_or_create(:pattern => "#{key}_#{Time.now.strftime('%Y%m%d')}").update :result => json
-      end
-    end
+end
 
-  end
 
-end
+require 'statlysis/cron/top/lastest_visits.rb'
+require 'statlysis/cron/top/hotest_items.rb'

data/lib/statlysis/cron/top/hotest_items.rb

@@ -0,0 +1,47 @@
+# encoding: UTF-8
+
+module Statlysis
+  # 一般最近热门列表通常采用简单对一个字段记录访问数的算法，但是这可能会导致刷量等问题。
+  #
+  # 解决方法为从用户行为中去综合分析，具体流程为：
+  # 从URI中抽取item_id, 从访问日志抽取排重IP和user_id，从like,fav,comment表获取更深的用户行为，把前两者通过一定比例相加得到排行。
+  # 最后用时间降温来避免马太效应，必可动态提升比例以使最近稍微热门的替换掉之前太热门的。
+  #
+  # 线性计算速度很快
+  #
+  class HotestItems < SingleKv
+    attr_accessor :key, :id_to_score_and_time_hash_proc
+    attr_accessor :limit
+
+    def initialize key, id_to_score_and_time_hash_proc
+      cron.key = key
+      cron.id_to_score_and_time_hash_proc = id_to_score_and_time_hash_proc
+      cron.limit = 20
+      super
+      cron
+    end
+
+    def output
+      t = cron.id_to_score_and_time_hash_proc
+      while t.is_a?(Proc) do
+        t = t.call
+      end
+      @id_to_score_and_time_hash = t
+      @id_to_day_hash = @id_to_score_and_time_hash.inject({}) {|h, ab| h[ab[0]] = (((Time.now - ab[1][1]) / (3600*24)).round + 1); h }
+
+      @id_to_timecooldown_hash = @id_to_score_and_time_hash.inject({}) {|h, kv| h[kv[0]] = (kv[1][0] / Math.sqrt(@id_to_day_hash[kv[0]])); h }
+      array = @id_to_timecooldown_hash.sort {|a, b| b[1] <=> a[1] }.map(&:first)
+      {cron.key => array}
+    end
+
+    def write
+      cron.output.each do |key, array|
+        json = array[0..140].to_json
+        StSingleKv.find_or_create(:pattern => key).update :result => json
+        StSingleKvHistory.find_or_create(:pattern => "#{key}_#{Time.now.strftime('%Y%m%d')}").update :result => json
+      end
+    end
+
+  end
+
+end