statlysis 0.0.1 → 0.0.2

data/.gitignore

@@ -49,3 +49,6 @@ tmtags
 *.rbc
 
 coverage
+*.gem
+Gemfile.lock
+tmp

data/Guardfile

@@ -0,0 +1,14 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :test do
+  watch(%r{^lib/(.+)\.rb$})     { |m| "test/#{m[1]}_test.rb" }
+  watch(%r{^test/.+_test\.rb$})
+  watch('test/test_helper.rb')  { "test" }
+
+  # Rails example
+  watch(%r{^app/models/(.+)\.rb$})                   { |m| "test/unit/#{m[1]}_test.rb" }
+  watch(%r{^app/controllers/(.+)\.rb$})              { |m| "test/functional/#{m[1]}_test.rb" }
+  watch(%r{^app/views/.+\.rb$})                      { "test/integration" }
+  watch('app/controllers/application_controller.rb') { ["test/functional", "test/integration"] }
+end

data/README.markdown

@@ -1,43 +1,93 @@
-statlysis
+Statlysis
 ===============================================
-statistical analysis in ruby dsl
+Statistical & Analysis in Ruby DSL
 
 Usage
 -----------------------------------------------
+### setup
+
 ```ruby
-module Statlysis
+Statlysis.setup do
   set_database :statlysis
-  update_time_columns :t
-  set_tablename_default_pre :st
-
-  # 初始化键值model
-  Statlysis::Top.new('', :test => true).pattern_table_and_model 'st_single_kvs'
-  Statlysis::Top.new('', :test => true).pattern_table_and_model 'st_single_kv_histories'
-
-  # 日常count
-  EoeLog.class # preload EoeLogTest
-  @log_model = IS_DEVELOP ? EoeLogTest : EoeLog
-  hourly @log_model, :t
-  daily  @log_model, :t
-  daily  @log_model.where(:ui => 0), :t
-  daily  @log_model.where(:ui => {"$ne" => 0}), :t
-
-  # 统计各个模块
-  daily  @log_model.where(:do => {"$in" => [DOMAINS_HASH[:blog], DOMAINS_HASH[:my]]}), :t
-  [:www, :code, :skill, :book, :edu, :news, :wiki, :salon, :android].each do |site|
-    daily  @log_model.where(:do => DOMAINS_HASH[site]), :t
+
+  hourly :time_column => :t
+  [EoeLog,
+   EoeLog.where(:ui => 0),
+   EoeLog.where(:ui => {"$ne" => 0}),
+   Mongoid[/eoe_logs_[0-9]+$/].where(:ui => {"$ne" => 0}),
+   EoeLog.where(:do => {"$in" => [DOMAINS_HASH[:blog], DOMAINS_HASH[:my]]}),
+  ].each do |s|
+    daily s, :time_column => :t
   end
+end
+```
+
+### access
+
+```ruby
+Statlysis.daily # => return daily crons
+Statlysis.daily.run # => run daily crons
+Statlysis.daily[/name_regexp/] # => return matched daily crons
+```
+
+### process
+
+```irb
+[23] pry(#<Statlysis::Configuration>)> Statlysis.daily['multi'].first
 ```
 
+Features
+-----------------------------------------------
+* Support time column that stored as integer.
+
 TODO
 -----------------------------------------------
-1. Admin interface
-2. statistical query api in Ruby and HTTP
-3. Interacting with Javascript charting library, e.g. Highcharts, D3.
-4. Add namespace to DSL, like rake
-5. More tests
+* Admin interface
+* statistical query api in Ruby and HTTP
+* Interacting with Javascript charting library, e.g. Highcharts, D3.
+* More tests
+* Add @criteria to MultipleDataset
+
+
+Statistical Process
+-----------------------------------------------
+1. Delete invalid statistical data, e.g. data in tomorrow
+2. Count data within the specified time by the dimensions
+3. Delete overlapping data, and insert new data
+
+
+FAQ
+-----------------------------------------------
+Q: Why use Sequel instead of ActiveRecord?
+
+A: When initialize an ORM object, ActiveRecord is 3 times slower than Sequel, and we just need the basic operations, including read, write, enumerate, etc. See more details in [Quick dive into Ruby ORM object initialization](http://merbist.com/2012/02/23/quick-dive-into-ruby-orm-object-initialization/) .
+
+
+Q: Why do you recommend using multiple collections to store logs rather than a single collection, or a capped collection?
+
+A: MongoDB can effectively reuse space freed by removing entire collections without leading to data fragmentation, see details at http://docs.mongodb.org/manual/use-cases/storing-log-data/#multiple-collections-single-database
 
 
 Copyright
 -----------------------------------------------
 MIT. David Chen at eoe.cn.
+
+
+Related
+-----------------------------------------------
+### Projects
+* https://github.com/paulasmuth/fnordmetric FnordMetric is a redis/ruby-based realtime Event-Tracking app
+* https://github.com/thirtysixthspan/descriptive_statistics adds methods to the Enumerable module to allow easy calculation of basic descriptive statistics for a set of data
+* https://github.com/tmcw/simple-statistics simple statistics for javascript in node and the browser
+* https://github.com/clbustos/statsample/  A suite for basic and advanced statistics on Ruby. 
+* https://github.com/SciRuby/sciruby Tools for scientific computation in Ruby/Rails
+
+### Articles
+* http://www.slideshare.net/WombatNation/logging-app-behavior-to-mongo-db
+
+### Event collector
+* https://github.com/fluent
+* https://github.com/logstash/logstash
+
+### Admin interface
+* http://three.kibana.org/ browser based analytics and search interface to Logstash and other timestamped data sets stored in ElasticSearch.

data/Rakefile

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 
-require 'rake'
+require "bundler/gem_tasks"
 require 'rake/testtask'
 
 Rake::TestTask.new do |t|

data/lib/statlysis.rb

@@ -1,134 +1,92 @@
 # encoding: UTF-8
+#
 # Sequel的操作均需通过Symbol
 #
 # 删除匹配的统计表
 # Statlysis.sequel.tables.select {|i| i.to_s.match(//i) }.each {|i| Statlysis.sequel.drop_table i }
 
+# TODO Statlysis.sequel.tables.map {|t| eval "class ::#{t.to_s.camelize} < ActiveRecord::Base; self.establish_connection Statlysis.database_opts; self.table_name = :#{t}; end; #{t.to_s.camelize}" }
+
 require "active_support/all"
+Time.zone ||= Time.now.utc_offset # require activesupport
+
+require "active_support/core_ext"
 require 'active_support/core_ext/module/attribute_accessors.rb'
 require 'active_record'
-require 'rails'
-%w[yaml sequel only_one_rake mongoid].map(&method(:require))
+require 'activerecord_idnamecache'
+%w[yaml sequel mongoid].map(&method(:require))
 
-module Statlysis
-  Units = %w[hour day week month year]
-  DefaultTableOpts = {:charset => "utf8", :collate => "utf8_general_ci", :engine => "MyISAM"}
-
-  def self.setup_stat_table_and_model cron, tablename = nil
-    tablename = cron.stat_table_name if tablename.nil?
-    tablename ||= cron.stat_table.first_source_table
-    cron.stat_table = Statlysis.sequel[tablename.to_sym]
-
-    str = tablename.to_s.singularize.camelize
-    eval("class ::#{str} < Sequel::Model;
-      self.set_dataset :#{tablename}
-      def self.[] item_id
-        JSON.parse(find_or_create(:pattern => item_id).result) rescue []
-      end
-    end; ")
-    cron.stat_model = str.constantize
-  end
-
-end
+# Fake a Rails environment
+module Rails; end
 
-require 'statlysis/common'
-require 'statlysis/timeseries'
-require 'statlysis/clock'
-require 'statlysis/rake'
-require 'statlysis/cron'
-require 'statlysis/similar'
+require 'statlysis/constants'
 
 module Statlysis
-  mattr_accessor :sequel, :default_time_columns, :database_opts, :tablename_default_pre
-  Units.each {|unit| module_eval "mattr_accessor :#{unit}_crons; self.#{unit}_crons = []" }
-  [:realtime, :similar, :hotest].each do |sym|
-    sym = "#{sym}_crons".to_sym
-    mattr_accessor sym; self.send "#{sym}=", []
-  end
-  # TODO _crons uniq, no readd
-  extend self
-
-  # 会在自动拼接统计数据库表名时去除这些时间字段
-  def update_time_columns *columns
-    self.default_time_columns ||= [:created_at, :updated_at]
-    columns.each {|column| self.default_time_columns.push column }
-    self.default_time_columns = self.default_time_columns.uniq
-  end
+  class << self
+    def setup &blk
+      raise "Need to setup proc" if not blk
 
-  def set_database sym_or_hash
-    self.database_opts = if sym_or_hash.is_a? Symbol
-      YAML.load_file(Rails.root.join("config/database.yml"))[sym_or_hash.to_s]
-    elsif Hash
-      sym_or_hash
-    else
-      raise "Statlysis#set_database only support symbol or hash params"
+      logger.info "Start to setup Statlysis"
+      time_log do
+        self.config.instance_exec(&blk)
+      end
+      logger.info
     end
-    self.sequel = Sequel.connect self.database_opts.except('database')
-    self.sequel.execute("CREATE DATABASE IF NOT EXISTS #{self.database_opts['database']} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;")
-    self.sequel.use self.database_opts['database']
-    # Statlysis.sequel.tables.map {|t| eval "class ::#{t.to_s.camelize} < ActiveRecord::Base; self.establish_connection Statlysis.database_opts; self.table_name = :#{t}; end; #{t.to_s.camelize}" }
-  end
 
-  def set_tablename_default_pre str
-    self.tablename_default_pre = str.to_s
-  end
+    def time_log text = nil
+      t = Time.now
+      logger.info text if text
+      yield if block_given?
+      logger.info "Time spend #{(Time.now - t).round(2)} seconds."
+      logger.info "-" * 42
+    end
 
-  def daily source, time_column = :created_at; timely source, :time_unit => :day, :time_column => time_column end
-  def hourly source, time_column = :created_at; timely source, :time_unit => :hour, :time_column => time_column end
+    # delagate config methods to Configuration
+    def config; Configuration.instance end
+    require 'active_support/core_ext/module/delegation.rb'
+    [:sequel, :set_database, :check_set_database,
+     :default_time_zone,
+     :set_tablename_default_pre, :tablename_default_pre
+    ].each do |sym|
+      delegate sym, :to => :config
+    end
 
-  def check_set_database; raise "Please setup database first" if sequel.nil?  end
+    attr_accessor :logger
+    Statlysis.logger ||= Logger.new($stdout)
 
-  def timely source, opts
-    self.check_set_database
-    opts.reverse_merge! :time_column => :created_at, :time_unit => :day
-    t = Timely.new source, opts
-    module_eval("self.#{opts[:time_unit]}_crons").push t
-  end
+    def source_to_database_type; @_source_to_database_type ||= {} end
 
-  # the real requirement is to compute lastest items group by special pattens, like user_id, url prefix, ...
-  def lastest_visits source, opts
-    self.check_set_database
-    opts.reverse_merge! :time_column => :created_at
-    self.realtime_crons.push LastestVisits.new(source, opts)
-  end
 
-  # TODO 为什么一层proc的话会直接执行的
-  def hotest_items key, id_to_score_and_time_hash = {}
-    _p = proc { if block_given?
-      (proc do
-        id_to_score_and_time_hash = Hash.new
-        yield id_to_score_and_time_hash
-        id_to_score_and_time_hash
-      end)
-    else
-      (proc { id_to_score_and_time_hash })
-    end}
-
-    self.hotest_crons.push HotestItems.new(key, _p)
+    def daily; CronSet.new(Statlysis.config.day_crons) end
+    def hourly; CronSet.new(Statlysis.config.hour_crons) end
+
   end
 
-  # TODO support mongoid
-  def similar_items model_name, id_to_text_hash = {}
-    _p = if block_given?
-      (proc do
-        id_to_text_hash = Hash.new {|hash, key| hash[key] = "" }
-        yield id_to_text_hash
-        id_to_text_hash
-      end)
-    else
-      (proc { id_to_text_hash })
-    end
+end
 
-    self.similar_crons.push Similar.new(model_name, _p)
-  end
+require 'statlysis/utils'
+require 'statlysis/configuration'
+require 'statlysis/common'
+require 'statlysis/timeseries'
+require 'statlysis/clock'
+require 'statlysis/rake'
+require 'statlysis/cron'
+require 'statlysis/cron_set'
+require 'statlysis/similar'
+require 'statlysis/multiple_dataset'
 
+module Statlysis
+  require 'short_inspect'
+  ShortInspect.apply_to Cron, CronSet, MultipleDataset
+  ShortInspect.apply_minimal_to ActiveRecord::Relation # lazy load
 end
 
 
+# load rake tasks
 module Statlysis
   class Railtie < Rails::Railtie
     rake_tasks do
       load File.expand_path('../statlysis/rake.rb', __FILE__)
     end
-  end if defined? Rails
-end
+  end
+end if defined? Rails::Railtie

data/lib/statlysis/clock.rb

@@ -1,12 +1,11 @@
 # encoding: UTF-8
 
 module Statlysis
-  DateTime1970 = DateTime.parse("19700101").in_time_zone
-
   class Clock
     attr_accessor :clock
     include Common
 
+    # feature is a string
     def initialize feature, default_time
       raise "Please assign default_time params" if not default_time
       cron.stat_table_name = [Statlysis.tablename_default_pre, 'clocks'].compact.join("_")
@@ -18,7 +17,8 @@ module Statlysis
           index       :feature, :unique => true
         end
       end
-      Statlysis.setup_stat_table_and_model cron
+      h = Utils.setup_pattern_table_and_model cron.stat_table_name
+      cron.stat_model = h[:model]
       cron.clock = cron.stat_model.find_or_create(:feature => feature)
       cron.clock.update :t => default_time if cron.current.nil?
       cron

data/lib/statlysis/common.rb

@@ -2,26 +2,14 @@
 
 module Statlysis
   module Common
-    attr_accessor :stat_table_name, :stat_model, :stat_table
-    def pattern_table_and_model tn
-      # ensure statlysis table
-      tn = tn.pluralize
-      unless Statlysis.sequel.table_exists?(tn)
-        Statlysis.sequel.create_table tn, DefaultTableOpts.merge(:engine => "InnoDB") do
-          primary_key :id
-          String :pattern
-          index  :pattern
-        end
-        Statlysis.sequel.add_column tn, :result, String, :text => true
-      end
+    extend ActiveSupport::Concern
 
-      # generate a statlysis model
-      cron.stat_model = Statlysis.setup_stat_table_and_model cron, tn
+    self.included do
+      attr_accessor :stat_table_name, :stat_model
     end
 
     def cron; self end
-    # TODO remove puts, conflict user, user logger
-    def puts(*strs); $stdout.puts(*strs) if ENV['DEBUG'] end
+    delegate :logger, :to => Statlysis
 
   end
 end

data/lib/statlysis/configuration.rb

@@ -1,9 +1,104 @@
 # encoding: UTF-8
+#
+# see original implementation at http://mvj3.github.io/2013/04/17/statlysis-analysis-design-solve-two-problems-lazy-loading-and-scope/
+#
+
+require 'singleton'
 
 module Statlysis
-  # TODO config methods here
   class Configuration
-    def inherited(base)
+    include Singleton
+
+    attr_accessor :sequel, :default_time_columns, :default_time_zone, :database_opts, :tablename_default_pre
+    attr_accessor :is_skip_database_index
+    TimeUnits.each {|unit| module_eval "attr_accessor :#{unit}_crons; self.instance.#{unit}_crons = []" }
+    [:realtime, :similar, :hotest].each do |sym|
+      sym = "#{sym}_crons"
+      attr_accessor sym; self.instance.send "#{sym}=", []
+    end
+    self.instance.send "tablename_default_pre=", "st"
+    self.instance.send "is_skip_database_index=", false
+
+    # 会在自动拼接统计数据库表名时去除这些时间字段
+    def update_time_columns *columns
+      self.default_time_columns ||= [:created_at, :updated_at]
+      columns.each {|column| self.default_time_columns.push column }
+      self.default_time_columns = self.default_time_columns.uniq
+    end
+
+    def set_database sym_or_hash
+      self.database_opts = if sym_or_hash.is_a? Symbol
+        YAML.load_file(Rails.root.join("config/database.yml"))[sym_or_hash.to_s]
+      elsif Hash
+        sym_or_hash
+      else
+        raise "Statlysis#set_database only support symbol or hash params"
+      end
+      self.sequel = Sequel.connect(self.database_opts)
+
+      # 初始化键值model
+      ["#{self.tablename_default_pre}_single_kvs", "#{self.tablename_default_pre}_single_kv_histories"].each do |tn|
+        Utils.setup_pattern_table_and_model tn
+      end
+      return self
+    end
+
+    def set_default_time_zone zone
+      self.default_time_zone = zone
+      return self
+    end
+
+    def set_tablename_default_pre str
+      self.tablename_default_pre = str.to_s
+    end
+
+    def daily  source, opts = {}; timely source, {:time_unit => :day }.merge(opts) end
+    def hourly source, opts = {}; timely source, {:time_unit => :hour}.merge(opts) end
+
+    def check_set_database; raise "Please setup database first" if sequel.nil?  end
+
+    def timely source, opts
+      self.check_set_database
+      opts.reverse_merge! :time_column => :created_at, :time_unit => :day
+      t = Timely.new source, opts
+      self.send("#{opts[:time_unit]}_crons").push t
+    end
+
+    # the real requirement is to compute lastest items group by special pattens, like user_id, url prefix, ...
+    def lastest_visits source, opts
+      self.check_set_database
+      opts.reverse_merge! :time_column => :created_at
+      self.realtime_crons.push LastestVisits.new(source, opts)
+    end
+
+    # TODO 为什么一层proc的话会直接执行的
+    def hotest_items key, id_to_score_and_time_hash = {}
+      _p = proc { if block_given?
+        (proc do
+          id_to_score_and_time_hash = Hash.new
+          yield id_to_score_and_time_hash
+          id_to_score_and_time_hash
+        end)
+      else
+        (proc { id_to_score_and_time_hash })
+      end}
+
+      self.hotest_crons.push HotestItems.new(key, _p)
+    end
+
+    # TODO support mongoid
+    def similar_items model_name, id_to_text_hash = {}
+      _p = if block_given?
+        (proc do
+          id_to_text_hash = Hash.new {|hash, key| hash[key] = "" }
+          yield id_to_text_hash
+          id_to_text_hash
+        end)
+      else
+        (proc { id_to_text_hash })
+      end
+
+      self.similar_crons.push Similar.new(model_name, _p)
     end
 
   end