statlysis 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,51 @@
+# rcov generated
+coverage
+coverage.data
+
+# rdoc generated
+rdoc
+
+# yard generated
+doc
+.yardoc
+
+# bundler
+.bundle
+
+# jeweler generated
+pkg
+
+# Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore: 
+#
+# * Create a file at ~/.gitignore
+# * Include files you want ignored
+# * Run: git config --global core.excludesfile ~/.gitignore
+#
+# After doing this, these files will be ignored in all your git projects,
+# saving you from having to 'pollute' every project you touch with them
+#
+# Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
+#
+# For MacOS:
+#
+.DS_Store
+
+# For TextMate
+*.tmproj
+tmtags
+
+# For emacs:
+*~
+\#*
+.\#*
+
+# For vim:
+*.swp
+
+# For redcar:
+.redcar
+
+# For rubinius:
+*.rbc
+
+coverage

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,110 @@
+PATH
+  remote: .
+  specs:
+    statlysis (0.0.1)
+      activerecord
+      activerecord_idnamecache
+      activesupport
+      mongoid
+      mysql2
+      only_one_rake
+      rails
+      rake
+      sequel
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionmailer (3.2.13)
+      actionpack (= 3.2.13)
+      mail (~> 2.5.3)
+    actionpack (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.4)
+      rack (~> 1.4.5)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.2.1)
+    activemodel (3.2.13)
+      activesupport (= 3.2.13)
+      builder (~> 3.0.0)
+    activerecord (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activerecord_idnamecache (0.1)
+    activeresource (3.2.13)
+      activemodel (= 3.2.13)
+      activesupport (= 3.2.13)
+    activesupport (3.2.13)
+      i18n (= 0.6.1)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.4)
+    erubis (2.7.0)
+    hike (1.2.3)
+    i18n (0.6.1)
+    journey (1.0.4)
+    json (1.8.0)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.23)
+    mongoid (3.1.4)
+      activemodel (~> 3.2)
+      moped (~> 1.4)
+      origin (~> 1.0)
+      tzinfo (~> 0.3.22)
+    moped (1.5.0)
+    multi_json (1.7.7)
+    mysql2 (0.3.11)
+    only_one_rake (0.0.4)
+    origin (1.1.0)
+    polyglot (0.3.3)
+    rack (1.4.5)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-ssl (1.3.3)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rails (3.2.13)
+      actionmailer (= 3.2.13)
+      actionpack (= 3.2.13)
+      activerecord (= 3.2.13)
+      activeresource (= 3.2.13)
+      activesupport (= 3.2.13)
+      bundler (~> 1.0)
+      railties (= 3.2.13)
+    railties (3.2.13)
+      actionpack (= 3.2.13)
+      activesupport (= 3.2.13)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (>= 0.14.6, < 2.0)
+    rake (10.1.0)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    sequel (4.0.0)
+    sprockets (2.2.2)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    thor (0.18.1)
+    tilt (1.4.1)
+    treetop (1.4.14)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.37)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  statlysis!

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 David Chen at eoe.cn
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,43 @@
+statlysis
+===============================================
+statistical analysis in ruby dsl
+
+Usage
+-----------------------------------------------
+```ruby
+module Statlysis
+  set_database :statlysis
+  update_time_columns :t
+  set_tablename_default_pre :st
+
+  # 初始化键值model
+  Statlysis::Top.new('', :test => true).pattern_table_and_model 'st_single_kvs'
+  Statlysis::Top.new('', :test => true).pattern_table_and_model 'st_single_kv_histories'
+
+  # 日常count
+  EoeLog.class # preload EoeLogTest
+  @log_model = IS_DEVELOP ? EoeLogTest : EoeLog
+  hourly @log_model, :t
+  daily  @log_model, :t
+  daily  @log_model.where(:ui => 0), :t
+  daily  @log_model.where(:ui => {"$ne" => 0}), :t
+
+  # 统计各个模块
+  daily  @log_model.where(:do => {"$in" => [DOMAINS_HASH[:blog], DOMAINS_HASH[:my]]}), :t
+  [:www, :code, :skill, :book, :edu, :news, :wiki, :salon, :android].each do |site|
+    daily  @log_model.where(:do => DOMAINS_HASH[site]), :t
+  end
+```
+
+TODO
+-----------------------------------------------
+1. Admin interface
+2. statistical query api in Ruby and HTTP
+3. Interacting with Javascript charting library, e.g. Highcharts, D3.
+4. Add namespace to DSL, like rake
+5. More tests
+
+
+Copyright
+-----------------------------------------------
+MIT. David Chen at eoe.cn.

data/Rakefile

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+
+require 'rake'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+ t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/lib/statlysis.rb

@@ -0,0 +1,134 @@
+# encoding: UTF-8
+# Sequel的操作均需通过Symbol
+#
+# 删除匹配的统计表
+# Statlysis.sequel.tables.select {|i| i.to_s.match(//i) }.each {|i| Statlysis.sequel.drop_table i }
+
+require "active_support/all"
+require 'active_support/core_ext/module/attribute_accessors.rb'
+require 'active_record'
+require 'rails'
+%w[yaml sequel only_one_rake mongoid].map(&method(:require))
+
+module Statlysis
+  Units = %w[hour day week month year]
+  DefaultTableOpts = {:charset => "utf8", :collate => "utf8_general_ci", :engine => "MyISAM"}
+
+  def self.setup_stat_table_and_model cron, tablename = nil
+    tablename = cron.stat_table_name if tablename.nil?
+    tablename ||= cron.stat_table.first_source_table
+    cron.stat_table = Statlysis.sequel[tablename.to_sym]
+
+    str = tablename.to_s.singularize.camelize
+    eval("class ::#{str} < Sequel::Model;
+      self.set_dataset :#{tablename}
+      def self.[] item_id
+        JSON.parse(find_or_create(:pattern => item_id).result) rescue []
+      end
+    end; ")
+    cron.stat_model = str.constantize
+  end
+
+end
+
+require 'statlysis/common'
+require 'statlysis/timeseries'
+require 'statlysis/clock'
+require 'statlysis/rake'
+require 'statlysis/cron'
+require 'statlysis/similar'
+
+module Statlysis
+  mattr_accessor :sequel, :default_time_columns, :database_opts, :tablename_default_pre
+  Units.each {|unit| module_eval "mattr_accessor :#{unit}_crons; self.#{unit}_crons = []" }
+  [:realtime, :similar, :hotest].each do |sym|
+    sym = "#{sym}_crons".to_sym
+    mattr_accessor sym; self.send "#{sym}=", []
+  end
+  # TODO _crons uniq, no readd
+  extend self
+
+  # 会在自动拼接统计数据库表名时去除这些时间字段
+  def update_time_columns *columns
+    self.default_time_columns ||= [:created_at, :updated_at]
+    columns.each {|column| self.default_time_columns.push column }
+    self.default_time_columns = self.default_time_columns.uniq
+  end
+
+  def set_database sym_or_hash
+    self.database_opts = if sym_or_hash.is_a? Symbol
+      YAML.load_file(Rails.root.join("config/database.yml"))[sym_or_hash.to_s]
+    elsif Hash
+      sym_or_hash
+    else
+      raise "Statlysis#set_database only support symbol or hash params"
+    end
+    self.sequel = Sequel.connect self.database_opts.except('database')
+    self.sequel.execute("CREATE DATABASE IF NOT EXISTS #{self.database_opts['database']} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;")
+    self.sequel.use self.database_opts['database']
+    # Statlysis.sequel.tables.map {|t| eval "class ::#{t.to_s.camelize} < ActiveRecord::Base; self.establish_connection Statlysis.database_opts; self.table_name = :#{t}; end; #{t.to_s.camelize}" }
+  end
+
+  def set_tablename_default_pre str
+    self.tablename_default_pre = str.to_s
+  end
+
+  def daily source, time_column = :created_at; timely source, :time_unit => :day, :time_column => time_column end
+  def hourly source, time_column = :created_at; timely source, :time_unit => :hour, :time_column => time_column end
+
+  def check_set_database; raise "Please setup database first" if sequel.nil?  end
+
+  def timely source, opts
+    self.check_set_database
+    opts.reverse_merge! :time_column => :created_at, :time_unit => :day
+    t = Timely.new source, opts
+    module_eval("self.#{opts[:time_unit]}_crons").push t
+  end
+
+  # the real requirement is to compute lastest items group by special pattens, like user_id, url prefix, ...
+  def lastest_visits source, opts
+    self.check_set_database
+    opts.reverse_merge! :time_column => :created_at
+    self.realtime_crons.push LastestVisits.new(source, opts)
+  end
+
+  # TODO 为什么一层proc的话会直接执行的
+  def hotest_items key, id_to_score_and_time_hash = {}
+    _p = proc { if block_given?
+      (proc do
+        id_to_score_and_time_hash = Hash.new
+        yield id_to_score_and_time_hash
+        id_to_score_and_time_hash
+      end)
+    else
+      (proc { id_to_score_and_time_hash })
+    end}
+
+    self.hotest_crons.push HotestItems.new(key, _p)
+  end
+
+  # TODO support mongoid
+  def similar_items model_name, id_to_text_hash = {}
+    _p = if block_given?
+      (proc do
+        id_to_text_hash = Hash.new {|hash, key| hash[key] = "" }
+        yield id_to_text_hash
+        id_to_text_hash
+      end)
+    else
+      (proc { id_to_text_hash })
+    end
+
+    self.similar_crons.push Similar.new(model_name, _p)
+  end
+
+end
+
+
+module Statlysis
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load File.expand_path('../statlysis/rake.rb', __FILE__)
+    end
+  end if defined? Rails
+end

data/lib/statlysis/clock.rb

@@ -0,0 +1,36 @@
+# encoding: UTF-8
+
+module Statlysis
+  DateTime1970 = DateTime.parse("19700101").in_time_zone
+
+  class Clock
+    attr_accessor :clock
+    include Common
+
+    def initialize feature, default_time
+      raise "Please assign default_time params" if not default_time
+      cron.stat_table_name = [Statlysis.tablename_default_pre, 'clocks'].compact.join("_")
+      unless Statlysis.sequel.table_exists?(cron.stat_table_name)
+        Statlysis.sequel.create_table cron.stat_table_name, DefaultTableOpts.merge(:engine => "InnoDB") do
+          primary_key :id
+          String      :feature
+          DateTime    :t
+          index       :feature, :unique => true
+        end
+      end
+      Statlysis.setup_stat_table_and_model cron
+      cron.clock = cron.stat_model.find_or_create(:feature => feature)
+      cron.clock.update :t => default_time if cron.current.nil?
+      cron
+    end
+
+    def update time
+      time = DateTime.now if time == DateTime1970
+      return false if time && (time < cron.current)
+      cron.clock.update :t => time
+    end
+
+    def current; cron.clock.t end
+  end
+
+end

data/lib/statlysis/common.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+module Statlysis
+  module Common
+    attr_accessor :stat_table_name, :stat_model, :stat_table
+    def pattern_table_and_model tn
+      # ensure statlysis table
+      tn = tn.pluralize
+      unless Statlysis.sequel.table_exists?(tn)
+        Statlysis.sequel.create_table tn, DefaultTableOpts.merge(:engine => "InnoDB") do
+          primary_key :id
+          String :pattern
+          index  :pattern
+        end
+        Statlysis.sequel.add_column tn, :result, String, :text => true
+      end
+
+      # generate a statlysis model
+      cron.stat_model = Statlysis.setup_stat_table_and_model cron, tn
+    end
+
+    def cron; self end
+    # TODO remove puts, conflict user, user logger
+    def puts(*strs); $stdout.puts(*strs) if ENV['DEBUG'] end
+
+  end
+end