statlysis 0.0.1

data/lib/statlysis/results.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+
+module Statlysis
+  class Results
+    attr_accessor :data
+    # 1, inline
+    # 2, collection
+    def initialize data
+      self.data = data
+      self
+    end
+
+    def output
+      self.data.to_a
+    end
+  end
+end

data/lib/statlysis/similar.rb

@@ -0,0 +1,89 @@
+# encoding: UTF-8
+# TODO support mongoid
+
+module Statlysis
+  class Similar
+    attr_accessor :id_to_text_hash_proc, :corpus, :matrix, :id_to_similar_ids
+    include Common
+
+    def initialize model_name, id_to_text_hash_proc
+      # 初始化数据
+      cron.id_to_text_hash_proc = id_to_text_hash_proc
+
+      # 初始化表和模型
+      cron.stat_table_name = [Statlysis.tablename_default_pre, "similar", model_name].compact.join("_")
+      cron.pattern_table_and_model cron.stat_table_name
+
+      cron.id_to_similar_ids = {}
+      cron
+    end
+
+    def process
+      puts "SimilarProcess #{cron.stat_model} at #{DateTime.now}"
+      require 'gsl'
+      require 'tf-idf-similarity'
+
+      # 初始化文档
+      puts "开始取出 cron.id_to_text_hash_proc"
+      @id_to_text_hash = cron.id_to_text_hash_proc.call
+
+      puts "开始把@id_to_text_hash转化为数组"
+      as = @id_to_text_hash.to_a
+
+      puts "开始把as slice为1200每次"
+      as.each_slice(1200) do |a|
+      puts "开始跑 #{a.size}  个条目的相似性"
+      cron.corpus = TfIdfSimilarity::Collection.new
+      a.each do |id, text|
+        cron.corpus << TfIdfSimilarity::Document.new(text, :id => id)
+      end
+=begin
+      @id_to_text_hash.each do |id, text|
+        cron.corpus << TfIdfSimilarity::Document.new(text, :id => id)
+      end
+=end
+
+      cron.matrix = cron.corpus.similarity_matrix
+      matrix_array = cron.matrix.to_a
+
+      # matrix的数组下标对应到真实的item_id
+      matrix_idx_to_item_id_hash = {}
+      cron.corpus.documents.each_with_index do |document, idx1|
+        matrix_idx_to_item_id_hash[idx1] = document.id
+      end
+
+      # 取出matrix里各item的按相关度倒序的item_ids，并保存
+      cron.corpus.documents.each_with_index do |document, idx1|
+        _item_id_to_score = Hash.new 0
+        matrix_array[idx1].each_with_index do |num, idx2|
+          _item_id_to_score[matrix_idx_to_item_id_hash[idx2]] = (num.nan? ? 0.0 : num)
+        end
+        _item_id_to_score.delete document.id
+        puts "对比文档："
+        puts "#{document.id} # #{summary(document.id)}"
+        puts "相关文档："
+        _item_ids = _item_id_to_score.sort {|a1, b1| b1[1] <=> a1[1] }
+        _item_ids[0..9].each do |item_id, score|
+          puts "#{score} #  #{summary(item_id)}"
+        end
+        cron.id_to_similar_ids[document.id] = _item_ids[0..99].map(&:first)
+        puts
+      end
+
+      # save results to database
+      cron.id_to_similar_ids.each do |id, similar_ids|
+        s = cron.stat_model.find_or_create(:pattern => id)
+        s.update :result => similar_ids.to_json
+      end
+      end # @id_to_text_hash.to_a.each_slice(1000) do |a|
+
+      return true
+    end
+    alias run process
+
+    def summary doc_id
+      @id_to_text_hash[doc_id].mb_chars[0..41].split("\n").join
+    end
+
+  end
+end

data/lib/statlysis/timeseries.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+module Statlysis
+  module TimeSeries
+    # range支持如下三种时间范围格式
+    #   20121201 20121221
+    #   DateTime.parse('20121221')
+    #   DateTime.parse('20121201')..DateTime.parse('20121221')
+    # opts[:unit]支持:hour, :day, :week, :month等时间单位
+    # 返回的结果为时间范围内的序列数组
+    def self.parse range, opts = {}
+      opts = opts.reverse_merge :unit => :day, :utc => true, :offset => nil
+      unit = opts[:unit]
+
+      range = Range.new(*range.split.map {|i| DateTime.parse(i).to_time_in_current_zone }) if range.is_a?(String)
+
+      begin_unit = "beginning_of_#{unit}".to_sym
+      array = if range.respond_to?(:to_datetime)
+        [range.in_time_zone.send(begin_unit)]
+      elsif range.is_a?(Range)
+        ary = [range.first.in_time_zone, range.last.in_time_zone].map(&begin_unit).uniq
+
+        _ary = []
+        _ary.push ary[0]
+        tmp = ary[0]
+        loop do
+          tmp += 1.send(unit)
+          break if tmp >= ary[-1]
+          _ary << tmp
+        end
+        _ary.push(ary[1]).compact
+        _ary.compact.reject {|i| (i < range.first) && (i >= range.last) }
+      end
+
+      array = array.map {|s| s.to_time } if opts[:utc]
+      array = array.map {|i| i + opts[:offset] } if opts[:offset]
+      array.map(&:in_time_zone)
+    end
+
+  end
+end

data/statlysis.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = 'statlysis'
+  s.version       = '0.0.1'
+  s.date          = '2013-07-10'
+  s.summary       = File.read("README.markdown").split(/===+/)[1].strip.split("\n")[0]
+  s.description   = s.summary
+  s.authors       = ["David Chen"]
+  s.email         = 'mvjome@gmail.com'
+  s.homepage      = 'https://github.com/eoecn/statlysis'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/{functional,unit}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "rake"
+  s.add_dependency "rails"
+  s.add_dependency "mysql2"
+  s.add_dependency "mongoid"
+  s.add_dependency "activerecord"
+  s.add_dependency "activerecord_idnamecache"
+  s.add_dependency "activesupport"
+  s.add_dependency "sequel"
+  s.add_dependency 'only_one_rake'
+
+end

data/test/helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'statlysis'
+
+class Test::Unit::TestCase
+end

data/test/models/company.rb

@@ -0,0 +1,12 @@
+# copied from git://github.com/joe1chen/mongoid-mapreduce.git
+
+class Company
+  include Mongoid::Document
+
+  field :name, :type =>  String
+  field :market, :type =>  String
+  field :shares, :type =>  Integer
+  field :quote, :type =>  Float
+
+  has_many :employees
+end

data/test/models/employee.rb

@@ -0,0 +1,14 @@
+# copied from git://github.com/joe1chen/mongoid-mapreduce.git
+
+class Employee
+  include Mongoid::Document
+
+  field :name
+  field :division
+  field :awards, :type => Integer
+  field :age, :type => Integer
+  field :rooms, :type => Array
+  field :active, :type => Boolean
+
+  belongs_to :company
+end

data/test/test_mapreduce.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+require 'helper'
+
+Mongoid.connect_to "mongoid-mapreduce-test"
+Dir["#{File.dirname(__FILE__)}/models/*.rb"].each { |f| require f }
+Mongoid.default_session.collections.select {|c| c.name !~ /system/ }.each(&:drop)
+
+class TestMapReduce < Test::Unit::TestCase
+  include Statlysis
+
+  def setup
+    # copied from git://github.com/joe1chen/mongoid-mapreduce.git
+    @aapl = Company.create :name => 'Apple', :market => 'Technology', :quote => 401.82, :shares => 972_090_000
+    @msft = Company.create :name => 'Microsoft', :market => 'Technology', :quote => 25.06, :shares => 8_380_000_000
+    @sbux = Company.create :name => 'Starbucks', :market => 'Food', :quote => 38.60, :shares => 746_010_000
+    Employee.create :name => 'Alan', :division => 'Software', :age => 30, :awards => 5, :rooms => [1,2], :active => true, :company => @aapl
+    Employee.create :name => 'Bob', :division => 'Software', :age => 30, :awards => 4, :rooms => [3,4,5], :active => true, :company => @aapl
+    Employee.create :name => 'Chris', :division => 'Hardware', :age => 30, :awards => 3, :rooms => [1,2,3,4], :active => false, :company => @aapl
+  end
+
+  def test_hotest_items_mapreduce
+  end
+
+
+end

data/test/test_statlysis.rb

@@ -0,0 +1,76 @@
+# encoding: UTF-8
+
+require 'helper'
+
+def Rails.root; Pathname.new(ENV['RAILS_ROOT'] || "#{Dir.pwd}/../..") end
+raise "Please setup RAILS_ROOT shell env first!" if not File.exists?(Rails.root.join("config/database.yml"))
+
+class TestStatlysis < Test::Unit::TestCase
+  include Statlysis
+
+  def setup
+    super
+    @dt = DateTime.parse "20121221 +0800"
+    @dt1 = DateTime.parse "20111221 +0800"
+    @dt2 = DateTime.parse "20121221 +0800"
+    Statlysis.set_database :statlysis
+    @old_datetime = DateTime.parse("20130105")
+  end
+
+  def test_parse_datetime
+    assert_equal [@dt], TimeSeries.parse(@dt), "抽取单个时间没通过"
+  end
+
+  def test_parse_special_datetime
+    assert_equal 1, TimeSeries.parse(DateTime.parse('2012122110')).length, "抽取单个时间没通过"
+  end
+
+  def test_parse_range_in_hour
+    # (@dt2 - @dt1).to_i  == 366
+    assert_equal 24, TimeSeries.parse(@dt1..(@dt1+1.day-1.second), :unit => :hour).length, "抽取小时的时间范围没通过"
+  end
+
+  def test_parse_range_in_day
+    # (@dt2 - @dt1).to_i  == 366
+    assert_equal 366, TimeSeries.parse(@dt1..(@dt2-1.second)).length, "抽取天的时间范围没通过"
+  end
+
+  def test_parse_range_in_week
+    # (@dt2 - @dt1).to_i / 7.0 == 52.285714285714285
+    assert_equal 53, TimeSeries.parse(@dt1..(@dt2-1.second), :unit => :week).length, "抽取周的时间范围没通过"
+  end
+
+  def test_parse_range_in_201212_week
+    w1 = DateTime.parse "20121201 +0800"
+    w2 = DateTime.parse "20121231 +0800"
+    assert_equal 6, TimeSeries.parse(w1..w2, :unit => :week).length, "2012十二月应该有六周"
+  end
+
+  def test_setup_count_stat_table
+    eval("class CodeGist < ActiveRecord::Base; end")
+    t = Statlysis::Timely.new CodeGist.where(:user_id => 470700), :time_column => :created_at, :time_unit => :day
+    t.setup_stat_table
+    is_created = Statlysis.sequel.table_exists?(t.stat_table_name)
+    Statlysis.sequel.drop_table t.stat_table_name
+
+    assert(is_created, "统计表#{t.stat_table_name}没有成功创建")
+  end
+
+  def test_setup_lastest_visits_stat_table
+    tn = 'st_blog_lastest_visits_tests'
+    lv = Statlysis::LastestVisits.new "FakeLogCollection", :stat_table_name => tn, :test => true, :default_time => @old_datetime
+    lv.pattern_table_and_model tn
+    is_sequel_model = lv.stat_model.respond_to?(:count)
+    Statlysis.sequel.drop_table tn
+
+    assert(is_sequel_model, "统计表#{lv.stat_table_name}没有成功创建")
+  end
+
+  def test_clock_set_time
+    clock = Statlysis::Clock.new "mvj3", Time.now
+    clock.update @old_datetime
+    update_old_time = (@old_datetime != clock.current)
+    assert(update_old_time, "Can't update old time")
+  end
+
+end

data/test/test_timeseries.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+
+require 'helper'
+
+class TestTimeSeries < Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,216 @@
+--- !ruby/object:Gem::Specification
+name: statlysis
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- David Chen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongoid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord_idnamecache
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: only_one_rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: statistical analysis in ruby dsl
+email: mvjome@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .document
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- lib/statlysis.rb
+- lib/statlysis/clock.rb
+- lib/statlysis/common.rb
+- lib/statlysis/configuration.rb
+- lib/statlysis/cron.rb
+- lib/statlysis/cron/count.rb
+- lib/statlysis/cron/top.rb
+- lib/statlysis/formula.rb
+- lib/statlysis/javascript/count.rb
+- lib/statlysis/map_reduce.rb
+- lib/statlysis/rake.rb
+- lib/statlysis/results.rb
+- lib/statlysis/similar.rb
+- lib/statlysis/timeseries.rb
+- statlysis.gemspec
+- test/helper.rb
+- test/models/company.rb
+- test/models/employee.rb
+- test/test_mapreduce.rb
+- test/test_statlysis.rb
+- test/test_timeseries.rb
+homepage: https://github.com/eoecn/statlysis
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: statistical analysis in ruby dsl
+test_files: []