statlysis 0.0.1 → 0.0.2

data/test/helper.rb

@@ -1,3 +1,4 @@
+require File.join(ENV['HOME'], 'utils/ruby/irb') rescue nil
 require 'rubygems'
 require 'bundler'
 begin
@@ -9,9 +10,46 @@ rescue Bundler::BundlerError => e
 end
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+$LOAD_PATH.unshift File.dirname(__FILE__) # test dirs
+
+# load mongoid setup
+require 'mongoid'
+Mongoid.load!(File.expand_path("../config/mongoid.yml", __FILE__), :production)
+Mongoid.default_session.collections.select {|c| c.name !~ /system/ }.each(&:drop) # delete lastest data
+
 require 'statlysis'
 
-class Test::Unit::TestCase
+# load rails
+def Rails.root; Pathname.new(File.expand_path('../.', __FILE__)) end
+require 'sqlite3'
+
+# load ActiveRecord setup
+Statlysis.set_database :statlysis
+Statlysis.config.is_skip_database_index = true
+ActiveRecord::Base.establish_connection(Statlysis.config.database_opts.merge("adapter" => "sqlite3"))
+Dir[File.expand_path("../migrate/*.rb", __FILE__).to_s].each { |f| require f }
+Dir[File.expand_path("../models/*.rb", __FILE__).to_s].each { |f| require f }
+
+# load basic test data
+# copied from http://stackoverflow.com/questions/4410794/ruby-on-rails-import-data-from-a-csv-file/4410880#4410880
+require 'csv'
+csv = CSV.parse(File.read(File.expand_path('../data/code_gists_20130724.csv', __FILE__)), :headers => true) # data from code.eoe.cn
+csv.each {|row| CodeGist.create!(row.to_hash) }
+
+
+Statlysis.setup do
+  hourly EoeLog, :time_column => :t
+
+  daily  CodeGist
+
+  [EoeLog,
+   EoeLog.where(:do => 3),
+   Mongoid[/multiple_log_2013[0-9]{4}/],
+   Mongoid[/multiple_log_2013[0-9]{4}/].where(:ui => {"$ne" => 0})
+  ].each do |s|
+    daily s, :time_column => :t
+  end
+  cron = Statlysis.daily['mul'][1]
+  require 'pry-debugger';binding.pry
 end

data/test/migrate/1_active_record.rb

@@ -0,0 +1,8 @@
+class CreateActiveRecord < ActiveRecord::Migration
+  create_table :code_gists do |t|
+    t.string  :description
+    t.integer :user_id
+    t.timestamps
+    t.string :author
+  end
+end

data/test/models/code_gist.rb

@@ -0,0 +1,5 @@
+# encoding: UTF-8
+
+class CodeGist < ActiveRecord::Base
+
+end

data/test/models/eoe_log.rb

@@ -0,0 +1,53 @@
+# encoding: UTF-8
+
+class EoeLog
+  include Mongoid::Document
+
+  field :ii, :type => Integer, :default => 0 # ip_int, request.remote_ip
+  field :do, :type => Integer, :default => 1 # request.domain
+  field :fp, :type => String, :default => '' # request.original_fullpath
+  field :ui, :type => Integer, :default => 0 # user_id
+  field :re, :type => String, :default => '' # refer
+
+  field :t,  :type => DateTime, :default => Statlysis::DateTime1970 # timestamp
+
+  # request.headers['HTTP_USER_AGENT']
+  field :br, :type => Integer, :default => 0 # browser name
+  field :bv, :type => String, :default => '' # browser version
+  field :bo, :type => Integer, :default => 0 # os
+
+  index({t: -1, ui: 1}, {:background => true})
+end
+EoeLog.create
+
+
+# Setup a single log that combined by multiple collections
+{'07' => 1..31, '08' => 1..12}.map do |month, day_range|
+  day_range.map do |day|
+    # define model dynamically, e.g. MultipleLog20130729
+    date_str = "2013#{month}#{day.to_s.rjust(2, '0')}"
+    collection_class_name = "MultipleLog#{date_str}"
+    collection_name = collection_class_name.sub("MultipleLog", "multiple_log_")
+
+    # NOTE: Object.const_set(name, Class.new {}) cause failed with error 16256: "Invalid ns [statlysis_mongoid_test.]",
+    # and cann't Mongoid.create data
+    eval("
+      class #{collection_class_name}
+        include Mongoid::Document
+        self.default_collection_name = #{collection_name.to_json}
+        field :t, :type => DateTime
+        field :url, :type => String
+        index({t: -1}, {:background => true})
+      end
+    ")
+
+    collection_class = collection_class_name.constantize
+    t = Time.zone.parse(date_str)
+    1.upto(day) do |i|
+      puts "#{month} #{day_range} #{day} #{i}" if ENV['DEBUG']
+      collection_class.create :t => (t.to_time+rand(60*60*24-1)).to_datetime, :url => '/'
+    end
+
+    collection_class.count
+  end
+end

data/test/test_daily_count.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+
+require 'helper'
+
+class TestDailyCount < Test::Unit::TestCase
+  def setup
+    @output = Statlysis.daily['code_gist'].first.output
+  end
+
+  def test_timely
+    o = @output.map {|i| i[:timely_c] }
+    r = (o - [5,11,0,1,8,2,3,4,16,10,26,13,7,9,20,15,30,33,14,6,12,17,19,59,65,84,62,114,69,52,61,67,154,70]).reject(&:zero?).blank?
+    assert_equal r, true
+  end
+
+  def test_totally
+    o = @output.map {|i| i[:totally_c] }
+    r = (o - [5,16,17,25,27,30,34,36,37,53,55,56,57,59,60,64,66,67,68,70,71,73,74,75,80,90,116,129,136,145,165,185,200,230,234,235,236,237,270,273,274,288,299,304,305,312,327,337,345,359,374,380,392,418,435,446,452,463,466,473,493,506,512,520,525,545,549,553,558,577,636,701,785,805,867,981,1050,1102,1163,1230,1384,1454,1455,1457,1458]).reject(&:zero?).blank?
+    assert_equal r, true
+  end
+
+end 

data/test/test_mapreduce.rb

@@ -2,21 +2,8 @@
 
 require 'helper'
 
-Mongoid.connect_to "mongoid-mapreduce-test"
-Dir["#{File.dirname(__FILE__)}/models/*.rb"].each { |f| require f }
-Mongoid.default_session.collections.select {|c| c.name !~ /system/ }.each(&:drop)
-
 class TestMapReduce < Test::Unit::TestCase
-  include Statlysis
-
   def setup
-    # copied from git://github.com/joe1chen/mongoid-mapreduce.git
-    @aapl = Company.create :name => 'Apple', :market => 'Technology', :quote => 401.82, :shares => 972_090_000
-    @msft = Company.create :name => 'Microsoft', :market => 'Technology', :quote => 25.06, :shares => 8_380_000_000
-    @sbux = Company.create :name => 'Starbucks', :market => 'Food', :quote => 38.60, :shares => 746_010_000
-    Employee.create :name => 'Alan', :division => 'Software', :age => 30, :awards => 5, :rooms => [1,2], :active => true, :company => @aapl
-    Employee.create :name => 'Bob', :division => 'Software', :age => 30, :awards => 4, :rooms => [3,4,5], :active => true, :company => @aapl
-    Employee.create :name => 'Chris', :division => 'Hardware', :age => 30, :awards => 3, :rooms => [1,2,3,4], :active => false, :company => @aapl
   end
 
   def test_hotest_items_mapreduce

data/test/test_single_log_in_multiple_collections.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+
+require 'helper'
+
+class TestSingleLogInMultipleCollections < Test::Unit::TestCase
+  def setup
+    @output = Statlysis.daily['multi'].first.output
+  end
+
+  def test_timely
+    o = @output.map {|i| i[:timely_c] }[0..30]
+    r = (o - [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]).reject(&:zero?).blank?
+    assert_equal r, true
+  end
+
+  def test_totally
+    o = @output.map {|i| i[:totally_c] }[0..30]
+    r = (o - [1,3,6,10,15,21,28,36,45,55,66,78,91,105,120,136,153,171,190,210,231,253,276,300,325,351,378,406,435,465,496]).reject(&:zero?).blank?
+    assert_equal r, true
+  end
+
+end

data/test/test_statlysis.rb

@@ -2,54 +2,15 @@
 
 require 'helper'
 
-def Rails.root; Pathname.new(ENV['RAILS_ROOT'] || "#{Dir.pwd}/../..") end
-raise "Please setup RAILS_ROOT shell env first!" if not File.exists?(Rails.root.join("config/database.yml"))
-
 class TestStatlysis < Test::Unit::TestCase
-  include Statlysis
-
   def setup
-    super
-    @dt = DateTime.parse "20121221 +0800"
-    @dt1 = DateTime.parse "20111221 +0800"
-    @dt2 = DateTime.parse "20121221 +0800"
-    Statlysis.set_database :statlysis
-    @old_datetime = DateTime.parse("20130105")
-  end
-
-  def test_parse_datetime
-    assert_equal [@dt], TimeSeries.parse(@dt), "抽取单个时间没通过"
-  end
-
-  def test_parse_special_datetime
-    assert_equal 1, TimeSeries.parse(DateTime.parse('2012122110')).length, "抽取单个时间没通过"
-  end
-
-  def test_parse_range_in_hour
-    # (@dt2 - @dt1).to_i  == 366
-    assert_equal 24, TimeSeries.parse(@dt1..(@dt1+1.day-1.second), :unit => :hour).length, "抽取小时的时间范围没通过"
-  end
-
-  def test_parse_range_in_day
-    # (@dt2 - @dt1).to_i  == 366
-    assert_equal 366, TimeSeries.parse(@dt1..(@dt2-1.second)).length, "抽取天的时间范围没通过"
-  end
-
-  def test_parse_range_in_week
-    # (@dt2 - @dt1).to_i / 7.0 == 52.285714285714285
-    assert_equal 53, TimeSeries.parse(@dt1..(@dt2-1.second), :unit => :week).length, "抽取周的时间范围没通过"
-  end
-
-  def test_parse_range_in_201212_week
-    w1 = DateTime.parse "20121201 +0800"
-    w2 = DateTime.parse "20121231 +0800"
-    assert_equal 6, TimeSeries.parse(w1..w2, :unit => :week).length, "2012十二月应该有六周"
+    @old_datetime = Time.zone.parse("20130105")
   end
 
   def test_setup_count_stat_table
     eval("class CodeGist < ActiveRecord::Base; end")
     t = Statlysis::Timely.new CodeGist.where(:user_id => 470700), :time_column => :created_at, :time_unit => :day
-    t.setup_stat_table
+    t.setup_stat_model
     is_created = Statlysis.sequel.table_exists?(t.stat_table_name)
     Statlysis.sequel.drop_table t.stat_table_name
 
@@ -58,19 +19,13 @@ class TestStatlysis < Test::Unit::TestCase
 
   def test_setup_lastest_visits_stat_table
     tn = 'st_blog_lastest_visits_tests'
-    lv = Statlysis::LastestVisits.new "FakeLogCollection", :stat_table_name => tn, :test => true, :default_time => @old_datetime
-    lv.pattern_table_and_model tn
+    lv = Statlysis::LastestVisits.new EoeLog, :stat_table_name => tn, :test => true, :default_time => @old_datetime
+    h = Statlysis::Utils.setup_pattern_table_and_model tn
+    lv.stat_model = h[:model]
     is_sequel_model = lv.stat_model.respond_to?(:count)
     Statlysis.sequel.drop_table tn
 
     assert(is_sequel_model, "统计表#{lv.stat_table_name}没有成功创建")
   end
 
-  def test_clock_set_time
-    clock = Statlysis::Clock.new "mvj3", Time.now
-    clock.update @old_datetime
-    update_old_time = (@old_datetime != clock.current)
-    assert(update_old_time, "Can't update old time")
-  end
-
 end

data/test/test_timeseries.rb

@@ -3,4 +3,50 @@
 require 'helper'
 
 class TestTimeSeries < Test::Unit::TestCase
+  include Statlysis
+
+  def setup
+    super
+    @dt = Time.zone.parse "20121221 +0800"
+    @dt1 = Time.zone.parse "20111221 +0800"
+    @dt2 = Time.zone.parse "20121221 +0800"
+    @old_datetime = Time.zone.parse("20130105")
+  end
+
+  def test_parse_datetime
+    assert_equal [@dt], TimeSeries.parse(@dt), "抽取单个时间没通过"
+  end
+
+  def test_parse_special_datetime
+    assert_equal 1, TimeSeries.parse(Time.zone.parse('2012122110')).length, "抽取单个时间没通过"
+  end
+
+  def test_parse_range_in_hour
+    # (@dt2 - @dt1).to_i  == 366
+    assert_equal 24, TimeSeries.parse(@dt1..(@dt1+1.day-1.second), :unit => :hour).length, "抽取小时的时间范围没通过"
+  end
+
+  def test_parse_range_in_day
+    # (@dt2 - @dt1).to_i  == 366
+    assert_equal 366, TimeSeries.parse(@dt1..(@dt2-1.second)).length, "抽取天的时间范围没通过"
+  end
+
+  def test_parse_range_in_week
+    # (@dt2 - @dt1).to_i / 7.0 == 52.285714285714285
+    assert_equal 53, TimeSeries.parse(@dt1..(@dt2-1.second), :unit => :week).length, "抽取周的时间范围没通过"
+  end
+
+  def test_parse_range_in_201212_week
+    w1 = Time.zone.parse "20121201 +0800"
+    w2 = Time.zone.parse "20121231 +0800"
+    assert_equal 6, TimeSeries.parse(w1..w2, :unit => :week).length, "2012十二月应该有六周"
+  end
+
+  def test_clock_set_time
+    clock = Statlysis::Clock.new "parse_EoeLog", Time.now
+    clock.update @old_datetime
+    update_old_time = (@old_datetime != clock.current)
+    assert(update_old_time, "Can't update old time")
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statlysis
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-10 00:00:00.000000000 Z
+date: 2013-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -155,7 +155,103 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: statistical analysis in ruby dsl
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: origin
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: short_inspect
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-debugger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-test
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Statistical & Analysis in Ruby DSL
 email: mvjome@gmail.com
 executables: []
 extensions: []
@@ -164,33 +260,52 @@ files:
 - .document
 - .gitignore
 - Gemfile
-- Gemfile.lock
-- LICENSE.txt
+- Guardfile
 - README.markdown
 - Rakefile
 - lib/statlysis.rb
 - lib/statlysis/clock.rb
 - lib/statlysis/common.rb
 - lib/statlysis/configuration.rb
+- lib/statlysis/constants.rb
 - lib/statlysis/cron.rb
 - lib/statlysis/cron/count.rb
+- lib/statlysis/cron/count/dimensions.rb
+- lib/statlysis/cron/count/timely.rb
 - lib/statlysis/cron/top.rb
+- lib/statlysis/cron/top/hotest_items.rb
+- lib/statlysis/cron/top/lastest_visits.rb
+- lib/statlysis/cron_set.rb
+- lib/statlysis/dataset.rb
 - lib/statlysis/formula.rb
 - lib/statlysis/javascript/count.rb
 - lib/statlysis/map_reduce.rb
+- lib/statlysis/multiple_dataset.rb
+- lib/statlysis/multiple_dataset/active_record.rb
+- lib/statlysis/multiple_dataset/mongoid.rb
 - lib/statlysis/rake.rb
 - lib/statlysis/results.rb
 - lib/statlysis/similar.rb
 - lib/statlysis/timeseries.rb
+- lib/statlysis/utils.rb
 - statlysis.gemspec
+- test/config/database.yml
+- test/config/mongoid.yml
+- test/data/.gitkeep
+- test/data/code_gists_20130724.csv
 - test/helper.rb
-- test/models/company.rb
-- test/models/employee.rb
+- test/migrate/1_active_record.rb
+- test/models/.gitkeep
+- test/models/code_gist.rb
+- test/models/eoe_log.rb
+- test/test_daily_count.rb
 - test/test_mapreduce.rb
+- test/test_single_log_in_multiple_collections.rb
 - test/test_statlysis.rb
 - test/test_timeseries.rb
 homepage: https://github.com/eoecn/statlysis
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -201,16 +316,22 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 59716176471030881
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 59716176471030881
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
-summary: statistical analysis in ruby dsl
+summary: Statistical & Analysis in Ruby DSL
 test_files: []