rbbt-dm 1.1.19 → 1.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27188d3dbc9c8d88409655bea507f70bfd14e9e5
-  data.tar.gz: 69bc8947fb0f736721eff76d58ec6502f4cfb840
+  metadata.gz: 3ce285713356bad8c3fe55c26a2b3ac42f8c782b
+  data.tar.gz: ea8d760c4c2682df56a94c4772a53421d3ae89f7
 SHA512:
-  metadata.gz: bcd2fa58747a8373cc7cf1ccdfd2f5a27e2ed55f40152e9eb72c0ca15130db40d24fc92c554ddb6f6c1ad42cf4640b04fd8f93a605134692f9a761a7cc447da6
-  data.tar.gz: e807c21ef69d7a7aebbccb5f1710b199fe2a573c485cd2d04adddf42aff2042762aca332c54a2550ad514fd2a9816f3c2288b82bc3d49ac229eac301240a3fb3
+  metadata.gz: 15d49776f790c74e40b7c6c8f44ba320d35e66aff8525cff2ad7f375bec87fb1843fd2775128f6d06e64f37b371a727220d35b9ff0ce52e88b2c49ac8a998718
+  data.tar.gz: 522aa087395221563801e0a2d1a978b2759ad312ba6eea469049b386c44a7198d939f6f05c63aa59dc905b82727384c60e836ad0c4a188377aef7555562a3be2

data/lib/rbbt/statistics/random_walk.rb

@@ -256,26 +256,42 @@ module RandomWalk
     end
   end
 
-  def self.persisted_permutations(size, total, missing = 0, times = 10_000)
-    repo_file = "/tmp/rw_repo7"
-    repo = Persist.open_tokyocabinet(repo_file, false, :float_array)
-    key = Misc.digest([size, total, missing, times, scoring_method].inspect)
-    begin
-      repo.read
-      if repo[key]
-        repo[key]
-      else
-        p = permutations(size, total, missing, times)
-        repo.write_and_close do
-          repo[key] = p
-        end
-        p
+  def self.permutations_up_down(up_size, down_size, total, missing = 0, times = 10_000)
+    if up_size == 0 or down_size == 0
+      [0] * times
+    else
+      (1..times).collect do
+        up_p = []
+        sample_without_replacement(total, up_size, up_p)
+        down_p = []
+        sample_without_replacement(total, down_size, down_p)
+
+        score_up_down(up_p, down_p, total, missing).abs
       end
-    ensure
-      repo.close
     end
   end
 
+
+  #def self.persisted_permutations(size, total, missing = 0, times = 10_000)
+  #  repo_file = "/tmp/rw_repo7"
+  #  repo = Persist.open_tokyocabinet(repo_file, false, :float_array)
+  #  key = Misc.digest([size, total, missing, times, scoring_method].inspect)
+  #  begin
+  #    repo.read
+  #    if repo[key]
+  #      repo[key]
+  #    else
+  #      p = permutations(size, total, missing, times)
+  #      repo.write_and_close do
+  #        repo[key] = p
+  #      end
+  #      p
+  #    end
+  #  ensure
+  #    repo.close
+  #  end
+  #end
+
   def self.persisted_permutations(size, total, missing = 0, times = 10_000)
     require 'rbbt/util/tc_cache'
     repo_file = "/tmp/rw_repo9"
@@ -287,12 +303,27 @@ module RandomWalk
     p
   end
 
+  def self.persisted_permutations_up_down(up_size, down_size, total, missing = 0, times = 10_000)
+    require 'rbbt/util/tc_cache'
+    repo_file = "/tmp/rw_repo9"
+    key = Misc.digest([up_size, down_size, total, missing, times, scoring_method].inspect)
+    cache = TCCache.open(repo_file, :float_array)
+    p = cache.cache(key) do
+      permutations_up_down(up_size, down_size, total, missing, times)
+    end
+    p
+  end
+
+
   def self.pvalue(permutations, score)
+    positive = score > 0
     score = score.abs
-    permutations.inject(1){|acc, per| 
+    pvalue = permutations.inject(1){|acc, per| 
       acc += 1 if per > score
       acc
     }.to_f / permutations.length
+
+    positive ? pvalue : - pvalue
   end
 
   COLORS = {
@@ -445,6 +476,59 @@ module OrderedList
     end
   end
 
+  def pvalue_up_down(up_set, down_set, cutoff = 0.1, options = {})
+    up_set = Set.new(up_set.compact) unless Set === up_set
+    down_set = Set.new(down_set.compact) unless Set === down_set
+
+    options = Misc.add_defaults options, :permutations => 10000, :missing => 0
+    permutations, missing, persist_permutations = Misc.process_options options, :permutations, :missing, :persist_permutations
+
+    up_hits = hits(up_set)
+    down_hits = hits(down_set)
+   
+    return 1.0 if up_hits.empty? or down_hits.empty? # Repasar
+
+    target_score = RandomWalk.score_up_down(up_hits.sort, down_hits.sort, self.length, missing)
+
+    if persist_permutations 
+      permutations = RandomWalk.persisted_permutations_up_down(up_set.length, down_set.length, self.length, missing, permutations)
+      RandomWalk.pvalue(permutations, target_score)
+    else
+      # P-value computation
+      target_score_abs = target_score.abs
+
+      max = (permutations.to_f * cutoff).ceil
+
+      up_size = up_set.length
+      down_size = down_set.length
+
+      total = self.length
+      better_permutation_score_count = 1
+
+      if size == 0
+        1.0
+      else
+        (1..permutations).each do
+          up_p= []
+          RandomWalk.sample_without_replacement(total, up_size, up_p)
+          down_p= []
+          RandomWalk.sample_without_replacement(total, down_size, down_p)
+
+          permutation_score = RandomWalk.score_up_down(up_p.sort, down_p.sort, total, missing).abs
+          if permutation_score.abs > target_score_abs
+            better_permutation_score_count += 1
+          end
+
+          return 1.0 if better_permutation_score_count > max
+        end
+        p = (better_permutation_score_count.to_f + 1) / permutations
+        p = -p if target_score < 0
+        p
+      end
+    end
+  end
+
+
   def pvalue_weights(set, cutoff = 0.1, options = {})
     raise "No weight defined" if @weights.nil?
     @total_weights ||= Misc.sum(@weights)

data/test/rbbt/statistics/test_random_walk.rb

@@ -18,11 +18,50 @@ class TestRandomWalk < Test::Unit::TestCase
     list = (1..1000).to_a
     list.extend OrderedList
 
-    assert list.pvalue((1..100).to_a, 0.05) < 0.05
+    assert list.pvalue((1..100).to_a, 0.05).abs < 0.05
+    assert list.pvalue((1..100).to_a, 0.05) > 0
+    
+    assert list.pvalue((900..1000).to_a, 0.05).abs < 0.05
+    assert list.pvalue((900..1000).to_a, 0.05) < 0
+
     assert list.pvalue([100, 200, 300, 400, 500], 0.05) > 0.05
+
+    assert list.pvalue((1..100).to_a, 0.05, :persisted_permutations => true).abs < 0.05
+    assert list.pvalue((1..100).to_a, 0.05, :persisted_permutations => true) > 0
+    
+    assert list.pvalue((900..1000).to_a, 0.05, :persisted_permutations => true).abs < 0.05
+    assert list.pvalue((900..1000).to_a, 0.05, :persisted_permutations => true) < 0
+
+    assert list.pvalue([100, 200, 300, 400, 500], 0.05, :persisted_permutations => true) > 0.05
+  end
+
+  def test_pvalue_up_down
+    Log.severity = 0
+    list = (1..1000).to_a
+    list.extend OrderedList
+
+    assert list.pvalue_up_down((1..100).to_a, (900..1000).to_a, 0.05, :persist_permutations => false).abs < 0.05
+    assert list.pvalue_up_down((1..100).to_a, (900..1000).to_a, 0.05, :persist_permutations => false) > 0
+
+    assert list.pvalue_up_down((900..1000).to_a, (1..100).to_a, 0.05, :persist_permutations => false).abs < 0.05
+    assert list.pvalue_up_down((900..1000).to_a, (1..100).to_a, 0.05, :persist_permutations => false) < 0
+
+    rand_top = (1..100).to_a.collect{ rand(1000) }
+    rand_bottom = (1..100).to_a.collect{ rand(1000) }
+    assert list.pvalue_up_down(rand_top, rand_bottom, 0.05, :persisted_permutations => false).abs > 0.05
+
+    assert list.pvalue_up_down((1..100).to_a, (900..1000).to_a, 0.05, :persist_permutations => true).abs < 0.05
+    assert list.pvalue_up_down((1..100).to_a, (900..1000).to_a, 0.05, :persist_permutations => true) > 0
+
+    assert list.pvalue_up_down((900..1000).to_a, (1..100).to_a, 0.05, :persist_permutations => true).abs < 0.05
+    assert list.pvalue_up_down((900..1000).to_a, (1..100).to_a, 0.05, :persist_permutations => true) < 0
+
+    rand_top = (1..100).to_a.collect{ rand(1000) }
+    rand_bottom = (1..100).to_a.collect{ rand(1000) }
+    assert list.pvalue_up_down(rand_top, rand_bottom, 0.05, :persisted_permutations => true).abs > 0.05
   end
 
-  def test_pvalue_weights
+  def __test_pvalue_weights
     list = (1..1000).to_a
 
     weights = list.collect{|v| (Misc.mean(list) - v)**2}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.19
+  version: 1.1.20
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-10 00:00:00.000000000 Z
+date: 2016-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util