rbbt-dm 1.1.39 → 1.1.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a6a86141d16b2da2c4024bebcff9447e78add40
-  data.tar.gz: 062c8f641dfc0672d5d883958df950af7af49889
+  metadata.gz: f76827643000582061a88e4ea73e2bd75c77988c
+  data.tar.gz: bb6261e9ce5a53948b5429e7fcead2da551070d0
 SHA512:
-  metadata.gz: fa3016424aca9f00970abda11b08cbec48faf4ddb61031d9e646527fa69baff50f60de733c29a99189889bc084de2d0c831e857b9507d240be91e16fc9a25f76
-  data.tar.gz: 7fb134ffd67f043111c1bf6ec44d042fde6e53321b716ed3f2a1d0075b9b1b8e0d0ba0cbb3f1eb9d17f25ac849843e03c8c3aefb57ee377e12ecd766f79af636
+  metadata.gz: a1e4ccabc8088e6564cf1e2d2dc8e4cafc539d499fe1bdef6e4ce0ae7e41954457589fa79fc97371960d957b598394d41d4c36e515c6847427a69ca893955288
+  data.tar.gz: 6f2625689288ce82bcb7aac8f1ceca2f68300b5edb9ed63cb6a7ea4dbed1bc5c3442d38f0624d4e51224502e6a7e933e92d6eb3ae5851bcc00b7db8669e9bb52

data/lib/rbbt/stan.rb

@@ -0,0 +1,205 @@
+require 'rbbt/util/R'
+require 'mkfifo'
+
+module STAN
+
+  def self.save_data(directory, data = {})
+    directory = directory.find if Path === directory
+    types = {}
+    load_str = "input_data = list()\n"
+
+    data.each do |name, value|
+      file = File.join(directory, name.to_s)
+      case value
+      when Integer
+        Open.write(file, value.to_json)
+        types[name] = 'integer'
+        load_str << "input_data[['" << name.to_s << "']] = " << "fromJSON(txt='#{file}')" << "\n"
+      when Float
+        Open.write(file, value.to_json)
+        types[name] = 'real'
+        load_str << "input_data[['" << name.to_s << "']] = " << "fromJSON(txt='#{file}')" << "\n"
+      when Array
+        Open.write(file, value.to_json)
+        if value.select{|v| Float === v}.empty?
+          types[name] = 'iarray'
+        else
+          types[name] = 'array'
+        end
+        load_str << "input_data[['" << name.to_s << "_l']] = " << value.length.to_s << "\n"
+        load_str << "input_data[['" << name.to_s << "']] = " << "fromJSON(txt='#{file}')" << "\n"
+      when TSV
+        Open.write(file, value.to_s)
+        types[name] = 'matrix'
+        load_str << "input_data[['" << name.to_s << "_c']] = " << value.fields.length.to_s << "\n"
+        load_str << "input_data[['" << name.to_s << "_r']] = " << value.size.to_s << "\n"
+        load_str << "#{name}.tmp = " << "rbbt.impute(rbbt.tsv('#{file}'))" << "\n"
+        load_str << "input_data[['" << name.to_s << "']] = " << "#{name}.tmp" << "\n"
+      when Path
+        value = TSV.open(value)
+        Open.write(file, value.to_s)
+        types[name] = 'matrix'
+        load_str << "input_data[['" << name.to_s << "_c']] = " << value.fields.length.to_s << "\n"
+        load_str << "input_data[['" << name.to_s << "_r']] = " << value.size.to_s << "\n"
+        load_str << "#{name}.tmp = " << "rbbt.impute(rbbt.tsv('#{file}'))" << "\n"
+        load_str << "input_data[['" << name.to_s << "']] = " << "#{name}.tmp" << "\n"
+      else
+        raise "Unknown type of data #{ name }: #{Misc.fingerprint value}"
+      end
+    end
+
+    [types, load_str]
+  end
+
+  def self.data_header(types)
+
+    types_str = ""
+    types.each do |name,type|
+      name = name.to_s
+      case type
+      when 'real'
+        types_str << "  " << "real " << name << ";\n"
+      when 'integer'
+        types_str << "  " << "int " << name << ";\n"
+      when 'array'
+        types_str << "  " << "int<lower=0> " << name << "_l" << ";\n"
+        #types_str << "  " << "real " << name << "[" << name << "_l]" << ";\n"
+        types_str << "  " << "vector" << "[" << name << "_l] "<< name  << ";\n"
+      when 'iarray'
+        types_str << "  " << "int<lower=0> " << name << "_l" << ";\n"
+        #types_str << "  " << "real " << name << "[" << name << "_l]" << ";\n"
+        types_str << "  " << "int " << name << "[" << name << "_l]" << ";\n"
+      when 'matrix'
+        types_str << "  " << "int<lower=0> " << name << "_c" << ";\n"
+        types_str << "  " << "int<lower=0> " << name << "_r" << ";\n"
+        #types_str << "  " << "real " << name << "[" << name << "_r," << name << "_c]" << ";\n"
+        types_str << "  " << "matrix" << "[" << name << "_r," << name << "_c] " << name << ";\n"
+      else
+        raise "Unknown type for #{ name }: #{type}"
+      end
+
+    end
+    <<-EOF
+data{
+
+#{types_str}
+}
+    EOF
+  end
+
+  def self.exec(data, model, input_directory, parameter_chains, sample_file, debug = FALSE, stan_options = {})
+    stan_options = Misc.add_defaults stan_options, :iter => 1000, :warmup => 500, :chains => 1, :seed => 2887, :refresh => 1200
+
+    data = {} if data.nil?
+
+    types, load_str = save_data(input_directory, data)
+    data_header = self.data_header(types)
+    stan_model = data_header + "\n" + model
+
+    stan_file = Rbbt.var.stan_models[Misc.digest(stan_model).to_s << ".stan"].find
+    Open.write(stan_file, stan_model)
+
+    Log.debug "STAN model:\n" + stan_model
+
+    script = <<-EOF
+rbbt.require('rstan')
+rbbt.require('jsonlite')
+
+rstan_options(auto_write = TRUE)
+options(mc.cores = parallel::detectCores())
+
+#{load_str} 
+
+fit <- stan(file='#{stan_file}', data=input_data, sample_file='#{sample_file}', verbose=#{debug ? 'TRUE' : 'FALSE'}, #{R.hash2Rargs(stan_options)})
+
+params <- as.data.frame(fit)
+
+print(fit)
+#{parameter_chains.nil? ? "" : "rbbt.tsv.write('#{parameter_chains}', params)" }
+    EOF
+
+    R.run script, nil, :monitor => debug
+  end
+
+  def self.stream_chain(data, model, directory = nil, options = {})
+    options, directory = directory, nil if Hash === directory
+    debug = Misc.process_options options, :debug
+
+    if directory.nil?
+      directory = TmpFile.tmp_file 
+      erase = true
+    end
+
+    FileUtils.mkdir_p directory unless File.exists? directory
+    input_directory  = File.join(directory, 'inputs')
+    parameter_chains = File.join(directory, 'chains') unless erase
+    summary          = File.join(directory, 'summary') unless erase
+    sample_file      = File.join(directory, 'samples')
+
+    File.mkfifo(sample_file)
+
+    io = Misc.open_pipe do |sin|
+      iteration = 1
+      sin << "#: :type=:list#:cast=:to_f" << "\n"
+      begin
+        reader = File.open(sample_file, 'r')
+        while line = reader.gets
+          if line =~ /^#/
+            new_line = line
+            next
+          elsif line =~ /^lp__/
+            parts = line.split(",")
+            new_line = "#Iteration\t" << parts * "\t"
+          else
+            parts = line.split(",")
+            new_line = iteration.to_s << "\t" << parts * "\t"
+            iteration += 1
+          end
+          sin << new_line 
+        end
+      rescue 
+        Log.exception $!
+        raise $!
+      end
+    end
+
+    exec_thread = Thread.new do
+      res = self.exec(data, model, input_directory, parameter_chains, sample_file, debug, options)
+      Open.write(summary, res.read.to_s) unless summary.nil?
+      Log.debug "Result from STAN:\n" << res.read
+    end
+
+    ConcurrentStream.setup io, :threads => [exec_thread] do
+      Log.debug "Done chains for STAN"
+      if erase
+        FileUtils.rm_rf directory
+      end
+    end
+  end
+
+  def self.run(data, model, directory, options = {})
+    debug = Misc.process_options options, :debug
+
+    input_directory = File.join(directory, 'inputs')
+
+    parameter_chains = File.join(directory, 'chains')
+    summary = File.join(directory, 'summary')
+    sample_file = File.join(directory, 'sample')
+
+    res = self.exec(data, model, input_directory, parameter_chains, sample_file, debug, options)
+    Log.debug "Result from STAN:\n" << res.read
+    Open.write(summary, res.read)
+    
+    Open.open(parameter_chains)
+  end
+
+  def self.fit(data, model, options = {})
+    TmpFile.with_file do |directory|
+      FileUtils.mkdir_p directory
+      res = self.run(data, model, directory, options)
+      TSV.open(res, :type => :list, :cast => :to_f)
+    end
+  end
+
+
+end

data/test/rbbt/test_stan.rb

@@ -0,0 +1,159 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.rb')
+require 'rbbt/stan'
+
+class TestClass < Test::Unit::TestCase
+  def test_save_array
+
+    data = {}
+    data[:y] = [1,2,3,4]
+
+    TmpFile.with_file do |directory|
+      STAN.save_data(directory, data)
+      assert File.exists?(File.join(directory, 'y'))
+      assert_equal '[1,2,3,4]', Open.read(File.join(directory, 'y'))
+    end
+
+  end
+
+  def test_data_header
+    types = {:y => 'array'}
+
+    header = STAN.data_header(types)
+    assert header.include? 'vector[y_l] y;'
+  end
+
+  def test_simple_fit
+    Log.severity = 0
+    res = STAN.fit({}, <<-EOF)
+parameters{
+real y;
+}
+
+model {
+target += -0.5 * y * y;
+}
+    EOF
+    ppp res
+    Log.tsv res
+  end
+
+  def test_fit_vector
+    Log.severity = 0
+    data = {}
+    real_mean = 4
+    data[:y] = R.eval_a "rnorm(100, #{real_mean}, 1)"
+
+    res = STAN.fit(data, <<-EOF, :iter => 10_000, :chains => 2)
+parameters{
+  real mu;
+}
+model{
+  mu ~ normal(0,10);
+  y ~ normal(mu, 1);
+}
+    EOF
+
+    x = []
+    m = Misc.mean(res.column("mu").values)
+
+    assert (m - real_mean).abs < 0.5
+  end
+
+  def test_fit_matrix
+    Log.severity = 0
+
+    samples = 1000
+    real_mean = 5
+    y = TmpFile.with_file do |tsv|
+      R.run <<-EOF
+s = c(1,5,10)
+
+samples = #{samples}
+m = list()
+for (i in seq(1,samples)){
+  sample.name = paste("S",i, sep="")
+  sample.values = rnorm(3,#{real_mean},1) * s
+  m[[sample.name]] = sample.values
+}
+
+data = as.data.frame(m)
+rbbt.tsv.write('#{tsv}', data)
+      EOF
+
+      TSV.open(tsv, :type => :list, :cast => :to_f)
+    end
+
+    y = y.transpose("Sample")
+
+    res = STAN.fit({:y => y}, <<-EOF, :iter => 100, :warmup => 20, :chains => 1)
+parameters{
+  vector<lower=0>[y_c] s;
+  real<lower=0> w_m;
+}
+
+model{
+  s ~ cauchy(0,100);
+  w_m ~ uniform(0,10);
+
+  for (j in 1:y_c){
+      y[,j] ~ normal(w_m*s[j], 1/s[j]);
+  }
+}
+    EOF
+
+    m = Misc.mean(res.column("w_m").values) 
+    assert (m - real_mean).abs < (real_mean.to_f / 10)
+  end
+
+  def test_stream
+    Log.severity = 0
+
+    samples = 1000
+    real_mean = 5
+    y = TmpFile.with_file do |tsv|
+      R.run <<-EOF
+s = c(1,5,10)
+
+samples = #{samples}
+m = list()
+for (i in seq(1,samples)){
+  sample.name = paste("S",i, sep="")
+  sample.values = rnorm(3,#{real_mean},1) * s
+  m[[sample.name]] = sample.values
+}
+
+data = as.data.frame(m)
+rbbt.tsv.write('#{tsv}', data)
+      EOF
+
+      TSV.open(tsv, :type => :list, :cast => :to_f)
+    end
+
+    y = y.transpose("Sample")
+
+    io = STAN.stream_chain({:y => y}, <<-EOF, :iter => 100, :warmup => 20)
+parameters{
+  vector<lower=0>[y_c] s;
+  real<lower=0> w_m;
+}
+
+model{
+  s ~ cauchy(0,100);
+  w_m ~ uniform(0,10);
+
+  for (j in 1:y_c){
+      y[,j] ~ normal(w_m*s[j], 1/s[j]);
+  }
+}
+    EOF
+
+    lines = 0
+    while line = io.gets
+      lines += 1
+    end
+    io.close
+    io.join if io.respond_to? :join
+    assert_equal 102, lines
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.39
+  version: 1.1.40
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-12 00:00:00.000000000 Z
+date: 2018-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -98,6 +98,7 @@ files:
 - lib/rbbt/network/paths.rb
 - lib/rbbt/plots/bar.rb
 - lib/rbbt/plots/heatmap.rb
+- lib/rbbt/stan.rb
 - lib/rbbt/statistics/fdr.rb
 - lib/rbbt/statistics/fisher.rb
 - lib/rbbt/statistics/hypergeometric.rb
@@ -114,6 +115,7 @@ files:
 - test/rbbt/statistics/test_fisher.rb
 - test/rbbt/statistics/test_hypergeometric.rb
 - test/rbbt/statistics/test_random_walk.rb
+- test/rbbt/test_stan.rb
 - test/rbbt/vector/model/test_svm.rb
 - test/rbbt/vector/test_model.rb
 - test/test_helper.rb
@@ -149,4 +151,5 @@ test_files:
 - test/rbbt/statistics/test_hypergeometric.rb
 - test/rbbt/vector/test_model.rb
 - test/rbbt/vector/model/test_svm.rb
+- test/rbbt/test_stan.rb
 - test/test_helper.rb