rbbt-image 0.1.37 → 0.1.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45c46d847d1522d0d774cbdad29d9ba70bb1cc42036234dc53de6d0723403e02
-  data.tar.gz: eef195a183125dbb8f9c3d3f12157a3bff0c4ed12ef1118cd88acf3878d7bba4
+  metadata.gz: a15ae2d4c2941a4e142275466e4fbc6ec65391d778bc658fed5530b722d11b95
+  data.tar.gz: f2fbcb0d6e3c5e70b1f25665879b7a24be34bebc69592a04964630bf29ca5c82
 SHA512:
-  metadata.gz: e3d2cac785c2cd0d192f2913af0f40e6a1abb65b3daa8efd8a2e4f4c97aba672c6421e2e1a8222f753694ac7e52493de3be2af7080943e3bcaf3faec16e44aa2
-  data.tar.gz: 90c007ee52794941a8c421882779d2e6a1e147e83e9c77e59a1dd4e57e248bed87e4fd20165d530a71933e783b1e929eaa652aec86b6dc1afb176cd01a3984db
+  metadata.gz: 745c051c7406d23f7997952e664582aceea31ee7ec6a58574fa50af4143ab3e937f7880ed07701405387c5ce9b2daf69eb3c60a04b6120c7ac36a20822d53181
+  data.tar.gz: e2bf862a7c9c8a76342768100d7e8ef50bf3cd6d44b4ebd21c27b16bc6a9cb0714c49c934a5bebcb56347c660991331f056dc26c1e15a68f0e69499d439d75db

data/bin/build_rbbt_provision_sh.rb

@@ -30,6 +30,7 @@ $ #{$0} [options]
 -sr--skip_ruby Skip ruby setup installation
 -sp--skip_perl Skip perl setup installation
 -spy--skip_python Skip python setup installation
+-spsl--skip_slurm_loopback Skip setting the slurm loopback
 -sg--skip_gem Skip ruby gem installation
 -sR--skip_R Skip R setup
 -su--skip_user_setup Skip user setup
@@ -72,6 +73,7 @@ SKIP_TOKYOCABINET= options[:skip_tokyocabinet]
 SKIP_RUBY = options[:skip_ruby]
 SKIP_PERL = options[:skip_perl]
 SKIP_PYTHON = options[:skip_python]
+SKIP_SLURM_LOOPBACK = options[:skip_slurm_loopback]
 R_CUSTOM = options[:R_custom]
 SKIP_R = options[:skip_R]
 SKIP_BOOT = options[:skip_bootstrap]
@@ -123,6 +125,8 @@ end
 if not SKIP_BASE_SYSTEM and R_CUSTOM
   "echo 1.1 Setting custom R"
   File.read(script_dir + 'R_setup.sh') 
+else
+  "echo SKIPPED Custom R\necho"
 end 
 }
 
@@ -130,6 +134,8 @@ end
 if not SKIP_R 
   "echo 1.2 Install R packages"
   File.read(script_dir + 'R_packages.sh') 
+else
+  "echo SKIPPED installing R packages\necho"
 end 
 }
 
@@ -138,7 +144,7 @@ echo "2. Setting up tokyocabinet"
 if not SKIP_TOKYOCABINET
   File.read(script_dir + 'tokyocabinet_setup.sh')
 else
-  "echo SKIPPED\necho"
+  "echo SKIPPED TokyoCabinet\necho"
 end
 }
 
@@ -147,7 +153,7 @@ echo "3. Setting up ruby"
 if not SKIP_RUBY
   "export RUBY_VERSION='#{options[:ruby_version]}'\n" << File.read(script_dir + 'ruby_setup.sh') 
 else
-  "echo SKIPPED\necho"
+  "echo SKIPPED Ruby\necho"
 end 
 }
 
@@ -157,7 +163,7 @@ echo "3.1. Setting up gems"
 if not SKIP_GEM
   File.read(script_dir + 'gem_setup.sh') 
 else
-  "echo SKIPPED\necho"
+  "echo SKIPPED Ruby gems\necho"
 end 
 }
 
@@ -167,7 +173,7 @@ echo "4.1 Setting up perl"
 if not SKIP_PERL
   File.read(script_dir + 'perl_setup.sh') 
 else
-  "echo SKIPPED\necho"
+  "echo SKIPPED Perl\necho"
 end 
 }
 
@@ -176,7 +182,16 @@ echo "4.2 Setting up python"
 if not SKIP_PYTHON
   File.read(script_dir + 'python_setup.sh') 
 else
-  "echo SKIPPED\necho"
+  "echo SKIPPED Python\necho"
+end 
+}
+
+echo "4.3 Setting up SLURM loopback"
+#{
+if not SKIP_SLURM_LOOPBACK
+  File.read(script_dir + 'slurm_loopback.sh') 
+else
+  "echo SKIPPED SLURM lookback\necho"
 end 
 }
 
@@ -185,7 +200,7 @@ echo "source /etc/rbbt_environment" >> /etc/profile
 EOF
 
 provision_script +=<<-EOF
-echo "4. Configuring user"
+echo "5. Configuring user"
 EOF
 
 if not SKIP_USER
@@ -198,43 +213,47 @@ if [[ '#{ USER }' == 'root' ]] ; then
 else
   useradd -ms /bin/bash #{USER}
   home_dir='/home/#{USER}'
+
+  if [ -d /usr/local/miniconda3 ]; then
+    chgrp #{USER} -R /usr/local/miniconda3
+    chmod g+rw -R /usr/local/miniconda3
+  fi
 fi
 
 user_script=$home_dir/.rbbt/bin/config_user
 mkdir -p $(dirname $user_script)
-chown -R #{USER} /home/#{USER}/.rbbt/
-
+chown -R #{USER} $home_dir/.rbbt/
 
 # set user configuration script
 cat > $user_script <<'EUSER'
 
 . /etc/profile
 
-echo "4.1. Loading custom variables"
+echo "5.1. Loading custom variables"
 #{
   VARIABLES.collect do |variable,value|
     "export " << ([variable,'"' << value.to_s << '"'] * "=")
   end * "\n"
 }
 
-echo "4.2. Loading default variables"
+echo "5.2. Loading default variables"
 #{File.read(script_dir + 'variables.sh')}
 
-echo "4.3. Configuring rbbt"
+echo "5.3. Configuring rbbt"
 #{File.read(script_dir + 'user_setup.sh')}
 EUSER
 
-echo "4.4. Running user configuration as '#{USER}'"
+echo "5.4. Running user configuration as '#{USER}'"
 chown #{USER} $user_script;
 su -l -c "bash $user_script" #{USER}
 
   EOF
 else
-  provision_script += "echo SKIPPED\necho\n\n"
+  provision_script += "echo SKIPPED user configuration\necho\n\n"
 end
 
 provision_script +=<<-EOF
-echo "5. Bootstrapping workflows as '#{USER}'"
+echo "6. Bootstrapping workflows as '#{USER}'"
 echo
 EOF
 
@@ -252,21 +271,22 @@ user_script=$home_dir/.rbbt/bin/bootstrap
 cat > $user_script <<'EUSER'
 
 . /etc/profile
+. /etc/rbbt_environment
 
-echo "5.1. Loading custom variables"
+echo "6.1. Loading custom variables"
 #{
   VARIABLES.collect do |variable,value|
     "export " << ([variable,'"' << value.to_s << '"'] * "=")
   end * "\n"
 }
 
-echo "5.2. Loading default variables"
+echo "6.2. Loading default variables"
 #{File.read(script_dir + 'variables.sh')}
 
-echo "5.3. Configuring rbbt"
+echo "6.3. Configuring rbbt"
 #{File.read(script_dir + 'user_setup.sh')}
 #
-echo "5.4. Install and bootstrap"
+echo "6.4. Install and bootstrap"
 #{File.read(script_dir + "bootstrap.sh")}
 EUSER
 
@@ -275,7 +295,7 @@ su -l -c "bash $user_script" #{USER}
 
   EOF
 else
-  provision_script += "echo SKIPPED\necho\n\n"
+  provision_script += "echo SKIPPED User configuration\necho\n\n"
 end
 
 provision_script +=<<-EOF
@@ -302,7 +322,7 @@ EOF
 
 
 if docker_image = options[:docker]
-  container_dependency = options[:container_dependency] || 'ubuntu:disco'
+  container_dependency = options[:container_dependency] || 'ubuntu:focal'
   dockerfile = options[:dockerfile] || File.join(root_dir, 'Dockerfile')
   dockerfile_text = Open.read(dockerfile)
   dockerfile_text.sub!(/^FROM.*/,'FROM ' + container_dependency) if container_dependency
@@ -339,7 +359,7 @@ end
 
 if singularity_image = options[:singularity]
 
-  container_dependency = options[:container_dependency] || 'ubuntu:disco'
+  container_dependency = options[:container_dependency] || 'ubuntu:focal'
   TmpFile.with_file(nil, false) do |dir|
     Path.setup(dir)
 
@@ -357,10 +377,12 @@ EOS
   ln -s /etc/rbbt_environment /.singularity.d/env/99-rbbt_environment.sh 
   chmod +x /.singularity.d/env/99-rbbt_environment.sh
   bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share' 
+  bash -c '[ -d /usr/local/workflows ] || mkdir -p /usr/local/workflows' 
   bash -c '[ -d /software/rbbt ] || mkdir -p /software/rbbt'
-  bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && mv /home/#{USER}/.rbbt/var/ /var/rbbt' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && mv /home/#{USER}/.rbbt/var/ /var/rbbt' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && mv /home/#{USER}/.rbbt/var/ /var/rbbt' || echo -n ""
+  bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && rm -Rf /home/#{USER}/.rbbt/var/' || echo -n ""
   bash -c '[ -d /home/#{USER}/.rbbt/share/ ] && mv /home/#{USER}/.rbbt/share/ /usr/local/share/rbbt' || echo -n ""
+  bash -c '[ -d /home/#{USER}/.rbbt/workflows/ ] && mv /home/#{USER}/.rbbt/workflows/ /usr/local/workflows/rbbt' || echo -n ""
   bash -c '[ -d /home/#{USER}/.rbbt/software/opt ] && mv /home/#{USER}/.rbbt/software/opt /software/rbbt/opt' || echo -n ""
   bash -c '[ -d /home/#{USER}/.rbbt/software/src ] && mv /home/#{USER}/.rbbt/software/src /software/rbbt/src' || echo -n ""
   bash -c '[ -d /home/#{USER}/.rbbt/software/scm ] && mv /home/#{USER}/.rbbt/software/scm /software/rbbt/scm' || echo -n ""
@@ -374,11 +396,11 @@ EOF
     cmd_boot =  "singularity build #{singularity_image} '#{dir["singularity_bootstrap"]}'"
 
 
-    puts "**************"
-    puts "CREATING IMAGE: #{dir["singularity_bootstrap"].find}"
-    puts "**************"
-    puts cmd_create
-    CMD.cmd_log(cmd_create, :log => 4)
+    #puts "**************"
+    #puts "CREATING IMAGE: #{dir["singularity_bootstrap"].find}"
+    #puts "**************"
+    #puts cmd_create
+    #CMD.cmd_log(cmd_create, :log => 4)
 
     puts "**************"
     puts "BUILDING IMAGE: #{dir["singularity_bootstrap"].find}"
@@ -390,6 +412,7 @@ EOF
 end
 
 if options[:virtualbox]
+  container_dependency = options[:container_dependency] || 'ubuntu:focal'
   TmpFile.with_file(nil, false) do |dir|
     Path.setup(dir)
 
@@ -416,11 +439,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 end
     EOF
 
+    Log.debug "Running vagrant on #{dir}"
     Misc.in_dir(dir) do
-      io = CMD.cmd('vagrant up', :pipe => true, :log => true)
-      while line = io.gets
-        puts line
-      end
+      CMD.cmd_log(:vagrant, "up")
     end
   end
 end

data/share/provision_scripts/R_packages.sh

@@ -37,10 +37,15 @@ R_install_packages 'digest' 'gtable' 'reshape2' 'scales' 'proto'
 
 R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/ggplot2/ggplot2_1.0.0.tar.gz' ggplot2
 
-R_install_packages NMF Cairo drc Rserve gridSVG ggthemes mclust randomForest viper
+R_install_packages NMF Cairo drc gridSVG ggthemes mclust randomForest viper
+
+echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla
+
 R_install_packages pheatmap VennDiagram Hmisc pls gridSVG
 R_install_packages tidyverse
 
+R_install_packages UpSetR
+
 R_biocLite limma viper
 
 R_install_packages pROC txtplot

data/share/provision_scripts/R_setup.sh

@@ -7,7 +7,7 @@ cd /tmp
 
 apt-get remove r-base-core
 
-wget https://cran.rediris.es/src/base/R-3/R-3.6.1.tar.gz -O R.tar.gz
+wget https://cran.r-project.org/src/base/R-4/R-4.2.1.tar.gz -O R.tar.gz
 tar -xvzf R.tar.gz
 
 cd R-*/

data/share/provision_scripts/bootstrap.sh

@@ -5,13 +5,25 @@
 
 for workflow in $BOOTSTRAP_WORKFLOWS; do
     rbbt workflow install $workflow 
+
+    pip_requirements_file=$(rbbt_find.rb -w $workflow requirements.pip --nocolor)
+    [ -f $pip_requirements_file ] && echo PIP $pip_requirements_file && pip install -r $pip_requirements_file
+    unset pip_requirements_file
+
+    pip_requirements_file2=$(rbbt_find.rb -w $workflow requirements.pip2 --nocolor)
+    [ -f $pip_requirements_file2 ] && pip install -r $pip_requirements_file2
+    unset pip_requirements_file2
+
+    pip_requirements_file3=$(rbbt_find.rb -w $workflow requirements.pip3 --nocolor)
+    [ -f $pip_requirements_file3 ] && pip install -r $pip_requirements_file3
+    unset pip_requirements_file3
 done
 
 export RBBT_WORKFLOW_AUTOINSTALL=true
-export RBBT_LOG
+export RBBT_LOG=0
 
 for workflow in $BOOTSTRAP_WORKFLOWS; do
     echo "Bootstrapping $workflow on $BOOTSTRAP_CPUS CPUs"
     [ -d $HOME/.rbbt/tmp/ ] || mkdir -p $HOME/.rbbt/tmp/
-    rbbt workflow cmd $workflow bootstrap --cpus $BOOTSTRAP_CPUS &> $HOME/.rbbt/tmp/${workflow}.bootstrap.log
+    rbbt workflow cmd $workflow bootstrap --config_keys "cpus $BOOTSTRAP_CPUS bootstrap" > $HOME/.rbbt/tmp/${workflow}.bootstrap.log
 done

data/share/provision_scripts/gem_setup.sh

@@ -27,14 +27,14 @@ echo "* Installing bulk gems and Rbbt"
 gem install --no-document --force \
     tokyocabinet \
     ruby-prof \
-    rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources rbbt-phgx rbbt-GE \
+    rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources \
     rserve-client \
 
 # Get good version of lockfile
 #wget http://ubio.bioinfo.cnio.es/people/mvazquezg/lockfile-2.1.4.gem -O /tmp/lockfile-2.1.4.gem
 echo "* Installing our version of lockfile"
-wget http://github.com/mikisvaz/lockfile/raw/master/lockfile-2.1.5.gem -O /tmp/lockfile-2.1.5.gem
-gem install --no-document /tmp/lockfile-2.1.5.gem
+wget https://github.com/mikisvaz/lockfile/raw/master/lockfile-2.1.6.gem -O /tmp/lockfile-2.1.6.gem
+gem install --no-document /tmp/lockfile-2.1.6.gem
 
 # Extra things for web interface
 gem install --no-document --force \
@@ -42,5 +42,5 @@ gem install --no-document --force \
     ruby-prof \
     rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources rbbt-phgx rbbt-GE \
     rserve-client \
-    uglifier therubyracer kramdown pandoc pandoc-ruby\
-    puma prawn prawn-svg
+    uglifier therubyracer kramdown pandoc pandoc-ruby \
+    puma prawn prawn-svg mimemagic 

data/share/provision_scripts/hacks.sh

@@ -1,2 +1,2 @@
 # For BNReduction work
-ln -s /bin/sed /usr/bin/sed
+#ln -s /bin/sed /usr/bin/sed

data/share/provision_scripts/python_setup.sh

@@ -2,15 +2,24 @@ wget "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O
 echo "# Python conda " >> /etc/rbbt_environment
 
 bash /tmp/miniconda.sh -b -p /usr/local/miniconda3
+
 ln -s /usr/local/miniconda3/bin/conda /usr/local/bin/conda
 echo '_add_path "/usr/local/miniconda3/bin/"' >> /etc/rbbt_environment
 
 conda create --yes -n python2 python=2.7 pip
 
-conda create --yes -n python3 python=3 pip
+conda create --yes -n python3 python=3.6 pip
 
 echo '_add_path "/usr/local/miniconda3/envs/python2/bin/"' >> /etc/rbbt_environment
 echo '_add_path "/usr/local/miniconda3/envs/python3/bin/"' >> /etc/rbbt_environment
+echo '' >> /etc/rbbt_environment
+echo '# Setup conda environment' >> /etc/rbbt_environment
+echo 'export CONDA_PKGS_DIRS=~/tmp/conda/pkgs' >> /etc/rbbt_environment
+echo '. /usr/local/miniconda3/etc/profile.d/conda.sh' >> /etc/rbbt_environment
 
 rm -Rf /usr/local/miniconda3/pkgs
 source /etc/rbbt_environment
+
+pip install ansible
+
+gem install pycall

data/share/provision_scripts/ruby_setup.sh

@@ -13,7 +13,7 @@ cd /tmp/ruby-install
 wget https://cache.ruby-lang.org/pub/ruby/$_small_version/ruby-${RUBY_VERSION}.tar.gz -O "ruby.tar.gz"
 tar -xvzf ruby.tar.gz
 cd ruby-*/
-./configure --prefix=/usr/local
+./configure --prefix=/usr/local --enable-shared
 make && make install
 
 unset _small_version

data/share/provision_scripts/slurm_loopback.sh

@@ -0,0 +1,45 @@
+#!/bin/bash -x
+
+# Prepare SLURM loopback
+# ======================
+
+cat <<'EOF' > /usr/local/bin/loopback_cmd
+#!/bin/bash
+
+if [ -z "$LOOPBACK_KEY" ]; then
+  LOOPBACK_KEY="$HOME/.ssh/slurm-loopback"
+fi
+
+if [ ! -f "$LOOPBACK_KEY" ]; then
+  echo "Prepare a ssh key on '$LOOPBACK_KEY' or point to it with environment variable LOOPBACK_KEY." >&2
+  exit -1
+else
+  ssh localhost -i $LOOPBACK_KEY "$@"
+fi
+EOF
+
+
+cat <<'EOF' > /usr/local/bin/squeue 
+#!/bin/bash
+
+loopback_cmd squeue "$@"
+EOF
+
+
+cat <<'EOF' > /usr/local/bin/sbatch 
+#!/bin/bash
+
+loopback_cmd sbatch "$@"
+EOF
+
+cat <<'EOF' > /usr/local/bin/sacct 
+#!/bin/bash
+
+loopback_cmd sacct "$@"
+EOF
+
+chmod +x /usr/local/bin/loopback_cmd /usr/local/bin/squeue /usr/local/bin/sbatch /usr/local/bin/sacct
+
+
+
+

data/share/provision_scripts/ubuntu_setup.sh

@@ -76,14 +76,16 @@ DEBIAN_FRONTEND=noninteractive apt-get -y install \
   vim \
   wget \
   libc6 \
+  time numactl xvfb \
   bison autoconf g++ libxslt1-dev make \
   zlib1g-dev libbz2-dev libreadline-dev \
   rsync wget curl git openssl libyaml-0-2 libyaml-dev \
   openjdk-8-jdk \
   libcairo2 libcairo2-dev r-base-core r-base-dev r-cran-rserve liblzma5 liblzma-dev libcurl4-openssl-dev \
   build-essential zlib1g-dev libssl-dev libyaml-dev libffi-dev \
-  python3 python \
+  python3 python libtorch \
   tcsh gawk \
+  ansible \
   pandoc \
   libtbb-dev
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-image
 version: !ruby/object:Gem::Version
-  version: 0.1.37
+  version: 0.1.39
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-31 00:00:00.000000000 Z
+date: 2023-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -48,6 +48,7 @@ files:
 - share/provision_scripts/perl_setup.sh
 - share/provision_scripts/python_setup.sh
 - share/provision_scripts/ruby_setup.sh
+- share/provision_scripts/slurm_loopback.sh
 - share/provision_scripts/tokyocabinet_setup.sh
 - share/provision_scripts/ubuntu_setup.sh
 - share/provision_scripts/user_setup.sh
@@ -71,7 +72,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Build docker and Vagrant (VM) images