rbbt-image 0.1.39 → 0.1.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a15ae2d4c2941a4e142275466e4fbc6ec65391d778bc658fed5530b722d11b95
-  data.tar.gz: f2fbcb0d6e3c5e70b1f25665879b7a24be34bebc69592a04964630bf29ca5c82
+  metadata.gz: db006b39f539ccfe01887fe76863904209abc11ab2caf2c53d9b77b35f463662
+  data.tar.gz: efdead2dfa7cd0d79303524e415cf1acdcc3db2cdeb1aa562e268d44e9f4a372
 SHA512:
-  metadata.gz: 745c051c7406d23f7997952e664582aceea31ee7ec6a58574fa50af4143ab3e937f7880ed07701405387c5ce9b2daf69eb3c60a04b6120c7ac36a20822d53181
-  data.tar.gz: e2bf862a7c9c8a76342768100d7e8ef50bf3cd6d44b4ebd21c27b16bc6a9cb0714c49c934a5bebcb56347c660991331f056dc26c1e15a68f0e69499d439d75db
+  metadata.gz: b481e7a4daf5a2f4155959d7e2b53aeee8d8cb2da95ba252df5f590407679bfe4b5faf70f736ffd735b8bf1cc08717fb8b80471042649a9a4ffc8a2aafd25b67
+  data.tar.gz: bb5db5c1d64cecdd49e8bb8e1557c296e2a29e36be7ce3fa3c12644e40448157456b988ed5c4ab1a5e170f14bfec4978768422444b2206a239bbcd0fce84cea1

data/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:cosmic
 USER root
 # START PROVISION
 ADD provision.sh /tmp/provision.sh
-RUN /bin/bash /tmp/provision.sh
+RUN /bin/sh +x /tmp/provision.sh
 # END PROVISION
-USER rbbt
-ENV HOME /home/rbbt
+#USER rbbt
+#ENV HOME /home/rbbt

data/bin/build_rbbt_provision_sh.rb

@@ -25,19 +25,10 @@ $ #{$0} [options]
 -ws--workflow_server* Rbbt remote workflow server
 -rr--remote_resources* Remote resources to gather from file-server
 -rw--remote_workflows* Remote workflows server from workflow-server
--ss--skip_base_system Skip base system installation
--st--skip_tokyocabinet Skip tokyocabinet setup installation
--sr--skip_ruby Skip ruby setup installation
--sp--skip_perl Skip perl setup installation
--spy--skip_python Skip python setup installation
--spsl--skip_slurm_loopback Skip setting the slurm loopback
--sg--skip_gem Skip ruby gem installation
--sR--skip_R Skip R setup
--su--skip_user_setup Skip user setup
--sb--skip_bootstrap Skip user bootstrap
--Rc--R_custom Install a custom installation of R
--Rp--R_packages Install basic R packages
+-bs--base_system* Version of base system initialization script to use (default: ubuntu)
 -Rbv--ruby_version* Ruby version to use, using three numbers (defaults to 2.4.1)
+-d--do* List of steps to do
+-nd--not_do* List of steps to not do
 -op--optimize Optimize files under ~/.rbbt
 -dep--container_dependency* Use a different image in Dockerfile, Singularity and Virtualbox
 -dt--docker* Build docker image using the provided name
@@ -67,28 +58,26 @@ script_dir = File.join(root_dir, "share/provision_scripts/")
 #  options[:skip_bootstrap] = true
 #end
 
-USER = options[:user] || 'rbbt'
-SKIP_BASE_SYSTEM = options[:skip_base_system]
-SKIP_TOKYOCABINET= options[:skip_tokyocabinet]
-SKIP_RUBY = options[:skip_ruby]
-SKIP_PERL = options[:skip_perl]
-SKIP_PYTHON = options[:skip_python]
-SKIP_SLURM_LOOPBACK = options[:skip_slurm_loopback]
-R_CUSTOM = options[:R_custom]
-SKIP_R = options[:skip_R]
-SKIP_BOOT = options[:skip_bootstrap]
-SKIP_USER = options[:skip_user_setup]
-SKIP_GEM = options[:skip_gem]
-OPTIMIZE = options[:optimize]
+all_steps = %w(functions base_system tokyocabinet ruby_custom gem java R_custom R perl_custom python_custom python user slurm_loopback hacks)
+
+
+do_steps = options.include?("do")? (all_steps & options[:do].split(",")) : all_steps
+not_do_steps = options.include?(:not_do)? options[:not_do].split(",") : all_steps - do_steps
+
+do_steps << 'base_system' if options[:base_system]
+do_steps << 'user' if options[:workflow]
+
+OPTIMIZE    = options[:optimize] 
+USER        = options[:user] || 'rbbt'
+CONTAINER_DEP = options[:container_dependency] ||  'alpine'
+BASE_SYSTEM = options[:base_system] || CONTAINER_DEP
 
 VARIABLES={
- :RBBT_LOG => 0,
- :BOOTSTRAP_WORKFLOWS => (options[:workflow] || "").split(/[\s,]+/)*" ",
- :REMOTE_RESOURCES => (options[:remote_resources] || "KEGG").split(/[\s,]+/)*" "
+  :RBBT_LOG => 0,
+  :BOOTSTRAP_WORKFLOWS => (options[:workflow] || "").split(/[\s,]+/)*" ",
+  :REMOTE_RESOURCES => (options[:remote_resources] || "").split(/[\s,]+/)*" "
 }
 
-VARIABLES[:BOOTSTRAP_WORKFLOWS] = "" if VARIABLES[:BOOTSTRAP_WORKFLOWS] == 'none'
-
 VARIABLES[:RBBT_SERVER] = options[:server] if options[:server]
 VARIABLES[:RBBT_FILE_SERVER] = options[:file_server] if options[:file_server]
 VARIABLES[:RBBT_WORKFLOW_SERVER] = options[:workflow_server] if options[:workflow_server]
@@ -98,231 +87,88 @@ VARIABLES[:RBBT_NO_PROGRESS] = "true" if options[:nobar]
 
 options[:ruby_version] ||= "2.6.4"
 
-
 provision_script =<<-EOF
 #!/bin/bash -x
 
-echo "RUNNING PROVISION"
-echo
+# PROVISION FILE
 echo "CMD: #{File.basename($0) + " " + orig_argv.collect{|a| a =~ /\s/ ? "\'#{a}\'" : a }.join(" ")}"
-echo
-echo -n "Starting: "
-date
 
-EOF
+test -f /etc/profile && source /etc/profile
+test -f /etc/rbbt_environment && source /etc/rbbt_environment
 
-provision_script +=<<-EOF
-echo "1. Provisioning base system"
-#{
-if not SKIP_BASE_SYSTEM
-  File.read(script_dir + 'ubuntu_setup.sh') 
-else
-  "echo SKIPPED\necho"
-end 
-}
-
-#{
-if not SKIP_BASE_SYSTEM and R_CUSTOM
-  "echo 1.1 Setting custom R"
-  File.read(script_dir + 'R_setup.sh') 
-else
-  "echo SKIPPED Custom R\necho"
-end 
-}
-
-#{
-if not SKIP_R 
-  "echo 1.2 Install R packages"
-  File.read(script_dir + 'R_packages.sh') 
-else
-  "echo SKIPPED installing R packages\necho"
-end 
-}
-
-echo "2. Setting up tokyocabinet"
-#{
-if not SKIP_TOKYOCABINET
-  File.read(script_dir + 'tokyocabinet_setup.sh')
-else
-  "echo SKIPPED TokyoCabinet\necho"
-end
-}
-
-echo "3. Setting up ruby"
-#{
-if not SKIP_RUBY
-  "export RUBY_VERSION='#{options[:ruby_version]}'\n" << File.read(script_dir + 'ruby_setup.sh') 
-else
-  "echo SKIPPED Ruby\necho"
-end 
-}
-
-
-echo "3.1. Setting up gems"
-#{
-if not SKIP_GEM
-  File.read(script_dir + 'gem_setup.sh') 
-else
-  "echo SKIPPED Ruby gems\necho"
-end 
-}
-
-echo "4 Setting up other stuff"
-echo "4.1 Setting up perl"
-#{
-if not SKIP_PERL
-  File.read(script_dir + 'perl_setup.sh') 
-else
-  "echo SKIPPED Perl\necho"
-end 
-}
-
-echo "4.2 Setting up python"
-#{
-if not SKIP_PYTHON
-  File.read(script_dir + 'python_setup.sh') 
-else
-  "echo SKIPPED Python\necho"
-end 
-}
-
-echo "4.3 Setting up SLURM loopback"
-#{
-if not SKIP_SLURM_LOOPBACK
-  File.read(script_dir + 'slurm_loopback.sh') 
-else
-  "echo SKIPPED SLURM lookback\necho"
-end 
-}
-
-echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
-echo "source /etc/rbbt_environment" >> /etc/profile
 EOF
 
-provision_script +=<<-EOF
-echo "5. Configuring user"
-EOF
-
-if not SKIP_USER
-  provision_script +=<<-EOF
-####################
-# USER CONFIGURATION
-
-if [[ '#{ USER }' == 'root' ]] ; then
-  home_dir='/root'
-else
-  useradd -ms /bin/bash #{USER}
-  home_dir='/home/#{USER}'
-
-  if [ -d /usr/local/miniconda3 ]; then
-    chgrp #{USER} -R /usr/local/miniconda3
-    chmod g+rw -R /usr/local/miniconda3
-  fi
-fi
-
-user_script=$home_dir/.rbbt/bin/config_user
-mkdir -p $(dirname $user_script)
-chown -R #{USER} $home_dir/.rbbt/
-
-# set user configuration script
-cat > $user_script <<'EUSER'
-
-. /etc/profile
-
-echo "5.1. Loading custom variables"
-#{
-  VARIABLES.collect do |variable,value|
-    "export " << ([variable,'"' << value.to_s << '"'] * "=")
-  end * "\n"
-}
-
-echo "5.2. Loading default variables"
-#{File.read(script_dir + 'variables.sh')}
-
-echo "5.3. Configuring rbbt"
-#{File.read(script_dir + 'user_setup.sh')}
-EUSER
-
-echo "5.4. Running user configuration as '#{USER}'"
-chown #{USER} $user_script;
-su -l -c "bash $user_script" #{USER}
-
-  EOF
-else
-  provision_script += "echo SKIPPED user configuration\necho\n\n"
-end
-
-provision_script +=<<-EOF
-echo "6. Bootstrapping workflows as '#{USER}'"
-echo
-EOF
-
-if not SKIP_BOOT
-  provision_script +=<<-EOF
-
-if [[ '#{ USER }' == 'root' ]] ; then
-  home_dir='/root'
-else
-  home_dir='/home/#{USER}'
-fi
-
-user_script=$home_dir/.rbbt/bin/bootstrap
-
-cat > $user_script <<'EUSER'
-
-. /etc/profile
-. /etc/rbbt_environment
-
-echo "6.1. Loading custom variables"
-#{
-  VARIABLES.collect do |variable,value|
-    "export " << ([variable,'"' << value.to_s << '"'] * "=")
-  end * "\n"
-}
-
-echo "6.2. Loading default variables"
-#{File.read(script_dir + 'variables.sh')}
-
-echo "6.3. Configuring rbbt"
-#{File.read(script_dir + 'user_setup.sh')}
-#
-echo "6.4. Install and bootstrap"
-#{File.read(script_dir + "bootstrap.sh")}
-EUSER
-
-chown #{USER} $user_script;
-su -l -c "bash $user_script" #{USER}
+all_steps.each_with_index do |step,i|
+  if ! do_steps.include?(step)
+    provision_script += "#" + "NOT DO #{step}\n"
+    next
+  end
 
-  EOF
-else
-  provision_script += "echo SKIPPED User configuration\necho\n\n"
+  provision_script += "#" + "DO #{step}\n"
+  provision_script += case step
+                      when 'base_system'
+                        File.read(script_dir + "#{BASE_SYSTEM}_setup.sh") 
+                      when 'user'
+                        user_script =<<~EOF
+                          . /etc/profile
+                          . /etc/rbbt_environment
+
+                          echo "6.1. Loading custom variables"
+                          #{
+                          VARIABLES.collect do |variable,value|
+                            "export " << ([variable,'"' << value.to_s << '"'] * "=")
+                          end * "\n"
+                          }
+
+                          echo "6.2. Loading default variables"
+                          #{File.read(script_dir + 'variables.sh')}
+
+                          echo "6.3. Configuring rbbt"
+                          #{File.read(script_dir + 'user.sh')}
+                          
+                          echo "6.4. Install and bootstrap"
+                          #{File.read(script_dir + "bootstrap.sh")}
+                          
+                          echo "6.5. Migrate shared files"
+                          #{File.read(script_dir + "migrate.sh")}
+                        EOF
+
+                        <<~EOF
+                          if [[ '#{ USER }' == 'root' ]] ; then
+                            home_dir='/root'
+                          else
+                            adduser --disabled-password --gecos "" #{USER}
+                            addgroup rbbt
+                            adduser #{USER} rbbt
+                            home_dir='/home/#{USER}'
+                            chown -R #{USER} $home_dir/
+                          fi
+
+                          mkdir -p $home_dir/.rbbt/bin/
+                          user_script=$home_dir/.rbbt/bin/bootstrap
+                          chown -R #{USER} $home_dir/
+
+                          for d in /usr/local/var/rbbt /usr/local/share/rbbt /usr/local/workflows/rbbt /usr/local/software/rbbt; do
+                            mkdir -p $d
+                            chgrp rbbt $d
+                            chown rbbt $d
+                            chmod g+w $d
+                          done
+
+                          cat > $user_script <<'EUSER'
+                          #{user_script}
+                          EUSER
+
+
+                          su -l -c "sh $user_script" #{USER}
+                        EOF
+                      else
+                        File.read(script_dir + "#{step}.sh") 
+                      end
 end
 
-provision_script +=<<-EOF
-# HACKS
-# =====
-
-#{File.read(script_dir + 'hacks.sh')}
-EOF
-
-provision_script +=<<-EOF
-# CODA
-# ====
-
-apt-get clean
-rm -rf /var/lib/apt/lists/* /var/tmp/* /usr/share/doc /usr/share/man /usr/local/share/ri
-
-#{ "su -l -c 'rbbt system optimize /home/#{USER}/.rbbt ' #{USER}" if OPTIMIZE}
-
-echo
-echo -n "Installation done: "
-date
-
-EOF
-
-
-if docker_image = options[:docker]
-  container_dependency = options[:container_dependency] || 'ubuntu:focal'
+if docker_image = options[:docker] 
+  container_dependency = CONTAINER_DEP
   dockerfile = options[:dockerfile] || File.join(root_dir, 'Dockerfile')
   dockerfile_text = Open.read(dockerfile)
   dockerfile_text.sub!(/^FROM.*/,'FROM ' + container_dependency) if container_dependency
@@ -359,33 +205,44 @@ end
 
 if singularity_image = options[:singularity]
 
-  container_dependency = options[:container_dependency] || 'ubuntu:focal'
+  container_dependency = CONTAINER_DEP
   TmpFile.with_file(nil, false) do |dir|
     Path.setup(dir)
 
     provision_file = dir['provision.sh']
 
-    bootstrap_text=<<-EOF
+    if container_dependency.include? '.sif'
+      bootstrap_text=<<-EOF
+Bootstrap: localimage
+From: #{container_dependency}
+EOF
+    else
+      bootstrap_text=<<-EOF
 Bootstrap: docker
 From: #{container_dependency}
+EOF
+    end
 
+    bootstrap_text+=<<-EOF
 %post
   cat > /image_provision.sh <<"EOS"
   #{provision_script}
 EOS
-  bash -x /image_provision.sh 2>&1 | tee /image_provision.log
-  ln -s /etc/rbbt_environment /.singularity.d/env/99-rbbt_environment.sh 
+  sh -x /image_provision.sh 2>&1 | tee /image_provision.log
+  bash -c '[ -f /.singularity.d/env/99-rbbt_environment.sh ] || ln -s /etc/rbbt_environment /.singularity.d/env/99-rbbt_environment.sh'
   chmod +x /.singularity.d/env/99-rbbt_environment.sh
-  bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share' 
-  bash -c '[ -d /usr/local/workflows ] || mkdir -p /usr/local/workflows' 
-  bash -c '[ -d /software/rbbt ] || mkdir -p /software/rbbt'
+  #bash -c '[ -d /home/#{USER}/.local/lib/ ] && (rsync -av /home/#{USER}/.local/lib/ /usr/lib/' && chmod 755 /usr/lib/python*/site-packages/ && chown -R root /usr/lib/python*/site-packages/ && rm /home/#{USER}/.local/lib/) || echo -n ""
+  #bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share' 
+  #bash -c '[ -d /usr/local/share ] || mkdir -p /usr/local/share' 
+  #bash -c '[ -d /usr/local/workflows ] || mkdir -p /usr/local/workflows' 
+  #bash -c '[ -d /software/rbbt ] || mkdir -p /software/rbbt'
   #bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && mv /home/#{USER}/.rbbt/var/ /var/rbbt' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && rm -Rf /home/#{USER}/.rbbt/var/' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/share/ ] && mv /home/#{USER}/.rbbt/share/ /usr/local/share/rbbt' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/workflows/ ] && mv /home/#{USER}/.rbbt/workflows/ /usr/local/workflows/rbbt' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/software/opt ] && mv /home/#{USER}/.rbbt/software/opt /software/rbbt/opt' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/software/src ] && mv /home/#{USER}/.rbbt/software/src /software/rbbt/src' || echo -n ""
-  bash -c '[ -d /home/#{USER}/.rbbt/software/scm ] && mv /home/#{USER}/.rbbt/software/scm /software/rbbt/scm' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/var/ ] && rm -Rf /home/#{USER}/.rbbt/var/' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/share/ ] && mv /home/#{USER}/.rbbt/share/ /usr/local/share/rbbt' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/workflows/ ] && mv /home/#{USER}/.rbbt/workflows/ /usr/local/workflows/rbbt' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/software/opt ] && mv /home/#{USER}/.rbbt/software/opt /software/rbbt/opt' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/software/src ] && mv /home/#{USER}/.rbbt/software/src /software/rbbt/src' || echo -n ""
+  #bash -c '[ -d /home/#{USER}/.rbbt/software/scm ] && mv /home/#{USER}/.rbbt/software/scm /software/rbbt/scm' || echo -n ""
 EOF
     FileUtils.mkdir_p dir
     Open.write(dir["singularity_bootstrap"].find, bootstrap_text)
@@ -412,7 +269,7 @@ EOF
 end
 
 if options[:virtualbox]
-  container_dependency = options[:container_dependency] || 'ubuntu:focal'
+  container_dependency = CONTAINER_DEP
   TmpFile.with_file(nil, false) do |dir|
     Path.setup(dir)
 

data/share/provision_scripts/R.sh

@@ -0,0 +1,66 @@
+#!/bin/bash -x
+
+# INSTALL
+# =======
+
+# R packages
+# ----------
+
+function R_install_packages(){
+  pkgs="'$1'"
+  shift
+  for p in $@; do
+    pkgs="$pkgs, '$p'"
+  done
+  echo "install.packages(c($pkgs), repos='http://cran.us.r-project.org')" | R --vanilla --quiet
+}
+
+function R_biocLite(){
+  pkgs="'$1'"
+  shift
+  for p in $@; do
+    echo "BiocManager::install(c('$p'))" | R --vanilla --quiet
+  done
+}
+
+function R_CMD_install(){
+  url=$1
+  name=$2
+  wget "$url" -O /tmp/R-pkg-"$name".tar.gz
+  R CMD INSTALL /tmp/R-pkg-"$name".tar.gz
+}
+
+
+export R_HOME=$(R RHOME)
+export R_INCLUDE="$R_HOME/../../include/R"
+gem install rsruby -- --with-R-dir=$R_HOME --with-R-include=$R_INCLUDE
+
+echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla --quiet
+gem install rserve-client
+
+#R_install_packages 'Rcpp'
+#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.8.1.tar.gz' plyr
+
+#R_install_packages 'MASS' 'nnet'
+#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/car/car_2.0-22.tar.gz' car
+
+R_install_packages 'tidyverse'
+
+R_install_packages 'RJSONIO' 'XML' 'digest' 'gtable' 'reshape2' 'scales' 'proto'
+#R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/ggplot2/ggplot2_1.0.0.tar.gz' ggplot2
+
+R_install_packages 'BiocManager'
+R_biocLite Biobase
+
+R_install_packages NMF Cairo drc gridSVG ggthemes mclust randomForest viper
+
+R_install_packages pheatmap VennDiagram Hmisc pls gridSVG
+
+R_install_packages UpSetR
+
+R_biocLite limma viper
+
+R_install_packages pROC txtplot
+
+rm /tmp/R-pkg-*.tar.gz
+

data/share/provision_scripts/alpine_basic_setup.sh

@@ -0,0 +1,17 @@
+# Basic alpine setup
+apk add ruby ruby-dev  # Ruby
+apk add git make gcc g++ cmake # Building
+apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
+apk add bash openssh-client wget curl rsync gnu-libiconv  # Tools
+
+gem install rbbt-util rbbt-sources
+gem install RubyInline
+
+gem install specific_install hoe minitest
+gem specific_install -l https://github.com/mikisvaz/lockfile.git
+gem specific_install -l https://github.com/mikisvaz/rubyinline.git
+
+rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
+
+grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
+grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile

data/share/provision_scripts/alpine_extended_setup.sh

@@ -0,0 +1,47 @@
+# Basic alpine setup
+apk add ruby ruby-dev  # Ruby
+apk add git make gcc g++ cmake # Building
+apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
+apk add bash openssh-client wget curl rsync gnu-libiconv  # Tools
+
+gem install rbbt-util rbbt-sources
+gem install RubyInline
+
+gem install specific_install hoe minitest
+gem specific_install -l https://github.com/mikisvaz/lockfile.git
+gem specific_install -l https://github.com/mikisvaz/rubyinline.git
+
+rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
+
+grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
+grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile
+
+# Main alpine setup
+apk add xvfb bison autoconf rsync curl openssl numactl zlib-dev zlib yaml-dev openssl xz-dev xz-libs libffi tcsh gawk pandoc libtbb-dev yaml libxml2-dev libxml2 shared-mime-info 
+
+# Extended alpine setup
+apk add openjdk17 R R-dev python3 python3-dev py3-pip ansible
+
+apk add fontconfig fontconfig-dev harfbuzz fribidi harfbuzz-dev fribidi-dev jpeg jpeg-dev tiff tiff-dev cairo cairo-dev libxt-dev libxt
+
+# Allow USER to install things globaly during bootstrap
+chmod 777 -R /usr/lib/python*/site-packages
+
+gem install pycall
+
+export R_HOME=$(R RHOME)
+export R_INCLUDE="$R_HOME/../../include/R"
+cat >> /etc/rbbt_environment <<'EOF'
+export R_HOME=$(R RHOME)
+export R_INCLUDE="$R_HOME/../../include/R"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$R_HOME/lib"
+export PATH="$PATH:$R_HOME/bin"
+EOF
+gem install rsruby -- --with-R-dir=$R_HOME --with-R-include=$R_INCLUDE
+
+echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla --quiet
+gem install rserve-client
+
+export JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | cut -f 2 -d "=" | awk '{$1=$1};1')
+echo $JAVA_HOME
+gem install rjb

data/share/provision_scripts/alpine_setup.sh

@@ -0,0 +1,20 @@
+# Basic alpine setup
+apk add ruby ruby-dev  # Ruby
+apk add git make gcc g++ cmake # Building
+apk add bzip2 bzip2-dev zlib zlib-dev krb5 gcompat # Libs
+apk add bash openssh-client wget curl rsync gnu-libiconv  # Tools
+
+gem install rbbt-util rbbt-sources
+gem install RubyInline
+
+gem install specific_install hoe minitest
+gem specific_install -l https://github.com/mikisvaz/lockfile.git
+gem specific_install -l https://github.com/mikisvaz/rubyinline.git
+
+rm -Rf /usr/lib/ruby/gems/*/doc /usr/lib/ruby/gems/*/cache
+
+grep rbbt_environment /etc/rbbt_environment || echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
+grep rbbt_environment /etc/profile || echo echo "source /etc/rbbt_environment" >> /etc/profile
+
+# Main alpine setup
+apk add xvfb bison autoconf rsync curl openssl numactl zlib-dev zlib yaml-dev openssl xz-dev xz-libs libffi tcsh gawk pandoc libtbb-dev yaml libxml2-dev libxml2 shared-mime-info

data/share/provision_scripts/bootstrap.sh

@@ -3,25 +3,25 @@
 # USER RBBT BOOTSTRAP
 # ===================
 
+export RBBT_WORKFLOW_AUTOINSTALL=true
+export RBBT_LOG=0
+
 for workflow in $BOOTSTRAP_WORKFLOWS; do
     rbbt workflow install $workflow 
 
     pip_requirements_file=$(rbbt_find.rb -w $workflow requirements.pip --nocolor)
-    [ -f $pip_requirements_file ] && echo PIP $pip_requirements_file && pip install -r $pip_requirements_file
+    test -z $pip_requirements_file || pip install -r $pip_requirements_file
     unset pip_requirements_file
 
     pip_requirements_file2=$(rbbt_find.rb -w $workflow requirements.pip2 --nocolor)
-    [ -f $pip_requirements_file2 ] && pip install -r $pip_requirements_file2
+    test -z $pip_requirements_file2 || pip install -r $pip_requirements_file2
     unset pip_requirements_file2
 
     pip_requirements_file3=$(rbbt_find.rb -w $workflow requirements.pip3 --nocolor)
-    [ -f $pip_requirements_file3 ] && pip install -r $pip_requirements_file3
+    test -z $pip_requirements_file3 || pip install -r $pip_requirements_file3
     unset pip_requirements_file3
 done
 
-export RBBT_WORKFLOW_AUTOINSTALL=true
-export RBBT_LOG=0
-
 for workflow in $BOOTSTRAP_WORKFLOWS; do
     echo "Bootstrapping $workflow on $BOOTSTRAP_CPUS CPUs"
     [ -d $HOME/.rbbt/tmp/ ] || mkdir -p $HOME/.rbbt/tmp/

data/share/provision_scripts/functions.sh

@@ -0,0 +1,58 @@
+_var_content(){
+    var=$1
+    eval echo "$"$(echo $var)
+}
+
+_append_with_colon(){
+    var=$1
+    value=$2
+
+    current=`_var_content $var`
+
+    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
+        eval $(echo $var)="\"$current:$value\""
+    fi
+}
+
+_append_with_space(){
+    var=$1
+    value=$2
+
+    current=`_var_content $var`
+
+    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
+        eval $(echo $var)="\"$current:$value\""
+    fi
+}
+
+_prepend_with_colon(){
+    var=$1
+    value=$2
+
+    current=`_var_content $var`
+
+    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
+        eval $(echo $var)="\"$value:$current\""
+    fi
+}
+
+_prepend_with_space(){
+    var=$1
+    value=$2
+
+    current=`_var_content $var`
+
+    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
+        eval $(echo $var)="\"$value $current\""
+    fi
+}
+
+_add_path(){
+  _prepend_with_colon PATH "${1%/}"
+}
+
+_add_libpath(){
+  _prepend_with_colon LD_LIBRARY_PATH "${1%/}"
+  _prepend_with_colon LD_RUN_PATH "${1%/}"
+}
+

data/share/provision_scripts/gem.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -x
+
+# RUBY GEMS and RBBT
+# =================
+
+echo "* Updating RubyGem"
+export REALLY_GEM_UPDATE_SYSTEM=true
+env REALLY_GEM_UPDATE_SYSTEM=true gem update --no-document --system
+
+#echo "* Installing difficult gems: ZenTest, RubyInline, rsruby and rjb"
+#gem install --force --no-document RubyInline
+gem install --force --no-document ZenTest
+
+## R (extra config in gem)
+#export R_INCLUDE="$(echo "$R_HOME" | sed 's@/usr/lib\(32\|64\)*@/usr/share@')/include"
+#gem install --conservative --no-document rsruby -- --with-R-dir="$R_HOME" --with-R-include="$R_INCLUDE" \
+#  --with_cflags="-fPIC -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wall -fno-strict-aliasing"
+
+# Java (extra config in gem)
+#export JAVA_HOME=$(echo /usr/lib/jvm/java-?-openjdk-*)
+#gem install --conservative --force --no-document rjb
+
+echo "* Installing all rbbt gems"
+# Rbbt and some optional gems
+gem install --no-document --force \
+    tokyocabinet \
+    ruby-prof \
+    rbbt-util rbbt-sources rbbt-dm rbbt-text rbbt-rest
+
+echo "* Installing extra gems for web stuff"
+# Extra things for web interface
+gem install --no-document \
+    sinatra puma \
+    rest-client \
+    kramdown pandoc pandoc-ruby \
+    spreadsheet rubyXL \
+    prawn prawn-svg mimemagic \
+    prime mechanize
+
+gem install --no-document mini_racer uglifier
+
+gem cleanup

data/share/provision_scripts/java.sh

@@ -0,0 +1,3 @@
+export JAVA_HOME=$(java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | cut -f 2 -d "=" | awk '{$1=$1};1')
+echo $JAVA_HOME
+gem install rjb

data/share/provision_scripts/migrate.sh

@@ -0,0 +1,6 @@
+rbbt migrate ~/.rbbt/etc/ local --delete
+rbbt migrate ~/.rbbt/var/ local --delete
+rbbt migrate ~/.rbbt/share/ local --delete
+rbbt migrate ~/.rbbt/software/ local --delete
+rbbt migrate ~/.rbbt/workflows/ local --delete
+rm -Rf ~/.rbbt/tmp/

data/share/provision_scripts/python.sh

@@ -0,0 +1,4 @@
+gem install pycall
+pip install --upgrade pip
+pip install ansible
+pip install pandas numpy matplotlib

data/share/provision_scripts/{python_setup.sh → python_custom.sh}

@@ -20,6 +20,3 @@ echo '. /usr/local/miniconda3/etc/profile.d/conda.sh' >> /etc/rbbt_environment
 rm -Rf /usr/local/miniconda3/pkgs
 source /etc/rbbt_environment
 
-pip install ansible
-
-gem install pycall

data/share/provision_scripts/{tokyocabinet_setup.sh → tokyocabinet.sh}

@@ -3,9 +3,14 @@
 # TOKYOCABINET INSTALL
 # ===================
 
-cd /tmp
+mkdir -p /opt/src/
+cd /opt/src/
 wget http://fallabs.com/tokyocabinet/tokyocabinet-1.4.48.tar.gz -O "tokyocabinet.tar.gz"
 tar -xvzf tokyocabinet.tar.gz
 cd tokyocabinet-1.4.48
 ./configure --prefix=/usr/local
-make && make install
+make > /tc_make.log
+make install > /tc_make_install.log
+gem install tokyocabinet
+cd /
+rm -Rf /opt/src/

data/share/provision_scripts/ubuntu_setup.sh

@@ -3,67 +3,65 @@
 # INSTALL
 # =======
 
-cat << 'EOF' > /etc/rbbt_environment 
-_var_content(){
-    var=$1
-    eval echo "$"$(echo $var)
-}
-
-_append_with_colon(){
-    var=$1
-    value=$2
-
-    current=`_var_content $var`
-
-    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
-        eval $(echo $var)="\"$current:$value\""
-    fi
-}
-
-_append_with_space(){
-    var=$1
-    value=$2
-
-    current=`_var_content $var`
-
-    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
-        eval $(echo $var)="\"$current:$value\""
-    fi
-}
-
-_prepend_with_colon(){
-    var=$1
-    value=$2
-
-    current=`_var_content $var`
-
-    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
-        eval $(echo $var)="\"$value:$current\""
-    fi
-}
-
-_prepend_with_space(){
-    var=$1
-    value=$2
-
-    current=`_var_content $var`
-
-    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
-        eval $(echo $var)="\"$value $current\""
-    fi
-}
-
-_add_path(){
-  _prepend_with_colon PATH "${1%/}"
-}
-
-_add_libpath(){
-  _prepend_with_colon LD_LIBRARY_PATH "${1%/}"
-  _prepend_with_colon LD_RUN_PATH "${1%/}"
-}
-EOF
-
-. /etc/rbbt_environment
+#cat << 'EOF' > /etc/rbbt_environment 
+#_var_content(){
+#    var=$1
+#    eval echo "$"$(echo $var)
+#}
+#
+#_append_with_colon(){
+#    var=$1
+#    value=$2
+#
+#    current=`_var_content $var`
+#
+#    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
+#        eval $(echo $var)="\"$current:$value\""
+#    fi
+#}
+#
+#_append_with_space(){
+#    var=$1
+#    value=$2
+#
+#    current=`_var_content $var`
+#
+#    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
+#        eval $(echo $var)="\"$current:$value\""
+#    fi
+#}
+#
+#_prepend_with_colon(){
+#    var=$1
+#    value=$2
+#
+#    current=`_var_content $var`
+#
+#    if ! echo $current | grep "\(^\|:\)$value\(:\|$\)" >/dev/null 2>&1; then
+#        eval $(echo $var)="\"$value:$current\""
+#    fi
+#}
+#
+#_prepend_with_space(){
+#    var=$1
+#    value=$2
+#
+#    current=`_var_content $var`
+#
+#    if ! echo $current | grep "\(^\| \)$value\( \|$\)" >/dev/null 2>&1; then
+#        eval $(echo $var)="\"$value $current\""
+#    fi
+#}
+#
+#_add_path(){
+#  _prepend_with_colon PATH "${1%/}"
+#}
+#
+#_add_libpath(){
+#  _prepend_with_colon LD_LIBRARY_PATH "${1%/}"
+#  _prepend_with_colon LD_RUN_PATH "${1%/}"
+#}
+#EOF
 
 # Basic system requirements
 # -------------------------
@@ -115,7 +113,8 @@ echo '_add_path "/usr/local/bin/"' >> /etc/rbbt_environment
 echo 'export LC_ALL=C.UTF-8' >> /etc/rbbt_environment
 
 
-. /etc/rbbt_environment
-
 apt-get clean
 rm -rf /var/lib/apt/lists/* 
+
+echo "[ -f ~/.rbbt/etc/environment ] && . ~/.rbbt/etc/environment" >> "/etc/rbbt_environment"
+echo "source /etc/rbbt_environment" >> /etc/profile

data/share/provision_scripts/variables.sh

@@ -1,19 +1,17 @@
 #!/bin/bash -x
 
-test -z ${RBBT_SERVER+x}           && RBBT_SERVER=http://rbbt.bsc.es/ 
-test -z ${RBBT_FILE_SERVER+x}      && RBBT_FILE_SERVER="$RBBT_SERVER"
-test -z ${RBBT_WORKFLOW_SERVER+x}  && RBBT_WORKFLOW_SERVER="$RBBT_SERVER"
+test -z $RBBT_SERVER               && export RBBT_SERVER=http://rbbt.bsc.es/
+test -z $RBBT_FILE_SERVER          && export RBBT_FILE_SERVER="$RBBT_SERVER"
+test -z $RBBT_WORKFLOW_SERVER      && export RBBT_WORKFLOW_SERVER="$RBBT_SERVER"
 
-test -z ${REMOTE_RESOURCES+x}  && REMOTE_RESOURCES="Organism ICGC COSMIC KEGG InterPro"
-test -z ${REMOTE_WORFLOWS+x}   && REMOTE_WORFLOWS=""
+test -z $REMOTE_RESOURCES          && export REMOTE_RESOURCES="Organism ICGC COSMIC KEGG InterPro Signor"
+test -z $REMOTE_WORFLOWS           && export REMOTE_WORFLOWS=""
 
-test -z ${RBBT_WORKFLOW_AUTOINSTALL+x}  && RBBT_WORKFLOW_AUTOINSTALL="true"
+test -z $RBBT_WORKFLOW_AUTOINSTALL && export RBBT_WORKFLOW_AUTOINSTALL="true"
 
-test -z ${WORKFLOWS+x}  && WORKFLOWS=""
+test -z $WORKFLOWS                 && export WORKFLOWS=""
 
-test -z ${BOOTSTRAP_WORKFLOWS+x}  && BOOTSTRAP_WORKFLOWS=""
-test -z ${BOOTSTRAP_CPUS+x}       && BOOTSTRAP_CPUS="2"
+test -z $BOOTSTRAP_WORKFLOWS       && export BOOTSTRAP_WORKFLOWS=""
+test -z $BOOTSTRAP_CPUS            && export BOOTSTRAP_CPUS="2"
 
-test -z ${RBBT_LOG+x}  && RBBT_LOG="LOW"
-
-test -z ${RUBY_VERSION+x}  && RUBY_VERSION="2.4.1"
+test -z $RBBT_LOG                  && export RBBT_LOG="LOW"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-image
 version: !ruby/object:Gem::Version
-  version: 0.1.39
+  version: 0.1.40
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-31 00:00:00.000000000 Z
+date: 2023-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -40,18 +40,25 @@ files:
 - bin/run_rbbt_docker.rb
 - lib/rbbt/docker.rb
 - share/provision_scripts/Dockerfile.R-packages
-- share/provision_scripts/R_packages.sh
-- share/provision_scripts/R_setup.sh
+- share/provision_scripts/R.sh
+- share/provision_scripts/R_custom.sh
+- share/provision_scripts/alpine_basic_setup.sh
+- share/provision_scripts/alpine_extended_setup.sh
+- share/provision_scripts/alpine_setup.sh
 - share/provision_scripts/bootstrap.sh
-- share/provision_scripts/gem_setup.sh
+- share/provision_scripts/functions.sh
+- share/provision_scripts/gem.sh
 - share/provision_scripts/hacks.sh
-- share/provision_scripts/perl_setup.sh
-- share/provision_scripts/python_setup.sh
-- share/provision_scripts/ruby_setup.sh
+- share/provision_scripts/java.sh
+- share/provision_scripts/migrate.sh
+- share/provision_scripts/perl_custom.sh
+- share/provision_scripts/python.sh
+- share/provision_scripts/python_custom.sh
+- share/provision_scripts/ruby_custom.sh
 - share/provision_scripts/slurm_loopback.sh
-- share/provision_scripts/tokyocabinet_setup.sh
+- share/provision_scripts/tokyocabinet.sh
 - share/provision_scripts/ubuntu_setup.sh
-- share/provision_scripts/user_setup.sh
+- share/provision_scripts/user.sh
 - share/provision_scripts/variables.sh
 homepage: http://github.com/mikisvaz/rbbt-image
 licenses:
@@ -72,7 +79,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.4.8
 signing_key: 
 specification_version: 4
 summary: Build docker and Vagrant (VM) images

data/share/provision_scripts/R_packages.sh

@@ -1,53 +0,0 @@
-#!/bin/bash -x
-
-# INSTALL
-# =======
-
-# R packages
-# ----------
-
-function R_install_packages(){
-  pkgs="'$1'"
-  shift
-  for p in $@; do
-    pkgs="$pkgs, '$p'"
-  done
-  echo "install.packages(c($pkgs), repos='http://cran.us.r-project.org')" | R --vanilla
-}
-
-function R_biocLite(){
-  pkgs="'$1'"
-  shift
-  for p in $@; do
-    echo "source('http://bioconductor.org/biocLite.R'); biocLite('$p')" | R --vanilla
-  done
-}
-
-function R_CMD_install(){
-  url=$1
-  name=$2
-  wget "$url" -O /tmp/R-pkg-"$name".tar.gz
-  R CMD INSTALL /tmp/R-pkg-"$name".tar.gz
-}
-
-R_install_packages Rcpp RJSONIO XML
-R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.8.1.tar.gz' plyr
-R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/car/car_2.0-22.tar.gz' car
-R_install_packages 'digest' 'gtable' 'reshape2' 'scales' 'proto'
-
-R_CMD_install 'https://cran.r-project.org/src/contrib/Archive/ggplot2/ggplot2_1.0.0.tar.gz' ggplot2
-
-R_install_packages NMF Cairo drc gridSVG ggthemes mclust randomForest viper
-
-echo 'install.packages("Rserve", repos="http://www.rforge.net/")' | R --vanilla
-
-R_install_packages pheatmap VennDiagram Hmisc pls gridSVG
-R_install_packages tidyverse
-
-R_install_packages UpSetR
-
-R_biocLite limma viper
-
-R_install_packages pROC txtplot
-
-rm /tmp/R-pkg-*.tar.gz

data/share/provision_scripts/gem_setup.sh

@@ -1,46 +0,0 @@
-#!/bin/bash -x
-
-# RUBY GEMS and RBBT
-# =================
-
-. /etc/rbbt_environment
-
-echo "* Updating RubyGem"
-export REALLY_GEM_UPDATE_SYSTEM=true
-env REALLY_GEM_UPDATE_SYSTEM=true gem update --no-document --system
-
-echo "* Installing difficult gems: ZenTest, RubyInline, rsruby and rjb"
-gem install --force --no-document ZenTest
-gem install --force --no-document RubyInline
-
-# R (extra config in gem)
-export R_INCLUDE="$(echo "$R_HOME" | sed 's@/usr/lib\(32\|64\)*@/usr/share@')/include"
-gem install --conservative --no-document rsruby -- --with-R-dir="$R_HOME" --with-R-include="$R_INCLUDE" \
-  --with_cflags="-fPIC -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Wall -fno-strict-aliasing"
-
-# Java (extra config in gem)
-export JAVA_HOME=$(echo /usr/lib/jvm/java-?-openjdk-*)
-gem install --conservative --force --no-document rjb
-
-echo "* Installing bulk gems and Rbbt"
-# Rbbt and some optional gems
-gem install --no-document --force \
-    tokyocabinet \
-    ruby-prof \
-    rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources \
-    rserve-client \
-
-# Get good version of lockfile
-#wget http://ubio.bioinfo.cnio.es/people/mvazquezg/lockfile-2.1.4.gem -O /tmp/lockfile-2.1.4.gem
-echo "* Installing our version of lockfile"
-wget https://github.com/mikisvaz/lockfile/raw/master/lockfile-2.1.6.gem -O /tmp/lockfile-2.1.6.gem
-gem install --no-document /tmp/lockfile-2.1.6.gem
-
-# Extra things for web interface
-gem install --no-document --force \
-    tokyocabinet \
-    ruby-prof \
-    rbbt-util rbbt-rest rbbt-dm rbbt-text rbbt-sources rbbt-phgx rbbt-GE \
-    rserve-client \
-    uglifier therubyracer kramdown pandoc pandoc-ruby \
-    puma prawn prawn-svg mimemagic