rbbt-util 5.25.7 → 5.25.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b343e64d8efd008218b6566a0ab462490d4edc56
-  data.tar.gz: 173df6bdb2479c4febe612d2ad7b4b71ab078008
+  metadata.gz: 3a8edeae60507177567648ab41bb5566d8707a31
+  data.tar.gz: 9e5188a78754f01da5c96eee01e0448bebb34a1c
 SHA512:
-  metadata.gz: c4af721ae27c0ba29988b8abc7ca02158e8aeecd5a21a77ea741794bea7a2ffee33be92a8f0349c902cbfed086cdced582d0ab94e08b203794605b9a5438a69f
-  data.tar.gz: d2dd2aa67c59d937811d0e8f0e7a4cd72392b022a3cf3499a5d6b36caa660d03cd80fee029c6dfca5824eace43d556c5df6c418c874ae755823e0d1a4130d92f
+  metadata.gz: 593de7cb23b2afd5562bc72cf44efc6d3a9138da692009aa8d15e54996ea31f3dfb9ac75cf0412b0e861c8014be62ff0f2b8ffd6ce74b28c1caa46e0dff53d83
+  data.tar.gz: dd4f867fa4e20a6a8eaf8e53d8cdd8a782f5181e2fb5863004c028f1a8ce8bcf82c2d67b5469e5f8d9c3a437417af4a053a05c80db85f55b0db61a44a58dbd84

data/lib/rbbt/hpc.rb

@@ -4,119 +4,361 @@ require 'rbbt/util/cmd'
 module Marenostrum
   SERVER='mn1'
   module SLURM
-    TEMPLATE=<<-EOF
 
+    def self.template(args, options = {})
 
-    EOF
-    def self.issue(cmd, name = nil, time = nil, options = {})
-      name = "rbbt-job-" << rand(10000).to_s if name.nil?
-      time = "00:00:10" if time.nil?
-      workdir = "rbbt-workdir/" << name
+      dry_run          = options.delete :dry_run
+      development      = options.delete :drbbt
+      contain          = options.delete :contain
+      sync             = options.delete :sync
+      contain_and_sync = options.delete :contain_and_sync
+      wipe_container   = options.delete :wipe_container
+      copy_image       = options.delete :copy_image
+      exclusive        = options.delete :exclusive
+      highmem          = options.delete :highmem
+
+      if contain_and_sync
+        contain = "/scratch/tmp/rbbt" if contain.nil?
+        sync = "~/.rbbt/var/jobs" if sync.nil?
+      end
+
+      contain = File.expand_path(contain) if contain
+
+      singularity = true if contain || ! development
+
+
+      name = options[:name] ||= Misc.obj2digest({:options => options.collect{|k,v| [k,v]}.sort_by{|k,v| k.to_s }, :args => args})
+      workdir = options[:workdir] ||= File.expand_path(File.join('~/rbbt-workdir', name)) if workdir.nil?
+
+      rbbt_cmd = args.reject{|e| e == '--' }.collect{|e| e.include?(" ")? '"' + e + '"' : e } * " "
+
+      queue = options[:queue] || 'bsc_ls'
+      tasks = options[:tasks] || 1
+      time = options[:time] || "0:00:10"
+
+      time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
 
-      new_options = Misc.add_defaults options, "job-name" => name, "workdir" => workdir,
-        "output" => "log", 
-        "error" => "log.err", 
-        "ntasks" => "1", 
-        "time" => time.to_s,
-        "user" => ENV["USER"],
-        "key_file" => File.join(ENV['HOME'], '.ssh/id_rsa.pub')
 
-      options.merge!(new_options)
-      IndiferentHash.setup(options)
+      #{{{ PREPARE LOCAL LOGFILES
 
-      key_file, user = options.values_at :key_file, :user
+      Open.mkdir workdir
 
-      template = <<-EOF
+      fout = File.join(workdir, 'std.out')
+      ferr = File.join(workdir, 'std.err')
+      fjob = File.join(workdir, 'job.id')
+      fexit = File.join(workdir, 'exit.status')
+      fsync = File.join(workdir, 'sync.log')
+      fcmd = File.join(workdir, 'command.slurm')
+
+      #{{{ GENERATE TEMPLATE
+
+      # HEADER
+      header =<<-EOF
 #!/bin/bash
-#{options.collect do |name,value|
-      next if name == 'user'
-      next if name == 'key_file'
-name = "--" << name.to_s.sub(/^--/,'')
-[name, '"' << value << '"'] * "="
-  end.compact.collect{|str| "#SBATCH #{str}"} * "\n"
-}
-#{ cmd }
+#SBATCH --qos="#{queue}"
+#SBATCH --job-name="#{name}"
+#SBATCH --workdir="#{Dir.pwd}"
+#SBATCH --output="#{fout}"
+#SBATCH --error="#{ferr}"
+#SBATCH --ntasks="#{tasks}"
+#SBATCH --time="#{time}"
       EOF
 
-      TmpFile.with_file do |slurm|
-        res = nil
-        begin
-          Open.write(slurm, template)
-          Log.medium("Issuing job:\n" + template)
-
-          cmd = "ssh -i #{key_file} #{ user }@#{ SERVER } mkdir -p '#{ workdir }'; scp -i #{key_file} #{ slurm } #{ user }@#{ SERVER }:'#{ workdir }/#{ name }.slurm'; ssh -i #{key_file} #{ user }@#{ SERVER } sbatch #{ workdir }/#{ name }.slurm"
-          Log.debug cmd
-          res = CMD.cmd(cmd).read
-        rescue
-          raise "Could not issue job"
+      if highmem
+        header +=<<-EOF
+#SBATCH --constraint=highmem 
+        EOF
+      end
+
+      if exclusive
+        header +=<<-EOF
+#SBATCH --exclusive 
+        EOF
+      end
+
+      header +=<<-EOF
+#CMD: #{rbbt_cmd}
+      EOF
+
+      # ENV
+      env = ""
+      env +=<<-EOF
+# Prepare env
+[[ -f ~/config/load.sh ]] && source ~/config/load.sh
+module load java
+      EOF
+
+      if singularity
+        env +=<<-EOF
+module load intel/2018.1
+module load singularity
+module load samtools
+SINGULARITY_IMG="$HOME/projects/rbbt.singularity.img"
+SINGULARITY_RUBY_INLINE="$HOME/.singularity_ruby_inline"
+mkdir -p "$SINGULARITY_RUBY_INLINE"
+        EOF
+      end
+
+      if contain
+        env +=<<-EOF
+CONTAINER_DIR="#{contain}"
+mkdir -p $CONTAINER_DIR/.rbbt/etc/
+for tmpd in persist_locks  produce_locks  R_sockets  sensiblewrite  sensiblewrite_locks  step_info_locks  tsv_open_locks; do
+    mkdir -p $CONTAINER_DIR/.rbbt/tmp/$tmpd
+done
+cp ~/.rbbt/etc/environment $CONTAINER_DIR/.rbbt/etc/
+echo "rbbt_user: /home/rbbt/.rbbt/{TOPLEVEL}/{SUBPATH}" > $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "home: $CONTAINER_DIR/home/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "group_projects: $CONTAINER_DIR/projects/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "group_scratch: $CONTAINER_DIR/scratch/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "user_projects: $CONTAINER_DIR/projects/#{ENV['USER']}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+echo "user_scratch: $CONTAINER_DIR/scratch/#{ENV['USER']}/{PKGDIR}/{TOPLEVEL}/{SUBPATH}" >> $CONTAINER_DIR/.rbbt/etc/search_paths
+        EOF
+
+        if copy_image
+          env +=<<EOF
+rsync -avz "$SINGULARITY_IMG" "$CONTAINER_DIR/rbbt.singularity.img"
+SINGULARITY_IMG="$CONTAINER_DIR/rbbt.singularity.img"
+EOF
         end
 
-        res.scan(/\d+/).first
+        if  wipe_container == "pre" || wipe_container == "both"
+          env +=<<-EOF
+singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/jobs &>> #{fsync}
+singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
+EOF
+        end
       end
-    end
 
-    def self.query(id, options = {})
-      key_file, user = options.values_at :key_file, :user
+      # RUN
+      run = ""
+
+      if singularity
+        if contain
+          group = File.basename(File.dirname(ENV['HOME']))
+          scratch_group_dir = File.join('/gpfs/scratch/', group)
+          projects_group_dir = File.join('/gpfs/projects/', group)
+          exec_cmd = %(singularity exec -e -C -H "$CONTAINER_DIR" -B "$SINGULARITY_RUBY_INLINE":"$CONTAINER_DIR/.ruby_inline":rw  -B ~/git:"$CONTAINER_DIR/git":ro -B #{scratch_group_dir}:"$CONTAINER_DIR/scratch":ro -B ~/.rbbt/software/opt/:"/opt/":ro  -B ~/.rbbt:"$CONTAINER_DIR/home/":ro -B #{projects_group_dir}:"$CONTAINER_DIR/projects":ro "$SINGULARITY_IMG" env TMPDIR="$CONTAINER_DIR/.rbbt/tmp" rbbt)
+        else
+          exec_cmd = %(singularity exec -e -B "$SINGULARITY_RUBY_INLINE":"$HOME/.ruby_inline":rw "$SINGULARITY_IMG" rbbt)
+        end
+
+        if development
+          exec_cmd += ' --dev=git'
+        end
+      else
+        exec_cmd = %(~/git/rbbt-util/bin/rbbt --dev=~/git/)
+      end
+
+
+      cmd =<<-EOF
+#{exec_cmd} \\
+#{rbbt_cmd}
+EOF
 
-      res = nil
-      begin
-        cmd = "ssh -i #{key_file} #{ user }@#{ SERVER } squeue |grep '#{id}\\|JOBID'"
-        Log.debug cmd
-        res = CMD.cmd(cmd).read
-      rescue
-        raise "Could not query job: #{ id }" << $!.message
+      run +=<<-EOF
+
+# Run command
+#{cmd}
+
+# Save exit status
+echo $? > #{fexit}
+
+# Clean job.id, since we are done
+rm #{fjob}
+EOF
+
+      # CODA
+      coda = ""
+      if sync
+        if singularity
+          coda +=<<-EOF
+singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean all -q &>> #{fsync}
+EOF
+        end
+
+        coda +=<<-EOF
+rsync -avt "#{File.join(File.expand_path(contain), '.rbbt/var/jobs')}/" "#{File.expand_path(sync)}/" &>> #{fsync} 
+EOF
+
+        if  contain && (wipe_container == "post" || wipe_container == "both")
+          coda +=<<-EOF
+sync_es="$?" 
+singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
+singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -v /dev/shm/sem.*.{in,out,process} /dev/shm/sem.Session-PID.*.sem 2> /dev/null >> #{fsync}
+if [ $sync_es == '0' ]; then 
+    singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/jobs &>> #{fsync}
+else
+    echo "WARNING: Results could not sync correctly. Job directory not purged"
+fi
+unset sync_es
+EOF
+        end
       end
 
-      res
+      template = [header, env, run, coda] * "\n"
+
+      template
     end
+    
+    def self.issue_template(template, options = {})
 
-    def self.done?(id, options = {})
-      ! query(id, options).include? id
+      workdir = options[:workdir]
+      Open.mkdir workdir
+      fout = File.join(workdir, 'std.out')
+      ferr = File.join(workdir, 'std.err')
+      fjob = File.join(workdir, 'job.id')
+      fexit = File.join(workdir, 'exit.status')
+      fsync = File.join(workdir, 'sync.log')
+      fcmd = File.join(workdir, 'command.slurm')
+
+      job = nil
+      if options[:clean_job]
+        [fcmd, fjob, fout, ferr, fsync, fexit].each do |file|
+          Open.rm file if Open.exists? file
+        end
+      end
+
+      return if Open.exists?(fexit)
+
+      STDERR.puts Log.color(:magenta, "Issuing SLURM file: #{fcmd}")
+      STDERR.puts template
+
+      Open.write(fcmd, template) unless File.exists? fcmd
+      if File.exists?(fjob)
+        job = Open.read(fjob).to_i
+      else
+        if File.exists?(fout)
+          return
+        else
+          Open.rm fsync
+          Open.rm fexit
+          Open.rm fout
+          Open.rm ferr
+          job = CMD.cmd("sbatch '#{fcmd}'").read.scan(/\d+/).first.to_i
+          Open.write(fjob, job.to_s)
+        end
+      end
     end
 
-    def self.gather(id, file = nil, options = {})
-      key_file, user, workdir, output = options.values_at :key_file, :user, :workdir, :output
+    def self.follow_job(workdir, tail = true)
+      fjob = File.join(workdir, 'job.id')
+      fout = File.join(workdir, 'std.out')
+      ferr = File.join(workdir, 'std.err')
+      fstatus = File.join(workdir, 'job.status')
+
+      job = Open.read(fjob) if Open.exists?(fjob)
 
-      TmpFile.with_file do |result|
+      if job
+        status_txt = CMD.cmd("squeue --job #{job}").read
+        STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
+        STDERR.puts status_txt
+        lines = status_txt.split("\n").length
+      end
 
+      if tail
+        while ! File.exists? fout
+          if job
+            STDERR.puts
+            Log.clear_line(STDERR)
+            STDERR.write Log.color(:magenta, "Waiting for Output")
+            3.times do
+              STDERR.write Log.color(:magenta, ".")
+              sleep 1
+            end
+            status_txt = CMD.cmd("squeue --job #{job}").read
+            lines.times do
+              Log.clear_line(STDERR)
+            end
+            Log.clear_line(STDERR)
+            STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
+            STDERR.puts status_txt
+            lines = status_txt.split("\n").length
+          end
+        end
+        STDERR.puts
+        Log.clear_line(STDERR)
+        STDERR.puts Log.color(:magenta, "Output:")
         begin
-          cmd = "scp -i #{key_file} #{ user }@#{ SERVER }:'#{workdir}/#{output}' '#{ result }' " 
-          Log.debug cmd
-          CMD.cmd(cmd)
-          if file.nil?
-            Open.read(result)
-          else
-            Misc.sensiblewrite(result, file)
+          CMD.cmd("squeue --job #{job} > #{fstatus}")
+          out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists? fout
+          err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists? ferr
+
+          Misc.consume_stream(err, true, STDERR) if err
+          Misc.consume_stream(out, true, STDOUT) if out
+
+          sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
+        ensure
+          begin
+            err.close if err
+            err.join if err
+          rescue Exception
+          end
+
+          begin
+            out.close if out
+            out.join if out
+          rescue Exception
           end
-        rescue
-          raise "Could not gather job: #{ id }: " << $!.message
         end
       end
     end
 
-    def self.run(cmd, name = nil, time = nil, options = {})
-      sleep_time = Misc.process_options options, :sleep
+    def self.wait_for_job(workdir, time = 1)
+      fexit = File.join(workdir, 'exit.status')
+      fjob = File.join(workdir, 'job.id')
+      job = Open.read(fjob) if Open.exists?(fjob)
 
-      id = issue(cmd, name, time, options)
+      status_txt = CMD.cmd("squeue --job #{job}").read
+      STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
+      STDERR.puts status_txt
+      lines = status_txt.split("\n").length
 
-      if sleep_time.nil?
-        times = [1,2,3,5,10,30]
-      else
-        times = Array ===  sleep_time ? sleep_time.dup : [sleep_time]
+      while ! Open.exists?(fexit)
+        STDERR.puts
+        Log.clear_line(STDERR)
+        STDERR.write Log.color(:magenta, "Waiting for end")
+        3.times do
+          STDERR.write Log.color(:magenta, ".")
+          sleep 1
+        end
+        status_txt = CMD.cmd("squeue --job #{job}").read
+        lines.times do
+          Log.clear_line(STDERR)
+        end
+        Log.clear_line(STDERR)
+        STDERR.puts Log.color(:magenta, "Status [#{job.to_i}]:")
+        STDERR.puts status_txt
+        lines = status_txt.split("\n").length
+        sleep time
       end
+    end
 
-      while not done?(id, options)
-        Log.debug "Waiting on #{ id }"
-        sleep_time = times.shift || sleep_time
-        sleep sleep_time
+    def self.run_job(job, options = {})
+      workflow = job.workflow
+      task = job.task_name
+      name = job.clean_name
+      TmpFile.with_file(nil, false) do |tmp_directory|
+        workdir = options[:workdir] ||= File.join(tmp_directory, 'workdir')
+        inputs_dir = File.join(tmp_directory, 'inputs_dir')
+        Step.save_job_inputs(job, inputs_dir)
+        cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '-jn', name, '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
+        template = self.template(cmd, options)
+        self.issue_template(template, options)
+        self.wait_for_job(workdir)
+        path = Open.read(File.join(workdir, 'std.out'))
+        if job.path != path
+          Log.medium "Path of SLURM job #{path} is different from original job #{job.path}. Stablishing link."
+          Open.ln path, job.path
+        end
       end
-
-      gather(id, nil, options)
     end
   end
 end
 
-Log.severity = 0 if __FILE__ == $0
-iii Marenostrum::SLURM.run('ls', nil, nil, :qos => "debug", :user => 'bsc26892') if __FILE__ == $0
+if __FILE__ == $0
+  Log.severity = 0
+  iii Marenostrum::SLURM.run('ls', nil, nil, :qos => "debug", :user => 'bsc26892') if __FILE__ == $0
+end
 
 

data/lib/rbbt/util/misc.rb

@@ -71,13 +71,15 @@ module Misc
   def self.timespan(str, default = "s")
     tokens = {
       "s" => (1),
+      "sec" => (1),
+      "m" => (60),
       "min" => (60),
       "''" => (1),
       "'" => (60),
       "h" => (60 * 60),
       "d" => (60 * 60 * 24),
       "w" => (60 * 60 * 24 * 7),
-      "m" => (60 * 60 * 24 * 30),
+      "mo" => (60 * 60 * 24 * 30),
       "y" => (60 * 60 * 24 * 365),
     }
 

data/lib/rbbt/workflow/accessor.rb

@@ -80,6 +80,29 @@ class Step
     end
   end
 
+  def self.save_job_inputs(job, dir)
+
+    task_name = job.task_name
+    task_info = job.workflow.task_info(task_name)
+    input_types = task_info[:input_types]
+    task_inputs = task_info[:inputs]
+    job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
+      next unless task_inputs.include? name.to_sym
+      next if value.nil?
+      path = File.join(dir, name.to_s)
+      type = input_types[name]
+      Log.debug "Saving job input #{name} (#{type}) into #{path}"
+      case
+      when Array === value
+        Open.write(path, value * "\n")
+      when IO === value
+        Open.write(path, value)
+      else
+        Open.write(path, value.to_s)
+      end
+    end
+  end
+
   def name
     @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
   end
@@ -108,7 +131,7 @@ class Step
                    #Lockfile.new path, :refresh => false, :dont_use_lock_id => true
                    Lockfile.new path
                  end if @info_lock.nil?
-    @info_lock
+                 @info_lock
   end
 
   def status_lock

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.25.7
+  version: 5.25.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-18 00:00:00.000000000 Z
+date: 2018-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake