rbbt-util 5.28.0 → 5.28.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/lib/rbbt/hpc.rb +5 -2
 - data/lib/rbbt/resource/path.rb +4 -2
 - data/lib/rbbt/tsv/change_id.rb +2 -2
 - data/lib/rbbt/tsv/parallel/traverse.rb +7 -8
 - data/lib/rbbt/tsv/util.rb +20 -12
 - data/lib/rbbt/util/misc/inspect.rb +3 -3
 - data/lib/rbbt/workflow.rb +1 -0
 - data/lib/rbbt/workflow/accessor.rb +94 -93
 - data/lib/rbbt/workflow/integration/cromwell.rb +10 -3
 - data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +5 -1
 - data/lib/rbbt/workflow/remote_workflow/remote_step.rb +5 -0
 - data/lib/rbbt/workflow/step/accessor.rb +10 -7
 - data/lib/rbbt/workflow/step/dependencies.rb +2 -2
 - data/lib/rbbt/workflow/usage.rb +1 -1
 - data/share/rbbt_commands/app/start +2 -2
 - data/share/rbbt_commands/workflow/server +2 -0
 - data/test/rbbt/tsv/parallel/test_traverse.rb +21 -1
 - metadata +2 -2
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA256:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 900106fb1799c857a482187c7cb7d18a40c35dbeb5dd231bf8e1851235e28c63
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: e5ec2271b3ada0b282963f4ffc7e2b623d8bb84fcb72867a6848b93a80052463
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 1a418d5b4fe4369f7c25dd063c60a65c2b6c0bd7e3dbf9fd156fe875f85ef1b2eeb4d2eb1a03472e191de98bbfbae20abdb86877d381193f86fe4c8325d2f391
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 5225d6dbdbab08d0ed5239eba91bbe3d2a8dd16c92c24fe177b66ed3b9d4baef4e24818057de8bf694edf715aabd72b481926a2466c76492c27b75be25634b27
         
     | 
    
        data/lib/rbbt/hpc.rb
    CHANGED
    
    | 
         @@ -451,12 +451,15 @@ EOF 
     | 
|
| 
       451 
451 
     | 
    
         
             
                      out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
         
     | 
| 
       452 
452 
     | 
    
         
             
                      err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
         
     | 
| 
       453 
453 
     | 
    
         | 
| 
       454 
     | 
    
         
            -
                      Misc.consume_stream(err, true, STDERR) if err
         
     | 
| 
       455 
     | 
    
         
            -
                      Misc.consume_stream(out, true, STDOUT) if out
         
     | 
| 
      
 454 
     | 
    
         
            +
                      terr = Misc.consume_stream(err, true, STDERR) if err
         
     | 
| 
      
 455 
     | 
    
         
            +
                      tout = Misc.consume_stream(out, true, STDOUT) if out
         
     | 
| 
       456 
456 
     | 
    
         | 
| 
       457 
457 
     | 
    
         
             
                      sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
         
     | 
| 
      
 458 
     | 
    
         
            +
                    rescue Aborted
         
     | 
| 
       458 
459 
     | 
    
         
             
                    ensure
         
     | 
| 
       459 
460 
     | 
    
         
             
                      begin
         
     | 
| 
      
 461 
     | 
    
         
            +
                        terr.exit if terr
         
     | 
| 
      
 462 
     | 
    
         
            +
                        tout.exit if tout
         
     | 
| 
       460 
463 
     | 
    
         
             
                        err.close if err
         
     | 
| 
       461 
464 
     | 
    
         
             
                        err.join if err
         
     | 
| 
       462 
465 
     | 
    
         
             
                      rescue Exception
         
     | 
    
        data/lib/rbbt/resource/path.rb
    CHANGED
    
    | 
         @@ -76,7 +76,7 @@ module Path 
     | 
|
| 
       76 
76 
     | 
    
         
             
                  return [] unless self.exists? 
         
     | 
| 
       77 
77 
     | 
    
         
             
                  found = self.find
         
     | 
| 
       78 
78 
     | 
    
         
             
                  exp = File.join(found, pattern)
         
     | 
| 
       79 
     | 
    
         
            -
                  paths = Dir.glob(exp).collect{|f|  
     | 
| 
      
 79 
     | 
    
         
            +
                  paths = Dir.glob(exp).collect{|f| self.annotate(f) }
         
     | 
| 
       80 
80 
     | 
    
         | 
| 
       81 
81 
     | 
    
         
             
                  paths.each do |p|
         
     | 
| 
       82 
82 
     | 
    
         
             
                    p.original = File.join(found.original, p.sub(/^#{found}/, ''))
         
     | 
| 
         @@ -412,7 +412,9 @@ module Path 
     | 
|
| 
       412 
412 
     | 
    
         
             
              end
         
     | 
| 
       413 
413 
     | 
    
         | 
| 
       414 
414 
     | 
    
         
             
              def replace_extension(new_extension = nil, multiple = false)
         
     | 
| 
       415 
     | 
    
         
            -
                if multiple
         
     | 
| 
      
 415 
     | 
    
         
            +
                if String === multiple
         
     | 
| 
      
 416 
     | 
    
         
            +
                  new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
         
     | 
| 
      
 417 
     | 
    
         
            +
                elsif multiple
         
     | 
| 
       416 
418 
     | 
    
         
             
                  new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
         
     | 
| 
       417 
419 
     | 
    
         
             
                else
         
     | 
| 
       418 
420 
     | 
    
         
             
                  new_path = self.sub(/\.[^\.\/]{1,5}$/,'')
         
     | 
    
        data/lib/rbbt/tsv/change_id.rb
    CHANGED
    
    | 
         @@ -58,13 +58,13 @@ module TSV 
     | 
|
| 
       58 
58 
     | 
    
         | 
| 
       59 
59 
     | 
    
         
             
                identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
         
     | 
| 
       60 
60 
     | 
    
         
             
                identifiers = tsv.identifier_files.first if identifiers.nil?
         
     | 
| 
       61 
     | 
    
         
            -
                identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil?  
     | 
| 
      
 61 
     | 
    
         
            +
                identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
         
     | 
| 
       62 
62 
     | 
    
         
             
                identifiers.namespace ||= tsv.namespace
         
     | 
| 
       63 
63 
     | 
    
         | 
| 
       64 
64 
     | 
    
         
             
                fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil 
         
     | 
| 
       65 
65 
     | 
    
         
             
                #index = identifiers.index :target => format, :fields => fields, :persist => persist_input, :order => true
         
     | 
| 
       66 
66 
     | 
    
         | 
| 
       67 
     | 
    
         
            -
                grep = Organism.blacklist_genes(tsv.namespace).list  if identifiers.namespace  
     | 
| 
      
 67 
     | 
    
         
            +
                grep = Organism.blacklist_genes(tsv.namespace).list  if defined?(Organism) && identifiers.namespace && Organism.blacklist_genes(tsv.namespace).exists?
         
     | 
| 
       68 
68 
     | 
    
         
             
                if fields.nil?
         
     | 
| 
       69 
69 
     | 
    
         
             
                  index = identifiers.index(:data_tsv_grep => grep, :data_invert_grep => true, :target => format, :persist => true, :order => true, :unnamed => true, :data_persist => true)
         
     | 
| 
       70 
70 
     | 
    
         
             
                else
         
     | 
| 
         @@ -101,8 +101,8 @@ module TSV 
     | 
|
| 
       101 
101 
     | 
    
         
             
                    error = true
         
     | 
| 
       102 
102 
     | 
    
         
             
                    raise $!
         
     | 
| 
       103 
103 
     | 
    
         
             
                  ensure
         
     | 
| 
       104 
     | 
    
         
            -
                    Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       105 
104 
     | 
    
         
             
                    join.call(error) if join
         
     | 
| 
      
 105 
     | 
    
         
            +
                    Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       106 
106 
     | 
    
         
             
                  end
         
     | 
| 
       107 
107 
     | 
    
         
             
              end
         
     | 
| 
       108 
108 
     | 
    
         | 
| 
         @@ -137,8 +137,8 @@ module TSV 
     | 
|
| 
       137 
137 
     | 
    
         
             
                  error = true
         
     | 
| 
       138 
138 
     | 
    
         
             
                  raise $!
         
     | 
| 
       139 
139 
     | 
    
         
             
                ensure
         
     | 
| 
       140 
     | 
    
         
            -
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       141 
140 
     | 
    
         
             
                  join.call(error) if join
         
     | 
| 
      
 141 
     | 
    
         
            +
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       142 
142 
     | 
    
         
             
                end
         
     | 
| 
       143 
143 
     | 
    
         
             
              end
         
     | 
| 
       144 
144 
     | 
    
         | 
| 
         @@ -177,8 +177,8 @@ module TSV 
     | 
|
| 
       177 
177 
     | 
    
         
             
                  error = true
         
     | 
| 
       178 
178 
     | 
    
         
             
                  raise $!
         
     | 
| 
       179 
179 
     | 
    
         
             
                ensure
         
     | 
| 
       180 
     | 
    
         
            -
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       181 
180 
     | 
    
         
             
                  join.call(error) if join
         
     | 
| 
      
 181 
     | 
    
         
            +
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       182 
182 
     | 
    
         
             
                end
         
     | 
| 
       183 
183 
     | 
    
         
             
              end
         
     | 
| 
       184 
184 
     | 
    
         | 
| 
         @@ -219,8 +219,8 @@ module TSV 
     | 
|
| 
       219 
219 
     | 
    
         
             
                  error = true
         
     | 
| 
       220 
220 
     | 
    
         
             
                  raise $!
         
     | 
| 
       221 
221 
     | 
    
         
             
                ensure
         
     | 
| 
       222 
     | 
    
         
            -
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       223 
222 
     | 
    
         
             
                  join.call(error) if join
         
     | 
| 
      
 223 
     | 
    
         
            +
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       224 
224 
     | 
    
         
             
                end
         
     | 
| 
       225 
225 
     | 
    
         
             
              end
         
     | 
| 
       226 
226 
     | 
    
         | 
| 
         @@ -273,8 +273,8 @@ module TSV 
     | 
|
| 
       273 
273 
     | 
    
         
             
                  error = true
         
     | 
| 
       274 
274 
     | 
    
         
             
                  raise $!
         
     | 
| 
       275 
275 
     | 
    
         
             
                ensure
         
     | 
| 
       276 
     | 
    
         
            -
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       277 
276 
     | 
    
         
             
                  join.call(error) if join
         
     | 
| 
      
 277 
     | 
    
         
            +
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       278 
278 
     | 
    
         
             
                end
         
     | 
| 
       279 
279 
     | 
    
         
             
              end
         
     | 
| 
       280 
280 
     | 
    
         | 
| 
         @@ -318,8 +318,8 @@ module TSV 
     | 
|
| 
       318 
318 
     | 
    
         
             
                  error = true
         
     | 
| 
       319 
319 
     | 
    
         
             
                  raise $!
         
     | 
| 
       320 
320 
     | 
    
         
             
                ensure
         
     | 
| 
       321 
     | 
    
         
            -
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       322 
321 
     | 
    
         
             
                  join.call(error) if join
         
     | 
| 
      
 322 
     | 
    
         
            +
                  Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       323 
323 
     | 
    
         
             
                end
         
     | 
| 
       324 
324 
     | 
    
         
             
              end
         
     | 
| 
       325 
325 
     | 
    
         | 
| 
         @@ -377,6 +377,7 @@ module TSV 
     | 
|
| 
       377 
377 
     | 
    
         
             
                    obj.run(true) unless obj.done? || obj.started? || obj.result
         
     | 
| 
       378 
378 
     | 
    
         | 
| 
       379 
379 
     | 
    
         
             
                    stream = obj.get_stream
         
     | 
| 
      
 380 
     | 
    
         
            +
                    options = {:type => :array}.merge(options) if obj.result_type == :array
         
     | 
| 
       380 
381 
     | 
    
         | 
| 
       381 
382 
     | 
    
         
             
                    if stream
         
     | 
| 
       382 
383 
     | 
    
         
             
                      traverse_obj(stream, options, &block)
         
     | 
| 
         @@ -727,8 +728,6 @@ module TSV 
     | 
|
| 
       727 
728 
     | 
    
         
             
                    Log.exception $!
         
     | 
| 
       728 
729 
     | 
    
         
             
                    abort_stream(into, $!)
         
     | 
| 
       729 
730 
     | 
    
         
             
                    raise $!
         
     | 
| 
       730 
     | 
    
         
            -
                  ensure
         
     | 
| 
       731 
     | 
    
         
            -
                    Log::ProgressBar.remove_bar(bar) if bar
         
     | 
| 
       732 
731 
     | 
    
         
             
                  end
         
     | 
| 
       733 
732 
     | 
    
         | 
| 
       734 
733 
     | 
    
         
             
                  into
         
     | 
    
        data/lib/rbbt/tsv/util.rb
    CHANGED
    
    | 
         @@ -316,22 +316,30 @@ module TSV 
     | 
|
| 
       316 
316 
     | 
    
         | 
| 
       317 
317 
     | 
    
         
             
              def to_single
         
     | 
| 
       318 
318 
     | 
    
         
             
                new = {}
         
     | 
| 
       319 
     | 
    
         
            -
             
     | 
| 
       320 
     | 
    
         
            -
                 
     | 
| 
       321 
     | 
    
         
            -
                  through do |k,v|
         
     | 
| 
       322 
     | 
    
         
            -
                    new[k] = v.first.first
         
     | 
| 
       323 
     | 
    
         
            -
                  end
         
     | 
| 
       324 
     | 
    
         
            -
                when :flat
         
     | 
| 
      
 319 
     | 
    
         
            +
             
     | 
| 
      
 320 
     | 
    
         
            +
                if block_given?
         
     | 
| 
       325 
321 
     | 
    
         
             
                  through do |k,v|
         
     | 
| 
       326 
     | 
    
         
            -
                    new[k] = v 
     | 
| 
      
 322 
     | 
    
         
            +
                    new[k] = yield v
         
     | 
| 
       327 
323 
     | 
    
         
             
                  end
         
     | 
| 
       328 
     | 
    
         
            -
                 
     | 
| 
       329 
     | 
    
         
            -
                   
     | 
| 
       330 
     | 
    
         
            -
             
     | 
| 
       331 
     | 
    
         
            -
             
     | 
| 
       332 
     | 
    
         
            -
             
     | 
| 
      
 324 
     | 
    
         
            +
                else
         
     | 
| 
      
 325 
     | 
    
         
            +
                  case type
         
     | 
| 
      
 326 
     | 
    
         
            +
                  when :double
         
     | 
| 
      
 327 
     | 
    
         
            +
                    through do |k,v|
         
     | 
| 
      
 328 
     | 
    
         
            +
                      new[k] = v.first.first
         
     | 
| 
      
 329 
     | 
    
         
            +
                    end
         
     | 
| 
      
 330 
     | 
    
         
            +
                  when :flat
         
     | 
| 
      
 331 
     | 
    
         
            +
                    through do |k,v|
         
     | 
| 
      
 332 
     | 
    
         
            +
                      new[k] = v.first
         
     | 
| 
      
 333 
     | 
    
         
            +
                    end
         
     | 
| 
      
 334 
     | 
    
         
            +
                  when :single
         
     | 
| 
      
 335 
     | 
    
         
            +
                    return self
         
     | 
| 
      
 336 
     | 
    
         
            +
                  when :list
         
     | 
| 
      
 337 
     | 
    
         
            +
                    through do |k,v|
         
     | 
| 
      
 338 
     | 
    
         
            +
                      new[k] = v.first
         
     | 
| 
      
 339 
     | 
    
         
            +
                    end
         
     | 
| 
       333 
340 
     | 
    
         
             
                  end
         
     | 
| 
       334 
341 
     | 
    
         
             
                end
         
     | 
| 
      
 342 
     | 
    
         
            +
             
     | 
| 
       335 
343 
     | 
    
         
             
                self.annotate(new)
         
     | 
| 
       336 
344 
     | 
    
         
             
                new.type = :single
         
     | 
| 
       337 
345 
     | 
    
         
             
                new.fields = [new.fields.first] if new.fields.length > 1
         
     | 
| 
         @@ -290,9 +290,9 @@ module Misc 
     | 
|
| 
       290 
290 
     | 
    
         
             
                        if obj.exists?
         
     | 
| 
       291 
291 
     | 
    
         
             
                          if obj.directory?
         
     | 
| 
       292 
292 
     | 
    
         
             
                            files = obj.glob("**/*")
         
     | 
| 
       293 
     | 
    
         
            -
                            "directory: #{files}"
         
     | 
| 
      
 293 
     | 
    
         
            +
                            "directory: #{Misc.fingerprint(files)}"
         
     | 
| 
       294 
294 
     | 
    
         
             
                          else
         
     | 
| 
       295 
     | 
    
         
            -
                            "file: " << obj << "--" << mtime_str(obj)
         
     | 
| 
      
 295 
     | 
    
         
            +
                            "file: " << Open.realpath(obj) << "--" << mtime_str(obj)
         
     | 
| 
       296 
296 
     | 
    
         
             
                          end
         
     | 
| 
       297 
297 
     | 
    
         
             
                        else
         
     | 
| 
       298 
298 
     | 
    
         
             
                          obj + " (file missing)"
         
     | 
| 
         @@ -401,7 +401,7 @@ module Misc 
     | 
|
| 
       401 
401 
     | 
    
         | 
| 
       402 
402 
     | 
    
         
             
              def self.scan_version_text(text, cmd = nil)
         
     | 
| 
       403 
403 
     | 
    
         
             
                cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
         
     | 
| 
       404 
     | 
    
         
            -
                m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
         
     | 
| 
      
 404 
     | 
    
         
            +
                m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
         
     | 
| 
       405 
405 
     | 
    
         
             
                return nil if m.nil?
         
     | 
| 
       406 
406 
     | 
    
         
             
                m[1]
         
     | 
| 
       407 
407 
     | 
    
         
             
              end
         
     | 
    
        data/lib/rbbt/workflow.rb
    CHANGED
    
    | 
         @@ -364,6 +364,7 @@ module Workflow 
     | 
|
| 
       364 
364 
     | 
    
         
             
                  raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
         
     | 
| 
       365 
365 
     | 
    
         
             
                end
         
     | 
| 
       366 
366 
     | 
    
         | 
| 
      
 367 
     | 
    
         
            +
                # jobname => true sets the value of the input to the name of the job
         
     | 
| 
       367 
368 
     | 
    
         
             
                if task.input_options
         
     | 
| 
       368 
369 
     | 
    
         
             
                  jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
         
     | 
| 
       369 
370 
     | 
    
         
             
                else
         
     | 
| 
         @@ -26,105 +26,106 @@ module Workflow 
     | 
|
| 
       26 
26 
     | 
    
         
             
                raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
         
     | 
| 
       27 
27 
     | 
    
         
             
                id = File.join(self.to_s, name.to_s)
         
     | 
| 
       28 
28 
     | 
    
         
             
                @task_info ||= {}
         
     | 
| 
       29 
     | 
    
         
            -
                @task_info[id] ||= begin  
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
             
     | 
| 
       32 
     | 
    
         
            -
             
     | 
| 
       33 
     | 
    
         
            -
             
     | 
| 
       34 
     | 
    
         
            -
             
     | 
| 
       35 
     | 
    
         
            -
             
     | 
| 
       36 
     | 
    
         
            -
             
     | 
| 
       37 
     | 
    
         
            -
             
     | 
| 
       38 
     | 
    
         
            -
             
     | 
| 
       39 
     | 
    
         
            -
             
     | 
| 
       40 
     | 
    
         
            -
             
     | 
| 
       41 
     | 
    
         
            -
             
     | 
| 
       42 
     | 
    
         
            -
                                                 
     | 
| 
       43 
     | 
    
         
            -
             
     | 
| 
       44 
     | 
    
         
            -
                                                 
     | 
| 
       45 
     | 
    
         
            -
             
     | 
| 
       46 
     | 
    
         
            -
                                                 
     | 
| 
       47 
     | 
    
         
            -
             
     | 
| 
       48 
     | 
    
         
            -
                                                 
     | 
| 
       49 
     | 
    
         
            -
             
     | 
| 
       50 
     | 
    
         
            -
                                                 
     | 
| 
      
 29 
     | 
    
         
            +
                @task_info[id] ||= begin 
         
     | 
| 
      
 30 
     | 
    
         
            +
                                     description = task.description
         
     | 
| 
      
 31 
     | 
    
         
            +
                                     result_description = task.result_description
         
     | 
| 
      
 32 
     | 
    
         
            +
                                     result_type = task.result_type
         
     | 
| 
      
 33 
     | 
    
         
            +
                                     inputs = rec_inputs(name).uniq
         
     | 
| 
      
 34 
     | 
    
         
            +
                                     input_types = rec_input_types(name)
         
     | 
| 
      
 35 
     | 
    
         
            +
                                     input_descriptions = rec_input_descriptions(name)
         
     | 
| 
      
 36 
     | 
    
         
            +
                                     input_use = rec_input_use(name)
         
     | 
| 
      
 37 
     | 
    
         
            +
                                     input_defaults = rec_input_defaults(name)
         
     | 
| 
      
 38 
     | 
    
         
            +
                                     input_options = rec_input_options(name)
         
     | 
| 
      
 39 
     | 
    
         
            +
                                     extension = task.extension
         
     | 
| 
      
 40 
     | 
    
         
            +
                                     export = case
         
     | 
| 
      
 41 
     | 
    
         
            +
                                              when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
         
     | 
| 
      
 42 
     | 
    
         
            +
                                                :synchronous
         
     | 
| 
      
 43 
     | 
    
         
            +
                                              when (asynchronous_exports.include?(name.to_sym) or asynchronous_exports.include?(name.to_s))
         
     | 
| 
      
 44 
     | 
    
         
            +
                                                :asynchronous
         
     | 
| 
      
 45 
     | 
    
         
            +
                                              when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
         
     | 
| 
      
 46 
     | 
    
         
            +
                                                :exec
         
     | 
| 
      
 47 
     | 
    
         
            +
                                              when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
         
     | 
| 
      
 48 
     | 
    
         
            +
                                                :stream
         
     | 
| 
      
 49 
     | 
    
         
            +
                                              else
         
     | 
| 
      
 50 
     | 
    
         
            +
                                                :none
         
     | 
| 
      
 51 
     | 
    
         
            +
                                              end
         
     | 
| 
       51 
52 
     | 
    
         | 
| 
       52 
     | 
    
         
            -
             
     | 
| 
       53 
     | 
    
         
            -
             
     | 
| 
       54 
     | 
    
         
            -
             
     | 
| 
       55 
     | 
    
         
            -
             
     | 
| 
       56 
     | 
    
         
            -
             
     | 
| 
       57 
     | 
    
         
            -
             
     | 
| 
       58 
     | 
    
         
            -
             
     | 
| 
       59 
     | 
    
         
            -
             
     | 
| 
       60 
     | 
    
         
            -
             
     | 
| 
       61 
     | 
    
         
            -
             
     | 
| 
       62 
     | 
    
         
            -
             
     | 
| 
       63 
     | 
    
         
            -
             
     | 
| 
       64 
     | 
    
         
            -
             
     | 
| 
       65 
     | 
    
         
            -
             
     | 
| 
       66 
     | 
    
         
            -
             
     | 
| 
       67 
     | 
    
         
            -
             
     | 
| 
      
 53 
     | 
    
         
            +
                                     dependencies = task_dependencies[name].select{|dep| String === dep or Symbol === dep}
         
     | 
| 
      
 54 
     | 
    
         
            +
                                     { :id => id,
         
     | 
| 
      
 55 
     | 
    
         
            +
                                       :description => description,
         
     | 
| 
      
 56 
     | 
    
         
            +
                                       :export => export,
         
     | 
| 
      
 57 
     | 
    
         
            +
                                       :inputs => inputs,
         
     | 
| 
      
 58 
     | 
    
         
            +
                                       :input_types => input_types,
         
     | 
| 
      
 59 
     | 
    
         
            +
                                       :input_descriptions => input_descriptions,
         
     | 
| 
      
 60 
     | 
    
         
            +
                                       :input_defaults => input_defaults,
         
     | 
| 
      
 61 
     | 
    
         
            +
                                       :input_options => input_options,
         
     | 
| 
      
 62 
     | 
    
         
            +
                                       :input_use => input_use,
         
     | 
| 
      
 63 
     | 
    
         
            +
                                       :result_type => result_type,
         
     | 
| 
      
 64 
     | 
    
         
            +
                                       :result_description => result_description,
         
     | 
| 
      
 65 
     | 
    
         
            +
                                       :dependencies => dependencies,
         
     | 
| 
      
 66 
     | 
    
         
            +
                                       :extension => extension
         
     | 
| 
      
 67 
     | 
    
         
            +
                                     }
         
     | 
| 
      
 68 
     | 
    
         
            +
                                   end
         
     | 
| 
       68 
69 
     | 
    
         
             
              end
         
     | 
| 
       69 
70 
     | 
    
         | 
| 
       70 
71 
     | 
    
         
             
              def rec_dependencies(taskname)
         
     | 
| 
       71 
72 
     | 
    
         
             
                @rec_dependencies ||= {}
         
     | 
| 
       72 
73 
     | 
    
         
             
                @rec_dependencies[taskname] ||= begin
         
     | 
| 
       73 
     | 
    
         
            -
             
     | 
| 
       74 
     | 
    
         
            -
             
     | 
| 
       75 
     | 
    
         
            -
             
     | 
| 
       76 
     | 
    
         
            -
             
     | 
| 
       77 
     | 
    
         
            -
             
     | 
| 
       78 
     | 
    
         
            -
             
     | 
| 
       79 
     | 
    
         
            -
             
     | 
| 
       80 
     | 
    
         
            -
             
     | 
| 
       81 
     | 
    
         
            -
             
     | 
| 
       82 
     | 
    
         
            -
             
     | 
| 
       83 
     | 
    
         
            -
             
     | 
| 
       84 
     | 
    
         
            -
             
     | 
| 
       85 
     | 
    
         
            -
             
     | 
| 
       86 
     | 
    
         
            -
             
     | 
| 
       87 
     | 
    
         
            -
             
     | 
| 
       88 
     | 
    
         
            -
             
     | 
| 
       89 
     | 
    
         
            -
             
     | 
| 
       90 
     | 
    
         
            -
             
     | 
| 
       91 
     | 
    
         
            -
             
     | 
| 
       92 
     | 
    
         
            -
             
     | 
| 
       93 
     | 
    
         
            -
             
     | 
| 
       94 
     | 
    
         
            -
             
     | 
| 
       95 
     | 
    
         
            -
             
     | 
| 
       96 
     | 
    
         
            -
             
     | 
| 
       97 
     | 
    
         
            -
             
     | 
| 
       98 
     | 
    
         
            -
             
     | 
| 
       99 
     | 
    
         
            -
             
     | 
| 
       100 
     | 
    
         
            -
             
     | 
| 
       101 
     | 
    
         
            -
             
     | 
| 
       102 
     | 
    
         
            -
             
     | 
| 
       103 
     | 
    
         
            -
             
     | 
| 
       104 
     | 
    
         
            -
             
     | 
| 
       105 
     | 
    
         
            -
             
     | 
| 
      
 74 
     | 
    
         
            +
                                                  if task_dependencies.include? taskname
         
     | 
| 
      
 75 
     | 
    
         
            +
             
     | 
| 
      
 76 
     | 
    
         
            +
                                                    deps = task_dependencies[taskname]
         
     | 
| 
      
 77 
     | 
    
         
            +
             
     | 
| 
      
 78 
     | 
    
         
            +
                                                    #all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
         
     | 
| 
      
 79 
     | 
    
         
            +
             
     | 
| 
      
 80 
     | 
    
         
            +
                                                    all_deps = []
         
     | 
| 
      
 81 
     | 
    
         
            +
                                                    deps.each do |dep| 
         
     | 
| 
      
 82 
     | 
    
         
            +
                                                      if DependencyBlock === dep
         
     | 
| 
      
 83 
     | 
    
         
            +
                                                        all_deps << dep.dependency if dep.dependency
         
     | 
| 
      
 84 
     | 
    
         
            +
                                                      else
         
     | 
| 
      
 85 
     | 
    
         
            +
                                                        all_deps << dep unless Proc === dep
         
     | 
| 
      
 86 
     | 
    
         
            +
                                                      end
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
                                                      begin
         
     | 
| 
      
 89 
     | 
    
         
            +
                                                        case dep
         
     | 
| 
      
 90 
     | 
    
         
            +
                                                        when Array
         
     | 
| 
      
 91 
     | 
    
         
            +
                                                          wf, t, o = dep
         
     | 
| 
      
 92 
     | 
    
         
            +
             
     | 
| 
      
 93 
     | 
    
         
            +
                                                          wf.rec_dependencies(t.to_sym).each do |d|
         
     | 
| 
      
 94 
     | 
    
         
            +
                                                            if Array === d
         
     | 
| 
      
 95 
     | 
    
         
            +
                                                              new = d.dup
         
     | 
| 
      
 96 
     | 
    
         
            +
                                                            else
         
     | 
| 
      
 97 
     | 
    
         
            +
                                                              new = [dep.first, d]
         
     | 
| 
      
 98 
     | 
    
         
            +
                                                            end
         
     | 
| 
      
 99 
     | 
    
         
            +
             
     | 
| 
      
 100 
     | 
    
         
            +
                                                            if Hash === o and not o.empty? 
         
     | 
| 
      
 101 
     | 
    
         
            +
                                                              if Hash === new.last
         
     | 
| 
      
 102 
     | 
    
         
            +
                                                                hash = new.last.dup
         
     | 
| 
      
 103 
     | 
    
         
            +
                                                                o.each{|k,v| hash[k] ||= v}
         
     | 
| 
      
 104 
     | 
    
         
            +
                                                                new[new.length-1] = hash
         
     | 
| 
      
 105 
     | 
    
         
            +
                                                              else
         
     | 
| 
      
 106 
     | 
    
         
            +
                                                                new.push o.dup
         
     | 
| 
      
 107 
     | 
    
         
            +
                                                              end
         
     | 
| 
      
 108 
     | 
    
         
            +
                                                            end
         
     | 
| 
      
 109 
     | 
    
         
            +
             
     | 
| 
      
 110 
     | 
    
         
            +
                                                            all_deps << new
         
     | 
| 
      
 111 
     | 
    
         
            +
                                                          end if wf && t
         
     | 
| 
      
 112 
     | 
    
         
            +
             
     | 
| 
      
 113 
     | 
    
         
            +
                                                        when String, Symbol
         
     | 
| 
      
 114 
     | 
    
         
            +
                                                          rec_deps = rec_dependencies(dep.to_sym)
         
     | 
| 
      
 115 
     | 
    
         
            +
                                                          all_deps.concat rec_deps
         
     | 
| 
      
 116 
     | 
    
         
            +
                                                        when DependencyBlock
         
     | 
| 
      
 117 
     | 
    
         
            +
                                                          dep = dep.dependency
         
     | 
| 
      
 118 
     | 
    
         
            +
                                                          raise TryAgain
         
     | 
| 
      
 119 
     | 
    
         
            +
                                                        end
         
     | 
| 
      
 120 
     | 
    
         
            +
                                                      rescue TryAgain
         
     | 
| 
      
 121 
     | 
    
         
            +
                                                        retry
         
     | 
| 
      
 122 
     | 
    
         
            +
                                                      end
         
     | 
| 
       106 
123 
     | 
    
         
             
                                                    end
         
     | 
| 
      
 124 
     | 
    
         
            +
                                                    all_deps.uniq
         
     | 
| 
      
 125 
     | 
    
         
            +
                                                  else
         
     | 
| 
      
 126 
     | 
    
         
            +
                                                    []
         
     | 
| 
       107 
127 
     | 
    
         
             
                                                  end
         
     | 
| 
       108 
     | 
    
         
            -
             
     | 
| 
       109 
     | 
    
         
            -
                                                  all_deps << new
         
     | 
| 
       110 
     | 
    
         
            -
                                                end if wf && t
         
     | 
| 
       111 
     | 
    
         
            -
             
     | 
| 
       112 
     | 
    
         
            -
                                              when String, Symbol
         
     | 
| 
       113 
     | 
    
         
            -
                                                rec_deps = rec_dependencies(dep.to_sym)
         
     | 
| 
       114 
     | 
    
         
            -
                                                all_deps.concat rec_deps
         
     | 
| 
       115 
     | 
    
         
            -
                                              when DependencyBlock
         
     | 
| 
       116 
     | 
    
         
            -
                                                dep = dep.dependency
         
     | 
| 
       117 
     | 
    
         
            -
                                                raise TryAgain
         
     | 
| 
       118 
     | 
    
         
            -
                                              end
         
     | 
| 
       119 
     | 
    
         
            -
                                            rescue TryAgain
         
     | 
| 
       120 
     | 
    
         
            -
                                              retry
         
     | 
| 
       121 
     | 
    
         
            -
                                            end
         
     | 
| 
       122 
     | 
    
         
            -
                                          end
         
     | 
| 
       123 
     | 
    
         
            -
                                          all_deps.uniq
         
     | 
| 
       124 
     | 
    
         
            -
                                        else
         
     | 
| 
       125 
     | 
    
         
            -
                                          []
         
     | 
| 
       126 
     | 
    
         
            -
                                        end
         
     | 
| 
       127 
     | 
    
         
            -
                                      end
         
     | 
| 
      
 128 
     | 
    
         
            +
                                                end
         
     | 
| 
       128 
129 
     | 
    
         
             
              end
         
     | 
| 
       129 
130 
     | 
    
         | 
| 
       130 
131 
     | 
    
         
             
              def task_from_dep(dep)
         
     | 
| 
         @@ -182,7 +183,7 @@ module Workflow 
     | 
|
| 
       182 
183 
     | 
    
         
             
                  acc
         
     | 
| 
       183 
184 
     | 
    
         
             
                }.tap{|h| IndiferentHash.setup(h)}
         
     | 
| 
       184 
185 
     | 
    
         
             
              end
         
     | 
| 
       185 
     | 
    
         
            -
             
     | 
| 
      
 186 
     | 
    
         
            +
             
     | 
| 
       186 
187 
     | 
    
         
             
              def rec_input_use(taskname)
         
     | 
| 
       187 
188 
     | 
    
         
             
                task = task_from_dep(taskname)
         
     | 
| 
       188 
189 
     | 
    
         
             
                deps = rec_dependencies(taskname)
         
     | 
| 
         @@ -474,7 +475,7 @@ module Workflow 
     | 
|
| 
       474 
475 
     | 
    
         
             
                else
         
     | 
| 
       475 
476 
     | 
    
         
             
                  workdir_find = workdir
         
     | 
| 
       476 
477 
     | 
    
         
             
                end
         
     | 
| 
       477 
     | 
    
         
            -
             
     | 
| 
      
 478 
     | 
    
         
            +
             
     | 
| 
       478 
479 
     | 
    
         
             
                workdir_find = File.expand_path(workdir_find)
         
     | 
| 
       479 
480 
     | 
    
         
             
                path = File.expand_path(path)
         
     | 
| 
       480 
481 
     | 
    
         
             
                dir = File.dirname(path)
         
     | 
| 
         @@ -1,13 +1,20 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module  
     | 
| 
      
 1 
     | 
    
         
            +
            module Cromwell
         
     | 
| 
       2 
2 
     | 
    
         | 
| 
       3 
3 
     | 
    
         
             
              Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
         
     | 
| 
       4 
4 
     | 
    
         
             
              Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
         
     | 
| 
       5 
5 
     | 
    
         | 
| 
       6 
     | 
    
         
            -
              def run_cromwell(file, work_dir, options = {})
         
     | 
| 
      
 6 
     | 
    
         
            +
              def self.run_cromwell(file, work_dir, options = {})
         
     | 
| 
      
 7 
     | 
    
         
            +
                cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
         
     | 
| 
       7 
8 
     | 
    
         
             
                jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
         
     | 
| 
       8 
     | 
    
         
            -
                 
     | 
| 
      
 9 
     | 
    
         
            +
                if cromwell_inputs_file
         
     | 
| 
      
 10 
     | 
    
         
            +
                  CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}' -i #{cromwell_inputs_file}", options.merge("add_option_dashes" => true))
         
     | 
| 
      
 11 
     | 
    
         
            +
                else
         
     | 
| 
      
 12 
     | 
    
         
            +
                  CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
         
     | 
| 
      
 13 
     | 
    
         
            +
                end
         
     | 
| 
       9 
14 
     | 
    
         
             
              end
         
     | 
| 
      
 15 
     | 
    
         
            +
            end
         
     | 
| 
       10 
16 
     | 
    
         | 
| 
      
 17 
     | 
    
         
            +
            module Workflow
         
     | 
| 
       11 
18 
     | 
    
         
             
              def load_cromwell(file)
         
     | 
| 
       12 
19 
     | 
    
         
             
                jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
         
     | 
| 
       13 
20 
     | 
    
         
             
                inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))
         
     | 
| 
         @@ -85,7 +85,11 @@ class RemoteWorkflow 
     | 
|
| 
       85 
85 
     | 
    
         | 
| 
       86 
86 
     | 
    
         
             
                  RemoteWorkflow::REST.__prepare_inputs_for_restclient(params)
         
     | 
| 
       87 
87 
     | 
    
         
             
                  name = RemoteWorkflow.capture_exception do
         
     | 
| 
       88 
     | 
    
         
            -
                     
     | 
| 
      
 88 
     | 
    
         
            +
                    begin
         
     | 
| 
      
 89 
     | 
    
         
            +
                      RestClient.post(self.encode(url), params)
         
     | 
| 
      
 90 
     | 
    
         
            +
                    rescue RestClient::MovedPermanently, RestClient::Found, RestClient::TemporaryRedirect
         
     | 
| 
      
 91 
     | 
    
         
            +
                      raise RbbtException, "REST end-point moved to: #{$!.response.headers[:location]}"
         
     | 
| 
      
 92 
     | 
    
         
            +
                    end
         
     | 
| 
       89 
93 
     | 
    
         
             
                  end
         
     | 
| 
       90 
94 
     | 
    
         | 
| 
       91 
95 
     | 
    
         
             
                  Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
         
     | 
| 
         @@ -340,6 +340,11 @@ class RemoteStep < Step 
     | 
|
| 
       340 
340 
     | 
    
         
             
                @inputs = new_inputs
         
     | 
| 
       341 
341 
     | 
    
         
             
                @info = nil
         
     | 
| 
       342 
342 
     | 
    
         
             
              end
         
     | 
| 
      
 343 
     | 
    
         
            +
             
     | 
| 
      
 344 
     | 
    
         
            +
              def init_info(*args)
         
     | 
| 
      
 345 
     | 
    
         
            +
                i = {:status => :waiting, :pid => Process.pid, :path => path}
         
     | 
| 
      
 346 
     | 
    
         
            +
                i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
         
     | 
| 
      
 347 
     | 
    
         
            +
              end
         
     | 
| 
       343 
348 
     | 
    
         
             
            end
         
     | 
| 
       344 
349 
     | 
    
         | 
| 
       345 
350 
     | 
    
         
             
            require 'rbbt/workflow/remote_workflow/remote_step/rest'
         
     | 
| 
         @@ -93,8 +93,8 @@ class Step 
     | 
|
| 
       93 
93 
     | 
    
         
             
                    else
         
     | 
| 
       94 
94 
     | 
    
         
             
                      Open.write(path + '.read', value.to_s)
         
     | 
| 
       95 
95 
     | 
    
         
             
                    end
         
     | 
| 
       96 
     | 
    
         
            -
                  when Step ===  
     | 
| 
       97 
     | 
    
         
            -
                     
     | 
| 
      
 96 
     | 
    
         
            +
                  when Step === value
         
     | 
| 
      
 97 
     | 
    
         
            +
                    value = value.produce.load 
         
     | 
| 
       98 
98 
     | 
    
         
             
                  else
         
     | 
| 
       99 
99 
     | 
    
         
             
                    Open.write(path, value.to_s)
         
     | 
| 
       100 
100 
     | 
    
         
             
                  end
         
     | 
| 
         @@ -110,12 +110,14 @@ class Step 
     | 
|
| 
       110 
110 
     | 
    
         
             
                task_info = workflow.task_info(task_name)
         
     | 
| 
       111 
111 
     | 
    
         
             
                input_types = task_info[:input_types]
         
     | 
| 
       112 
112 
     | 
    
         
             
                task_inputs = task_info[:inputs]
         
     | 
| 
      
 113 
     | 
    
         
            +
                input_defaults = task_info[:input_defaults]
         
     | 
| 
       113 
114 
     | 
    
         | 
| 
       114 
115 
     | 
    
         
             
                inputs = {}
         
     | 
| 
       115 
116 
     | 
    
         
             
                job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
         
     | 
| 
       116 
117 
     | 
    
         
             
                  next unless task_inputs.include? name.to_sym
         
     | 
| 
       117 
118 
     | 
    
         
             
                  next if options and ! options.include?(name)
         
     | 
| 
       118 
119 
     | 
    
         
             
                  next if value.nil?
         
     | 
| 
      
 120 
     | 
    
         
            +
                  next if input_defaults[name] == value
         
     | 
| 
       119 
121 
     | 
    
         
             
                  inputs[name] = value
         
     | 
| 
       120 
122 
     | 
    
         
             
                end
         
     | 
| 
       121 
123 
     | 
    
         | 
| 
         @@ -125,7 +127,7 @@ class Step 
     | 
|
| 
       125 
127 
     | 
    
         
             
                end
         
     | 
| 
       126 
128 
     | 
    
         
             
                save_inputs(inputs, input_types, dir)
         
     | 
| 
       127 
129 
     | 
    
         | 
| 
       128 
     | 
    
         
            -
                inputs. 
     | 
| 
      
 130 
     | 
    
         
            +
                inputs.keys
         
     | 
| 
       129 
131 
     | 
    
         
             
              end
         
     | 
| 
       130 
132 
     | 
    
         | 
| 
       131 
133 
     | 
    
         
             
              def name
         
     | 
| 
         @@ -208,7 +210,7 @@ class Step 
     | 
|
| 
       208 
210 
     | 
    
         
             
              end
         
     | 
| 
       209 
211 
     | 
    
         | 
| 
       210 
212 
     | 
    
         
             
              def init_info(force = false)
         
     | 
| 
       211 
     | 
    
         
            -
                return nil if @exec  
     | 
| 
      
 213 
     | 
    
         
            +
                return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
         
     | 
| 
       212 
214 
     | 
    
         
             
                Open.lock(info_file, :lock => info_lock) do
         
     | 
| 
       213 
215 
     | 
    
         
             
                  i = {:status => :waiting, :pid => Process.pid, :path => path}
         
     | 
| 
       214 
216 
     | 
    
         
             
                  i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
         
     | 
| 
         @@ -437,11 +439,12 @@ class Step 
     | 
|
| 
       437 
439 
     | 
    
         
             
                rec_dependencies = self.rec_dependencies
         
     | 
| 
       438 
440 
     | 
    
         
             
                return [] if rec_dependencies.empty?
         
     | 
| 
       439 
441 
     | 
    
         
             
                canfail_paths = self.canfail_paths
         
     | 
| 
      
 442 
     | 
    
         
            +
                dep = rec_dependencies.select{|d| d.task_name.to_s == 'contamination'}.first
         
     | 
| 
       440 
443 
     | 
    
         
             
                dirty_files = rec_dependencies.reject{|dep|
         
     | 
| 
       441 
444 
     | 
    
         
             
                  (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
         
     | 
| 
       442 
445 
     | 
    
         
             
                    ! Open.exists?(dep.info_file) ||
         
     | 
| 
       443 
446 
     | 
    
         
             
                    (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
         
     | 
| 
       444 
     | 
    
         
            -
                    ((dep.error? || dep.aborted? 
     | 
| 
      
 447 
     | 
    
         
            +
                    ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
         
     | 
| 
       445 
448 
     | 
    
         
             
                }
         
     | 
| 
       446 
449 
     | 
    
         
             
              end
         
     | 
| 
       447 
450 
     | 
    
         | 
| 
         @@ -508,12 +511,12 @@ class Step 
     | 
|
| 
       508 
511 
     | 
    
         | 
| 
       509 
512 
     | 
    
         
             
              def nopid?
         
     | 
| 
       510 
513 
     | 
    
         
             
                pid = info[:pid] || Open.exists?(pid_file)
         
     | 
| 
       511 
     | 
    
         
            -
                ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error)
         
     | 
| 
      
 514 
     | 
    
         
            +
                ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
         
     | 
| 
       512 
515 
     | 
    
         
             
              end
         
     | 
| 
       513 
516 
     | 
    
         | 
| 
       514 
517 
     | 
    
         
             
              def aborted?
         
     | 
| 
       515 
518 
     | 
    
         
             
                status = self.status
         
     | 
| 
       516 
     | 
    
         
            -
                status == :aborted || ((status != :noinfo && status != :setup && status != :noinfo) && nopid?)
         
     | 
| 
      
 519 
     | 
    
         
            +
                status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
         
     | 
| 
       517 
520 
     | 
    
         
             
              end
         
     | 
| 
       518 
521 
     | 
    
         | 
| 
       519 
522 
     | 
    
         
             
              # {{{ INFO
         
     | 
| 
         @@ -24,7 +24,7 @@ class Step 
     | 
|
| 
       24 
24 
     | 
    
         
             
                      Log.medium "Not duplicating stream #{stream_key}"
         
     | 
| 
       25 
25 
     | 
    
         
             
                      STREAM_CACHE[stream_key] = stream
         
     | 
| 
       26 
26 
     | 
    
         
             
                    when File
         
     | 
| 
       27 
     | 
    
         
            -
                      if Open.exists? 
     | 
| 
      
 27 
     | 
    
         
            +
                      if Open.exists?(current.path)
         
     | 
| 
       28 
28 
     | 
    
         
             
                        Log.medium "Reopening file #{stream_key}"
         
     | 
| 
       29 
29 
     | 
    
         
             
                        Open.open(current.path)
         
     | 
| 
       30 
30 
     | 
    
         
             
                      else
         
     | 
| 
         @@ -97,7 +97,7 @@ class Step 
     | 
|
| 
       97 
97 
     | 
    
         
             
                  end
         
     | 
| 
       98 
98 
     | 
    
         | 
| 
       99 
99 
     | 
    
         
             
                  job.dup_inputs unless status == 'done' or job.started?
         
     | 
| 
       100 
     | 
    
         
            -
                  job.init_info unless status == ' 
     | 
| 
      
 100 
     | 
    
         
            +
                  job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
         
     | 
| 
       101 
101 
     | 
    
         | 
| 
       102 
102 
     | 
    
         
             
                  canfail = ComputeDependency === job && job.canfail?
         
     | 
| 
       103 
103 
     | 
    
         
             
                end
         
     | 
    
        data/lib/rbbt/workflow/usage.rb
    CHANGED
    
    
| 
         @@ -54,9 +54,9 @@ app = ARGV.shift 
     | 
|
| 
       54 
54 
     | 
    
         | 
| 
       55 
55 
     | 
    
         
             
            ENV["RServe-session"] = options[:RServe_session] || app
         
     | 
| 
       56 
56 
     | 
    
         | 
| 
       57 
     | 
    
         
            -
            app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps 
     | 
| 
      
 57 
     | 
    
         
            +
            app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
         
     | 
| 
       58 
58 
     | 
    
         | 
| 
       59 
     | 
    
         
            -
            app_dir = app_dir[app]
         
     | 
| 
      
 59 
     | 
    
         
            +
            app_dir = app_dir[app].find
         
     | 
| 
       60 
60 
     | 
    
         | 
| 
       61 
61 
     | 
    
         
             
            server = options[:server] || 'puma'
         
     | 
| 
       62 
62 
     | 
    
         
             
            Misc.in_dir(app_dir) do
         
     | 
| 
         @@ -25,6 +25,7 @@ $ rbbt workflow server [options] <Workflow> 
     | 
|
| 
       25 
25 
     | 
    
         
             
            --stream Activate streaming of workflow tasks
         
     | 
| 
       26 
26 
     | 
    
         
             
            -fs--file_server Activate file serving for resources
         
     | 
| 
       27 
27 
     | 
    
         
             
            -mj--monitor_jobs Monitor jobs (UNSAFE)
         
     | 
| 
      
 28 
     | 
    
         
            +
            -a--app_dir* Application execution directory
         
     | 
| 
       28 
29 
     | 
    
         
             
            --export_all Export all workflow tasks (use with caution!)
         
     | 
| 
       29 
30 
     | 
    
         
             
            --export* Export workflow tasks (asynchronous)
         
     | 
| 
       30 
31 
     | 
    
         
             
            --export_asynchronous* Export workflow tasks as asynchronous
         
     | 
| 
         @@ -74,6 +75,7 @@ sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_syn 
     | 
|
| 
       74 
75 
     | 
    
         
             
            exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
         
     | 
| 
       75 
76 
     | 
    
         | 
| 
       76 
77 
     | 
    
         
             
            TmpFile.with_file do |app_dir|
         
     | 
| 
      
 78 
     | 
    
         
            +
              app_dir = options[:app_dir] if options[:app_dir]
         
     | 
| 
       77 
79 
     | 
    
         
             
              Misc.in_dir(app_dir) do
         
     | 
| 
       78 
80 
     | 
    
         
             
                app_dir = Path.setup(app_dir.dup)
         
     | 
| 
       79 
81 
     | 
    
         
             
                Open.write(app_dir.etc.target_workflow.find, workflow)
         
     | 
| 
         @@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase 
     | 
|
| 
       202 
202 
     | 
    
         
             
                stream = datafile_test('identifiers').open 
         
     | 
| 
       203 
203 
     | 
    
         
             
                dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
         
     | 
| 
       204 
204 
     | 
    
         
             
                dumper.init
         
     | 
| 
       205 
     | 
    
         
            -
                TSV.traverse stream, :head => head, :into => dumper do |k,v|
         
     | 
| 
      
 205 
     | 
    
         
            +
                TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
         
     | 
| 
       206 
206 
     | 
    
         
             
                  k = k.first
         
     | 
| 
       207 
207 
     | 
    
         
             
                  [k,v]
         
     | 
| 
       208 
208 
     | 
    
         
             
                end
         
     | 
| 
         @@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase 
     | 
|
| 
       212 
212 
     | 
    
         
             
                assert_equal head, res.size
         
     | 
| 
       213 
213 
     | 
    
         
             
              end
         
     | 
| 
       214 
214 
     | 
    
         | 
| 
      
 215 
     | 
    
         
            +
              def test_traverse_into_dumper_error_bar
         
     | 
| 
      
 216 
     | 
    
         
            +
                require 'rbbt/sources/organism'
         
     | 
| 
      
 217 
     | 
    
         
            +
             
     | 
| 
      
 218 
     | 
    
         
            +
                head = 2_000
         
     | 
| 
      
 219 
     | 
    
         
            +
             
     | 
| 
      
 220 
     | 
    
         
            +
                stream = datafile_test('identifiers').open 
         
     | 
| 
      
 221 
     | 
    
         
            +
                dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
         
     | 
| 
      
 222 
     | 
    
         
            +
                dumper.init
         
     | 
| 
      
 223 
     | 
    
         
            +
                TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
         
     | 
| 
      
 224 
     | 
    
         
            +
                  k = k.first
         
     | 
| 
      
 225 
     | 
    
         
            +
                  raise 
         
     | 
| 
      
 226 
     | 
    
         
            +
                  [k,v]
         
     | 
| 
      
 227 
     | 
    
         
            +
                end
         
     | 
| 
      
 228 
     | 
    
         
            +
             
     | 
| 
      
 229 
     | 
    
         
            +
                assert_raise do 
         
     | 
| 
      
 230 
     | 
    
         
            +
                  res = TSV.open(dumper.stream)
         
     | 
| 
      
 231 
     | 
    
         
            +
                end
         
     | 
| 
      
 232 
     | 
    
         
            +
             
     | 
| 
      
 233 
     | 
    
         
            +
              end
         
     | 
| 
      
 234 
     | 
    
         
            +
             
     | 
| 
       215 
235 
     | 
    
         
             
              def test_traverse_into_dumper_threads
         
     | 
| 
       216 
236 
     | 
    
         
             
                require 'rbbt/sources/organism'
         
     | 
| 
       217 
237 
     | 
    
         | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: rbbt-util
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 5.28. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 5.28.6
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Miguel Vazquez
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire: 
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: bin
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2020- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2020-10-15 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              name: rake
         
     |