rbbt-util 5.20.26 → 5.21.0

data/lib/rbbt/workflow/step/run.rb

@@ -94,7 +94,7 @@ class Step
 
     begin
       @mutex.synchronize do
-        no_load = no_load ? :stream : false
+        no_load = :stream if no_load
         result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
@@ -180,7 +180,6 @@ class Step
               rescue
                 Log.exception $!
               ensure
-                join
                 Step.purge_stream_cache
                 FileUtils.rm pid_file if File.exist?(pid_file)
               end
@@ -255,24 +254,41 @@ class Step
 
     clean if dirty? or (not running? and not done?)
 
-    run(:stream) unless started?
+    no_load = :stream
+    run(false) unless started?
 
     join unless done?
 
     self
   end
 
-  def fork(semaphore = nil)
+  def fork(no_load = false, semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
+    sout, sin = Misc.pipe if no_load == :stream
     @pid = Process.fork do
+      sout.close if sout
       Misc.pre_fork
       begin
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless File.exist? File.dirname(path)
         begin
           @forked = true
-          res = run true
+          res = run no_load
           set_info :forked, true
+          if sin
+            io = TSV.get_stream res
+            if io.respond_to? :setup
+              io.setup(sin) 
+              sin.pair = io
+              io.pair = sin
+            end
+            begin
+              Misc.consume_stream(io, false, sin)
+            rescue 
+              Log.warn "Could not consume stream (#{io.closed? ? 'closed' : 'open'}) into pipe for forked job: #{self.path}"
+              Misc.consume_stream(io) unless io.closed?
+            end
+          end
         rescue Aborted, Interrupt
           Log.debug{"Forked process aborted: #{path}"}
           log :aborted, "Job aborted (#{Process.pid})"
@@ -309,6 +325,8 @@ class Step
         Kernel.exit! 0
       end
     end
+    sin.close if sin
+    @result = sout if sout 
     Process.detach(@pid)
     self
   end

data/share/rbbt_commands/workflow/server

@@ -1,4 +1,3 @@
-#!/usr/bin/env ruby
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
@@ -17,12 +16,16 @@ $ rbbt workflow server [options] <Workflow>
 -p--port* TCP port
 -s--server* Server type:  thin, webrick, unicorn, etc
 -f--finder Start server with finder functionality
--R--Rserve_session* Rserve session to use, otherwise start new one
+-RS--Rserve_session* Rserve session to use, otherwise start new one
 -wd--workdir* Change the working directory of the workflow
 -W--workflows* List of additional workflows to load
 --views* Directory with view templates
 --stream Activate streaming of workflow tasks
---export* Export workflow tasks 
+--export* Export workflow tasks (asynchronous)
+--export_asynchronous* Export workflow tasks as asynchronous
+--export_synchronous* Export workflow tasks as synchronous
+--export_exec* Export workflow tasks as exec
+--export_stream* Export workflow tasks as stream
 --options* Additional options for server (e.g. option1=value1;option2=value2)
 EOF
 
@@ -54,12 +57,22 @@ ENV["RServe-session"] = options[:RServe_session] || workflow
 server = options[:server] || 'puma'
 
 exports = options[:export].split(/\s*,/) if options[:export]
+stream_exports = options[:export_stream].split(/\s*,/) if options[:export_stream]
+async_exports = options[:export_asynchronous].split(/\s*,/) if options[:export_asynchronous]
+sync_exports = options[:export_synchronous].split(/\s*,/) if options[:export_synchronous]
+exec_exports = options[:export_exec].split(/\s*,/) if options[:export_exec]
 
 TmpFile.with_file do |app_dir|
   Misc.in_dir(app_dir) do
     app_dir = Path.setup(app_dir.dup)
     Open.write(app_dir.etc.target_workflow.find, workflow)
+
     Open.write(app_dir.etc.target_workflow_exports.find, exports * "\n") if exports
+    Open.write(app_dir.etc.target_workflow_stream_exports.find, stream_exports * "\n") if stream_exports
+    Open.write(app_dir.etc.target_workflow_async_exports.find, async_exports * "\n") if async_exports
+    Open.write(app_dir.etc.target_workflow_sync_exports.find, sync_exports * "\n") if sync_exports
+    Open.write(app_dir.etc.target_workflow_exec_exports.find, exec_exports * "\n") if exec_exports
+
     Open.write(app_dir.etc.workflows.find, workflows.split(/,\s/)*"\n") if workflows 
 
     require 'rack'

data/share/rbbt_commands/workflow/task

@@ -192,6 +192,7 @@ the job dependencies recursively.
 --provenance Report the jobs provenance
 -W--workflows* Load a list of workflows
 -R--requires* Require a list of files
+-rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 EOF
 
 workflow = ARGV.shift
@@ -237,6 +238,13 @@ if options[:requires]
     require req
   end
 end
+
+if options[:remote_workflow_tasks]
+  yaml_text = Open.read(options[:remote_workflow_tasks])
+  remote_workflow_tasks = YAML.load(yaml_text)
+  Workflow.process_remote_tasks(remote_workflow_tasks)
+end
+
 # Set task
 namespace = nil, nil
 
@@ -354,6 +362,7 @@ begin
 
   if options.delete(:printpath)
     job.join
+    raise job.messages.last if job.error?
     if Open.remote? job.path
       puts job.url + Log.color(:blue, "?_format=raw")
     else

data/share/workflow_config.ru

@@ -30,13 +30,6 @@ require Rbbt.etc['app.d/init.rb'].find
 workflow = Rbbt.etc['target_workflow'].read
 wf = Workflow.require_workflow workflow, true
 
-if Rbbt.etc['target_workflow_exports'].exists?
-  exports = Rbbt.etc['target_workflow_exports'].read.split("\n")
-  exports.each do |task|
-    wf.export task.to_sym
-  end
-end
-
 $title = wf.to_s
 $app_name = app_name = wf.to_s + "REST"
 $app = app = eval "class #{app_name} < Sinatra::Base; self end"
@@ -80,6 +73,41 @@ load_file Rbbt.etc['app.d/post.rb'].find_all
 #{{{ PRELOAD
 load_file Rbbt.etc['app.d/preload.rb'].find_all
 
+if Rbbt.etc['target_workflow_exports'].exists?
+  exports = Rbbt.etc['target_workflow_exports'].read.split("\n")
+  exports.each do |task|
+    wf.export task.to_sym
+  end
+end
+
+if Rbbt.etc['target_workflow_stream_exports'].exists?
+  exports = Rbbt.etc['target_workflow_stream_exports'].read.split("\n")
+  exports.each do |task|
+    wf.export_stream task.to_sym
+  end
+end
+
+if Rbbt.etc['target_workflow_async_exports'].exists?
+  exports = Rbbt.etc['target_workflow_async_exports'].read.split("\n")
+  exports.each do |task|
+    wf.export_asynchronous task.to_sym
+  end
+end
+
+if Rbbt.etc['target_workflow_sync_exports'].exists?
+  exports = Rbbt.etc['target_workflow_sync_exports'].read.split("\n")
+  exports.each do |task|
+    wf.export_synchronous task.to_sym
+  end
+end
+
+if Rbbt.etc['target_workflow_exec_exports'].exists?
+  exports = Rbbt.etc['target_workflow_exec_exports'].read.split("\n")
+  exports.each do |task|
+    wf.export_exec task.to_sym
+  end
+end
+
 #{{{ RUN
 require 'rack'
 use Rack::Deflater

data/test/rbbt/test_workflow.rb

@@ -74,6 +74,15 @@ Returns numer * 2 lines containing TEST
     dependencies.collect{|d| d.load } * ":"
   end
 
+  task :stream => :array do
+    Misc.open_pipe do |sin|
+      5.times do |i|
+        sin.puts "line #{ i }"
+        sleep 1
+      end
+    end
+  end
+
 
 
 end
@@ -152,4 +161,19 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "V:AA", job.run
   end
 
+  def __test_stream
+    io = TestWF.job(:stream).run(:stream)
+    Misc.consume_stream(TSV.get_stream(io), false, STDOUT)
+    nil
+  end
+
+  def __test_fork_stream
+    job = TestWF.job(:stream)
+    job.clean
+    io = job.fork(:stream)
+    Misc.consume_stream(TSV.get_stream(io), false, STDOUT)
+    nil
+  end
+
+
 end

data/test/rbbt/tsv/test_stream.rb

@@ -197,6 +197,22 @@ row2 AAA
     assert_equal ["AA", "AAA"], tsv["row2"][0]
   end
 
+  def test_paste_stream_nohead
+    text1=<<-EOF
+row1\tA
+row2\tAA
+    EOF
+
+    text2=<<-EOF
+row2\tAAA
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :type => :double, :sort => false, :same_fields => true)
+    assert_equal ["AA", "AAA"], tsv["row2"][0]
+  end
+
   def test_flat2double
     text1=<<-EOF
 #: :sep= #:type=:flat

data/test/rbbt/util/misc/test_bgzf.rb

@@ -37,7 +37,7 @@ class TestBgzf < Test::Unit::TestCase
   end
 
   def test_bgzip
-    assert File.exists?(Bgzf.bgzip_cmd)
+    assert File.exist?(Bgzf.bgzip_cmd)
     assert 'bgzip', File.basename(Bgzf.bgzip_cmd)
   end
 end

data/test/rbbt/util/test_misc.rb

@@ -58,7 +58,7 @@ Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium
 doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore
 veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim
 ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia
-consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. 
+consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt.
 
 Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur,
 adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et
@@ -192,7 +192,7 @@ eum fugiat quo voluptas nulla pariatur?"
 
     stream1 = Misc.open_pipe(true) do |sin|
       t.times do |i|
-        sleep sleep_time 
+        sleep sleep_time
         sin.puts "LINE #{ i }"
       end
     end
@@ -228,7 +228,7 @@ eum fugiat quo voluptas nulla pariatur?"
 
     stream1 = Misc.open_pipe(true) do |sin|
       t.times do |i|
-        sleep sleep_time 
+        sleep sleep_time
         sin.puts "LINE #{ i }"
       end
     end
@@ -253,8 +253,8 @@ eum fugiat quo voluptas nulla pariatur?"
         lines1 << line.strip
       end
     end
-    
-    lines2 = [] 
+
+    lines2 = []
     th2 = Thread.new do
       while line = stream5.gets
         lines2 << line.strip
@@ -281,7 +281,7 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal('j', Misc.string2hash("a=b#c=d#:h='j'")[:h])
     assert_equal(:j, Misc.string2hash("a=b#c=d#:h=:j")[:h])
   end
-  
+
   def test_named_array
     a = NamedArray.setup([1,2,3,4], %w(a b c d))
     assert_equal(1, a['a'])
@@ -297,16 +297,16 @@ eum fugiat quo voluptas nulla pariatur?"
 
     hash = {:a => 1}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
- 
+
     hash = {:a => true}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
 
     hash = {:a => :b}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
- 
+
     hash = {:a => /test/}
     assert_equal({}, Misc.string2hash(Misc.hash2string(hash)))
- 
+
   end
 
   def test_merge
@@ -331,8 +331,8 @@ eum fugiat quo voluptas nulla pariatur?"
     TmpFile.with_file do |tmpfile|
       pids = []
       4.times do |i|
-        pids << Process.fork do 
-          status = Misc.lock(tmpfile) do 
+        pids << Process.fork do
+          status = Misc.lock(tmpfile) do
             pid = Process.pid.to_s
             Open.write(tmpfile, pid)
             sleep rand * 1
@@ -419,7 +419,7 @@ eum fugiat quo voluptas nulla pariatur?"
   end
 
   def test_fingerprint
-    assert_equal '{a=>1}', Misc.fingerprint({:a => 1})
+    assert_equal '{:a=>1}', Misc.fingerprint({:a => 1})
   end
 
   def test_tarize
@@ -530,4 +530,21 @@ eum fugiat quo voluptas nulla pariatur?"
     assert Misc.match_value("15", [14, 15, 25])
     assert ! Misc.match_value("15", [14, 25])
   end
+
+  def __test_bench_log
+    Log.severity = 1
+    Misc.benchmark(1000) do
+      Log.info { "Hola" }
+    end
+    Misc.benchmark(1000) do
+      Log.debug { "Hola" }
+    end
+    Misc.benchmark(1000) do
+      Log.debug  "Hola" 
+    end
+    p = Misc.pipe
+    Misc.benchmark(1000) do
+      Log.debug  "Hola #{[p.first.inspect, p.last.inspect] * "=>"}" 
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.20.26
+  version: 5.21.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -222,6 +222,7 @@ files:
 - lib/rbbt/rest/client.rb
 - lib/rbbt/rest/client/adaptor.rb
 - lib/rbbt/rest/client/get.rb
+- lib/rbbt/rest/client/run.rb
 - lib/rbbt/rest/client/step.rb
 - lib/rbbt/tsv.rb
 - lib/rbbt/tsv/accessor.rb
@@ -481,82 +482,82 @@ signing_key:
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/test_workflow.rb
+- test/test_helper.rb
 - test/rbbt/resource/test_path.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_log.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_R.rb
 - test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_association.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/test_tsv.rb
 - test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_doc.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/test_association.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/parallel/test_through.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/parallel/test_through.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_parser.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/test_persist.rb
 - test/rbbt/test_fix_width_table.rb
-- test/test_helper.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb