rbbt-util 5.21.7 → 5.21.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5b2f2a8df5296854ccc19614f6bdbac8ecf21e0
-  data.tar.gz: 82a36de680c7c5fd7a400ac23fb6bfe32f038b91
+  metadata.gz: 215819a1374483cea2057ae08343d6f03b71d41f
+  data.tar.gz: a22977fcc3e07a9eac6f544d77a61be31caddc9c
 SHA512:
-  metadata.gz: 10aab8f0aa9408a127e360a62d21cf9de4044fa49b5cd94d02df36c52c636585e7b8ac058970d2d0e4b5ee01448d8763b617699564c34b2d7c0bf4d16b98e2d4
-  data.tar.gz: f01033bcd9f5d8cf68620b95b8b0ae06450fd191e884041716f6dcfac7cc7a8da73619df8421b20bb22309378c7d0bc8354f0e424ba84153bf71c13886ffa916
+  metadata.gz: 3f74b31e23ec63d0f47998a144c0149109804a3421b6f35562769d1488f5fe78b7f8469933a2a8f207c9106b1a8b8daff0ecdd8b81ff6d21eece27d734ab6734
+  data.tar.gz: 833a891002ef13c8bd6e8655475843af420130b621ba15113e5cddb466f9118eb93e7b0628c2a2a0c97750e24ee37ad25f2a4c851b8618958f691b312420aaa0

data/lib/rbbt/resource/path.rb

@@ -96,7 +96,7 @@ module Path
     rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil 
     key_elems = [where, caller_lib, rsearch_paths, paths]
     key = Misc.digest(key_elems.inspect)
-    self.sub!('~/', Etc.getpwuid.dir + '/')
+    self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
     @path[key] ||= begin
                      paths = [paths, rsearch_paths, self.search_paths, SEARCH_PATHS].reverse.compact.inject({}){|acc,h| acc.merge! h; acc }
                      where = paths[:default] if where == :default

data/lib/rbbt/rest/client/get.rb

@@ -1,4 +1,13 @@
 class WorkflowRESTClient
+  def self.encode(url)
+    begin
+      URI.encode(url)
+    rescue
+      Log.warn $!.message
+      url
+    end
+  end
+
   def self.fix_hash(hash, fix_values = false)
     fixed = {}
     hash.each do |key, value|
@@ -59,7 +68,7 @@ class WorkflowRESTClient
     res = capture_exception do
       Misc.insist(2, 0.5) do
         Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
-        res = RestClient.get(URI.encode(url), :params => params)
+        res = RestClient.get(self.encode(url), :params => params)
         raise TryAgain if res.code == 202
         res
       end
@@ -73,7 +82,8 @@ class WorkflowRESTClient
     res = capture_exception do
       Misc.insist(2, 0.5) do
         Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
-        res = RestClient.get(URI.encode(url), :params => params)
+        raise "No url" if url.nil?
+        res = RestClient.get(self.encode(url), :params => params)
         raise TryAgain if res.code == 202
         res
       end
@@ -88,7 +98,7 @@ class WorkflowRESTClient
 
     res = capture_exception do
       Misc.insist(2, 0.5) do
-        RestClient.get(URI.encode(url), :params => params)
+        RestClient.get(self.encode(url), :params => params)
       end
     end
 
@@ -106,7 +116,7 @@ class WorkflowRESTClient
 
     WorkflowRESTClient.__prepare_inputs_for_restclient(params)
     name = capture_exception do
-      RestClient.post(URI.encode(url), params)
+      RestClient.post(self.encode(url), params)
     end
 
     Log.debug{ "RestClient jobname returned for #{ url } - #{Misc.fingerprint params}: #{name}" }
@@ -122,7 +132,7 @@ class WorkflowRESTClient
       params = fix_params params
 
       res = capture_exception do
-        RestClient.post(URI.encode(url), params)
+        RestClient.post(self.encode(url), params)
       end
 
       begin

data/lib/rbbt/rest/client/step.rb

@@ -204,7 +204,7 @@ class WorkflowRESTClient
     end
 
 
-    def fork
+    def fork(noload=false, semaphore=nil)
       init_job(:asynchronous)
     end
 
@@ -233,6 +233,7 @@ class WorkflowRESTClient
     end
 
     def join
+      init_job unless @url
       Log.debug{ "Joining RestClient: #{path}" }
       if IO === @result
         res = @result
@@ -258,6 +259,7 @@ class WorkflowRESTClient
       params = params.merge(:_format => [:string, :boolean, :tsv, :annotations,:array].include?(result_type.to_sym) ? :raw : :json )
       Misc.insist 3, rand(2) + 1 do
         begin
+          init_job if url.nil?
           WorkflowRESTClient.get_raw(url, params)
         rescue
           Log.exception $!
@@ -291,7 +293,11 @@ class WorkflowRESTClient
         (stream ? res.read : res).split("\n")
         res.split("\n")
       else
-        JSON.parse res
+        if IO === res
+          JSON.parse res.read
+        else
+          JSON.parse res
+        end
       end
     end
 

data/lib/rbbt/tsv/util.rb

@@ -133,6 +133,8 @@ module TSV
           sin.puts l
         end
       end
+    when Set
+      get_stream(file.to_a, open_options)
     else
       raise "Cannot get stream from: #{file.inspect}"
     end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -94,7 +94,7 @@ module ConcurrentStream
       @pids.each do |pid| 
         begin
           Process.waitpid(pid, Process::WUNTRACED)
-          raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success? or no_fail
+          raise ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
       end 

data/lib/rbbt/workflow.rb

@@ -326,10 +326,11 @@ module Workflow
 
     inputs.each do |k,v|
       default = defaults[k]
-      if (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s)) and 
-        (defaults[k].to_s != v.to_s and not (FalseClass === v and defaults[k].nil?))
-        real_inputs[k] = v 
-      end
+      next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
+      next if default == v 
+      next if (String === default and Symbol === v and v.to_s == default)
+      next if (Symbol === default and String === v and v == default.to_s)
+      real_inputs[k] = v 
     end
 
     if real_inputs.empty?

data/lib/rbbt/workflow/accessor.rb

@@ -373,13 +373,17 @@ class Step
     end
   end
 
+  def stalled?
+    ! (done? || error? || aborted?) && ! running?
+  end
+
   def error?
     status == :error
   end
 
   def nopid?
     pid = info[:pid]
-    pid.nil? && ! (status == :aborted || status == :done || status == :error)
+    pid.nil? && ! (status.nil? || status.nil? || status == :aborted || status == :done || status == :error)
   end
 
   def aborted?
@@ -542,46 +546,41 @@ module Workflow
                                 else
                                   all_deps << dep unless Proc === dep
                                 end
-                                case dep
-                                when Array
-                                  wf, t, o = dep
-
-                                  wf.rec_dependencies(t).each do |d|
-                                    if Array === d
-                                      new = d.dup
-                                    else
-                                      new = [dep.first, d]
-                                    end
 
-                                    if Hash === o and not o.empty? 
-                                      if Hash === new.last
-                                        hash = new.last.dup
-                                        o.each{|k,v| hash[k] ||= v}
-                                        new[new.length-1] = hash
+                                begin
+                                  case dep
+                                  when Array
+                                    wf, t, o = dep
+
+                                    wf.rec_dependencies(t).each do |d|
+                                      if Array === d
+                                        new = d.dup
                                       else
-                                        new.push o.dup
+                                        new = [dep.first, d]
                                       end
-                                    end
 
-                                    all_deps << new
-                                  end
+                                      if Hash === o and not o.empty? 
+                                        if Hash === new.last
+                                          hash = new.last.dup
+                                          o.each{|k,v| hash[k] ||= v}
+                                          new[new.length-1] = hash
+                                        else
+                                          new.push o.dup
+                                        end
+                                      end
 
-                                when String, Symbol
-                                  rec_deps = rec_dependencies(dep.to_sym)
-                                  all_deps.concat rec_deps
-                                when DependencyBlock
-                                  all_deps << dep.dependency if dep.dependency
-                                  case dep.dependency
-                                  when Array
-                                    dep_wf, dep_task, dep_options = dep.dependency
-                                    if dep_task === Symbol
-                                      dep_rec_dependencies = dep_wf.rec_dependencies(dep_task.to_sym)
-                                      dep_rec_dependencies.collect!{|d| Array === d ? d : [dep_wf, d]}
-                                      all_deps.concat dep_rec_dependencies
+                                      all_deps << new
                                     end
-                                  when Symbol, String
-                                    all_deps.concat rec_dependencies(dep.dependency.to_sym)
+
+                                  when String, Symbol
+                                    rec_deps = rec_dependencies(dep.to_sym)
+                                    all_deps.concat rec_deps
+                                  when DependencyBlock
+                                    dep = dep.dependency
+                                    raise TryAgain
                                   end
+                                rescue TryAgain
+                                  retry
                                 end
                               end
                               all_deps.uniq
@@ -706,7 +705,7 @@ module Workflow
               end
             else
               input_options = workflow.task_info(dep_task)[:input_options][i] || {}
-              if input_options[:stream]
+              if input_options[:stream] or true
                 #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
                 _inputs[i] = rec_dependency
               else

data/lib/rbbt/workflow/step/dependencies.rb

@@ -76,10 +76,11 @@ class Step
     return if job.done? && ! job.dirty?
 
     status = job.status.to_s
-    if defined? WorkflowRESTClient && WorkflowRESTClient::RemoteStep === job 
-      return if ! (status == 'done' and status == 'error' and status == 'aborted')
+
+    if defined?(WorkflowRESTClient) && WorkflowRESTClient::RemoteStep === job 
+      return unless (status == 'done' or status == 'error' or status == 'aborted')
     else
-      return if status == 'streaming' && job.running?
+      return if status == 'streaming' and job.running?
     end
 
     if (status == 'error' || job.aborted?) && job.recoverable_error?

data/lib/rbbt/workflow/step/run.rb

@@ -243,7 +243,8 @@ class Step
   def produce(force=false, dofork=false)
     return self if done? and not dirty?
 
-    if error? or aborted?
+    if error? or aborted? or stalled?
+      abort if stalled?
       if force or aborted? or recoverable_error?
         clean
       else
@@ -253,13 +254,12 @@ class Step
 
     clean if dirty? or (not running? and not done?)
 
-    no_load = :stream
     if dofork
       fork(true) unless started?
 
       join unless done?
     else
-      run(false) unless started?
+      run(true) unless started?
 
       join unless done?
     end
@@ -478,6 +478,7 @@ class Step
       self
     ensure
       set_info :joined, true
+      @result = nil
     end
   end
 end

data/lib/rbbt/workflow/task.rb

@@ -84,12 +84,14 @@ module Task
         task_name, wf = wf, workflow if task_name.nil? and Symbol === wf or String === wf
         next if task_name.nil?
         task = wf.tasks[task_name.to_sym]
-      else 
+      else
         next
       end
+
       maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
       raise "Dependency task not found: #{dep}" if task.nil?
       next if seen.include? [wf, task.name]
+
       seen << [wf, task.name]
       new_inputs = task.inputs - maps
       next unless new_inputs.any?

data/share/rbbt_commands/resource/produce

@@ -5,7 +5,7 @@ require 'rbbt/resource'
 require 'rbbt/workflow'
 
 options = SOPT.get <<EOF
--w--workflows* Workflows to use; 'all' for all in Rbbt.etc.workflows:
+-W--workflows* Workflows to use; 'all' for all in Rbbt.etc.workflows:
 -r--requires* Files to require; 'all' for all in Rbbt.etc.requires:
 -f--force Force the production if the file is already present
 -h--help Help

data/share/rbbt_commands/tsv/info

@@ -1,4 +1,4 @@
-#"!/usr/bin/env ruby
+#!/usr/bin/env ruby
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'

data/share/rbbt_commands/tsv/query

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Query a TSV file 
+
+$ rbbt tsv query [options] <file.tsv> <key>
+
+Display summary information for a TSV entry. Works with Tokyocabinet HDB and BDB.
+
+-tch--tokyocabinet File is a TC HDB
+-tcb--tokyocabinet_bd File is a TC BDB
+-t--type* Type of tsv (single, list, double, flat)
+-hh--header_hash* Change the character used to mark the header line (defaults to #)
+-k--key_field* Change the key field
+-f--field* Change the fields to display
+-s--sep* Change the fields separator (default TAB)
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file, key = ARGV
+
+file = STDIN if file == '-'
+
+raise ParameterException, "Please specify the tsv file as argument" if file.nil?
+
+options[:fields] = options[:fields].split(/[,\|]/) if options[:fields]
+options[:header_hash] = options["header_hash"]
+options[:sep] = options["sep"]
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+else
+  tsv = TSV.open(file, options)
+end
+
+values = tsv[key]
+
+head = "#{Log.color :magenta, tsv.key_field}: " << Log.color(:yellow, key)
+puts head
+puts (["-"] * Log.uncolor(head).length) * ""
+values.zip(tsv.fields) do |value,field|
+  if Array === value
+    values = value.collect{|v| Log.color(:yellow, v)} * ", "
+    puts "#{Log.color :magenta, field}: " << values
+  else
+    puts "#{Log.color :magenta, field}: " << Log.color(:yellow, value)
+  end
+end
+

data/share/rbbt_commands/workflow/init

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+options = SOPT.setup <<EOF
+
+Init a new workflow scaffold
+
+$ rbbt workflow init <workflow>
+EOF
+
+workflow = ARGV.shift
+if workflow.nil?
+  usage
+  puts
+  puts Log.color :magenta, "## Error"
+  puts
+  puts "No workflow name specified."
+  puts
+  exit -1
+end
+
+workflow_path = Path.setup(workflow) unless Path === workflow
+lib_path = workflow_path + '/lib'
+
+if Dir.exists?(workflow_path)
+  usage
+  puts
+  puts Log.color :magenta, "## Error"
+  puts
+  puts "The workflow '#{workflow}' already exists!"
+  puts
+  exit -1
+end
+
+template = <<-EOF
+require 'rbbt/workflow'
+
+module #{workflow}
+  extend Workflow
+
+  desc "Scaffold task"
+  task :scaffold_task => :string do
+    "Scaffold task"
+  end
+
+end
+EOF
+
+Dir.mkdir(workflow_path)
+workflow_file = workflow_path + '/workflow.rb'
+File.write(workflow_file, template)
+
+Dir.mkdir(lib_path)
+lib_file = lib_path + '/.keep'
+File.write(lib_file, '')

data/share/rbbt_commands/workflow/server

@@ -19,6 +19,7 @@ $ rbbt workflow server [options] <Workflow>
 -RS--Rserve_session* Rserve session to use, otherwise start new one
 -wd--workdir* Change the working directory of the workflow
 -W--workflows* List of additional workflows to load
+-R--requires* Require a list of files
 --views* Directory with view templates
 --stream Activate streaming of workflow tasks
 --export* Export workflow tasks (asynchronous)
@@ -49,6 +50,7 @@ options[:Bind] ||= "0.0.0.0"
 
 workflow = ARGV.shift
 workflows = options[:workflows] || "" 
+requires = options[:requires] || "" 
 
 workflow = File.expand_path(workflow) if File.exist?(workflow)
 
@@ -73,7 +75,8 @@ TmpFile.with_file do |app_dir|
     Open.write(app_dir.etc.target_workflow_sync_exports.find, sync_exports * "\n") if sync_exports
     Open.write(app_dir.etc.target_workflow_exec_exports.find, exec_exports * "\n") if exec_exports
 
-    Open.write(app_dir.etc.workflows.find, workflows.split(/,\s*/)*"\n") if workflows 
+    Open.write(app_dir.etc.workflows.find, workflows.split(/,\s*/)*"\n") if workflows and not workflows.empty?
+    Open.write(app_dir.etc.requires.find, requires.split(/,\s*/)*"\n") if requires and not requires.empty? 
 
     require 'rack'
     ENV["RBBT_FINDER"] = "true" if options.include?(:finder)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.7
+  version: 5.21.8
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2016-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -357,6 +357,7 @@ files:
 - share/rbbt_commands/tsv/head
 - share/rbbt_commands/tsv/info
 - share/rbbt_commands/tsv/json
+- share/rbbt_commands/tsv/query
 - share/rbbt_commands/tsv/read
 - share/rbbt_commands/tsv/slice
 - share/rbbt_commands/tsv/sort
@@ -367,6 +368,7 @@ files:
 - share/rbbt_commands/workflow/cmd
 - share/rbbt_commands/workflow/example
 - share/rbbt_commands/workflow/info
+- share/rbbt_commands/workflow/init
 - share/rbbt_commands/workflow/install
 - share/rbbt_commands/workflow/jobs
 - share/rbbt_commands/workflow/knowledge_base