rbbt-util 5.21.103 → 5.21.104

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14ea1d12f7ac8fe08dc04a5e46bcf7ab10d0a669
-  data.tar.gz: f11141d1270747aafef6293a845e8e62347ed716
+  metadata.gz: c0ffcca0065a417f82428c347e4283ec03ccfd75
+  data.tar.gz: 65a8de0c4d9e0bca0e23a8d0c27e7c2b89d9ca9c
 SHA512:
-  metadata.gz: 7ac4fc17f5c9c56e4b7150247da0ea5cc8ff72417efec635c199548e52ff6230785401f8b712717ec504585f366258e005c6507810c194aa3ced1fba7bbaee28
-  data.tar.gz: 9f7937bf5f62c23c73ede5dc1326fc1ff1a34e98dc9032ed3d600907c3f4e20064e17cef3c6c53eb39a7e0a725e18c10f2ee71836d46519c70cbb859658f1314
+  metadata.gz: 871b8ef7151a8a47f91994061603859b7dfa57402b4631f74eb4f6e5f747524bb48a91ff6e1bef7abe27fb7194b3ac4c6416686c63de4c7531cf052d75b6ad7f
+  data.tar.gz: 1db58bc5d69b28a0d58b6913cce05dae2a113c8a7723fa3bc34f08532a8886b4a2079ebd806292d92d8977c9d09591306ff7840d171473f4ff504a572ddee898

data/lib/rbbt/resource/rake.rb

@@ -29,16 +29,16 @@ module Rake
     Rake::Task.clear
     Rake::FileTask.clear_files
 
-    if block_given?
-      yield
-    else
-      load rakefile
-    end
-
-    raise TaskNotFound if Rake::Task[task].nil?
-
     t = nil
     pid = Process.fork{
+      if block_given?
+        yield
+      else
+        load rakefile
+      end
+
+      raise TaskNotFound if Rake::Task[task].nil?
+
       Misc.pre_fork
       begin
         Misc.in_dir(dir) do

data/lib/rbbt/util/misc/inspect.rb

@@ -220,6 +220,19 @@ module Misc
     end
   end
 
+  def self.txt_digest_str(txt)
+    "digest: " << digest(txt)
+  end
+
+  def self.mtime_str(path)
+    path = path.find if Path === path
+    if File.exists? path
+      "mtime: " << File.mtime(path).to_s
+    else
+      "mtime: not present"
+    end
+  end
+
 
   def self.obj2str(obj)
     _obj = obj
@@ -239,14 +252,14 @@ module Misc
               if obj.directory?
                 "directory: #{obj.glob("**/*")}"
               else
-                "file md5: #{digest(obj.read)}"
+                "file: " << obj << "--" << mtime_str(obj)
               end
             else
               obj + " (file missing)"
             end
           when String
             if obj.length > HASH2MD5_MAX_STRING_LENGTH
-              sample_large_obj(obj, HASH2MD5_MAX_STRING_LENGTH)
+              sample_large_obj(obj, HASH2MD5_MAX_STRING_LENGTH) << "--" << txt_digest_str(obj)
             else
               obj
             end
@@ -260,15 +273,15 @@ module Misc
             remove_long_items(obj)
           when File 
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << ">"
+              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
             else
-              "<IO:" << obj.path << ">"
+              "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
           when (defined? Step and Step)
             "<IO:" << obj.path << ">"
           else
             if obj.respond_to? :filename and obj.filename
-              "<IO:" << obj.filename << ">"
+              "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
             else
               obj_ins = obj.inspect
               if obj_ins =~ /:0x0/

data/lib/rbbt/workflow/accessor.rb

@@ -412,6 +412,10 @@ class Step
     started? && ! (done? || error? || aborted?) && ! running?
   end
 
+  def missing?
+    status == :done && ! Open.exists?(path)
+  end
+
   def error?
     status == :error
   end
@@ -808,6 +812,7 @@ module Workflow
                      _inputs = IndiferentHash.setup(_inputs.dup)
                      dep = dependency.call jobname, _inputs, real_dependencies
                      if Hash === dep
+                       dep[:workflow] ||= wf  || self
                        task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
                        inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
                        dep = dep[:workflow].job(dep[:task], dep[:jobname], inputs)

data/lib/rbbt/workflow/step/dependencies.rb

@@ -124,7 +124,7 @@ class Step
         return
       end
 
-      if dependency.aborted?
+      if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or dependency.missing?
         log_dependency_exec(dependency, "aborted (clean)")
         dependency.clean
         raise TryAgain

data/lib/rbbt/workflow/step/run.rb

@@ -140,7 +140,7 @@ class Step
             :issued => (issue_time = Time.now),
             :name => name,
             :clean_name => clean_name,
-            :workflow => @task.workflow.to_s,
+            :workflow => (@workflow || @task.workflow).to_s,
             :task_name => @task.name,
             :result_type => @task.result_type,
             :result_description => @task.result_description

data/lib/rbbt/workflow/step.rb

@@ -5,7 +5,7 @@ require 'rbbt/util/semaphore'
 require 'rbbt/workflow/accessor'
 
 class Step
-  attr_accessor :clean_name, :path, :task, :inputs, :dependencies, :bindings
+  attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
   attr_accessor :pid
   attr_accessor :exec
   attr_accessor :result, :mutex, :seen
@@ -50,7 +50,7 @@ class Step
   end
 
   def workflow
-    info[:workflow]
+    @workflow || info[:workflow]
   end
 
   def load_inputs_from_info

data/lib/rbbt/workflow.rb

@@ -307,8 +307,9 @@ module Workflow
 
     Workflow.resolve_locals(inputs)
 
-    task_inputs = task_info(taskname)[:inputs]
-    defaults = IndiferentHash.setup(task.input_defaults)
+    task_info = task_info(taskname)
+    task_inputs = task_info[:inputs]
+    defaults = IndiferentHash.setup(task_info[:input_defaults]).merge(task.input_defaults)
 
     missing_inputs = []
     task.required_inputs.each do |input|
@@ -346,6 +347,7 @@ module Workflow
     end
 
     job = get_job_step step_path, task, input_values, dependencies
+    job.workflow = self
     job.clean_name = jobname
     job
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.103
+  version: 5.21.104
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-09 00:00:00.000000000 Z
+date: 2017-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -503,88 +503,88 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/test_entity.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/test_step.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_pipes.rb
-- test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_tmpfile.rb
+- test/test_helper.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/test_association.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_traverse.rb
 - test/rbbt/util/test_R.rb
-- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_tmpfile.rb
+- test/rbbt/util/test_filecache.rb
 - test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/test_open.rb
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_chain_methods.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_attach.rb
-- test/rbbt/tsv/test_filter.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_parser.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/test_excel.rb
 - test/rbbt/tsv/parallel/test_through.rb
 - test/rbbt/tsv/parallel/test_traverse.rb
-- test/rbbt/tsv/test_stream.rb
-- test/rbbt/test_association.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_excel.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_entity.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
 - test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
 - test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
 - test/rbbt/persist/test_tsv.rb
 - test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
 - test/rbbt/test_fix_width_table.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/test_helper.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/resource/test_path.rb