rbbt-util 5.36.0 → 5.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d105d6c21ad018f28acf6a234feae8d81f327f3f94f228c01f118abed6c386f6
-  data.tar.gz: 5c883e0611f02203d394ec02e448664efdc42361be04e9d873e9ba3f5e6cc92e
+  metadata.gz: 281f115df5158fada081c65b87e292e69e4f378b8353d6e2542b3c45db0c03f1
+  data.tar.gz: e9801707391c731a2a4b4ead8156cff82fdf30c4e769a8bed34d1d77185b447a
 SHA512:
-  metadata.gz: 20a4bccdeb0c587041b7f20f09bda19949713fcb362911a249b08112399c09178f8e50b5ae90f390bd6cd1389b3c9a47b7374c4fc05a83b073f1d43bdfdc6611
-  data.tar.gz: 1b3b112106b4ac1e261bcf1630ecaeea0ed27f1975aeead9694708241e9268fdac36da4f5b4be25c12288cb7f97b8b4163815b8d9010aba40f6f2434e654f99c
+  metadata.gz: 3da2876b28c245b27f32b6dbd00188bdeea498fb118ccab048fb5d72925548e5e89926dea6b6316573e9d95a570c07983109e0e09de92360ec89ab2aec088439
+  data.tar.gz: 0166d47c3198bfc4c636a1f47ac9be1d2a64b8d97a2836166b02d1ac17d46c2b835ad210953da586f15a98f8593e18f6010ebc07220e95da149dcabec44b33a9

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -28,6 +28,7 @@ class RbbtProcessQueue
 
 
     def dump(obj, stream)
+      obj.concurrent_stream = nil if obj.respond_to?(:concurrent_stream)
       case obj
       when Annotated
         payload = @serializer.dump(obj)

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -65,11 +65,7 @@ class RbbtProcessQueue
           Log.high "Worker #{Process.pid} leaving"
         rescue Exception
           Log.high "Worker #{Process.pid} had exception: #{$!.message}"
-          Log.exception $!
-          begin
-            @callback_queue.push($!) if @callback_queue
-          rescue
-          end
+          @callback_queue.push($!) if @callback_queue
           Kernel.exit! -1
         ensure
           @callback_queue.close_write if @callback_queue 

data/lib/rbbt/util/concurrency/processes.rb

@@ -180,13 +180,15 @@ class RbbtProcessQueue
         rescue Exception
           Log.low "Process monitor exception [#{Process.pid}]: #{$!.message}"
           processes.each{|p| p.abort_and_join}
-          Log.low "Processes aborted #{Process.pid}"
+          Log.low "Processes aborted for monitor #{Process.pid}"
           processes.clear
 
+          @manager_thread.report_on_exception = false
           @manager_thread.raise $! if @manager_thread.alive?
           raise Aborted, "Aborted monitor thread with exception"
         end
       end
+      @monitor_thread.report_on_exception = false
 
       RbbtSemaphore.post_semaphore(@sem)
 
@@ -215,33 +217,36 @@ class RbbtProcessQueue
     init_master
 
     RbbtSemaphore.synchronize(@sem) do
-    @callback_thread = Thread.new do
-      begin
-        loop do
-          p = @callback_queue.pop unless @callback_queue.cleaned
+      @callback_thread = Thread.new do
+        begin
+          loop do
+            p = @callback_queue.pop unless @callback_queue.cleaned
 
-          if Exception === p or (Array === p and Exception === p.first)
-            e = Array === p ? p.first : p
-            Log.low "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
-            raise e 
-          end
+            if Exception === p or (Array === p and Exception === p.first)
+              e = Array === p ? p.first : p
+              Log.low "Callback recieved exception from worker: #{e.message}" unless Aborted === e or ClosedStream === e
+              raise e 
+            end
 
-          if @callback.arity == 0
-            @callback.call
-          else
-            @callback.call p
+            if @callback.arity == 0
+              @callback.call
+            else
+              @callback.call p
+            end
           end
+        rescue ClosedStream
+          Log.low "Callback thread closing"
+        rescue Aborted
+          Log.low "Callback thread aborted"
+          raise $!
+        rescue Exception
+          Log.low "Exception captured in callback: #{$!.message}"
+          raise $!
         end
-      rescue ClosedStream
-        Log.low "Callback thread closing"
-      rescue Aborted
-        Log.low "Callback thread aborted"
-        raise $!
-      rescue Exception
-        Log.low "Exception captured in callback: #{$!.message}"
-        raise $!
-      end
-    end if @callback_queue
+      end if @callback_queue
+
+      @callback_thread.report_on_exception = false
+      
     end
 
   end

data/lib/rbbt/util/misc/inspect.rb

@@ -343,9 +343,9 @@ module Misc
             end
           when Array
             if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
-              "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
+              "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v.nil? ? "" : v) } * "," << "]"
             else
-              "[" << obj.collect{|v| obj2str(v) } * "," << "]"
+              "[" << obj.collect{|v| obj2str(v.nil? ? "" : v) } * "," << "]"
             end
           when TSV::Parser
             remove_long_items(obj)

data/lib/rbbt/workflow/dependencies.rb

@@ -72,6 +72,7 @@ module Workflow
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
 
+    unlocated = unlocated_override?(dep)
     dep = Workflow.load_step(dep) if not Step === dep
 
     dep.original_workflow ||= dep.workflow if dep.workflow
@@ -91,7 +92,7 @@ module Workflow
     end
 
     dep.task_name = task_name
-    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name 
+    dep.overriden = dep.original_task_name.to_sym if dep.original_task_name && dep.original_task_name.to_s != task_name.to_s || ! unlocated
 
     dep.extend step_module
 
@@ -169,6 +170,16 @@ module Workflow
                    else
 
                      compute = options[:compute] if options
+                     if options && options[:canfail]
+                       compute = case compute
+                                 when nil
+                                   :canfail
+                                 when Array
+                                   compute + [:canfail]
+                                 else
+                                   [compute, :canfail]
+                                 end
+                     end
 
                      all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
 
@@ -207,6 +218,16 @@ module Workflow
 
                        options = {} if options.nil?
                        compute = options[:compute]
+                       if options[:canfail]
+                         compute = case compute
+                                   when nil
+                                     :canfail
+                                   when Array
+                                     compute + [:canfail]
+                                   else
+                                     [compute, :canfail]
+                                   end
+                       end
 
                        options = IndiferentHash.setup(options.dup)
                        dep = dependency.call jobname, _inputs.merge(options), real_dependencies
@@ -228,7 +249,8 @@ module Workflow
                                  task_info = d[:workflow].task_info(d[:task])
 
                                  _inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                                 job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                                 _jobname = d.include?(:jobname) ? d[:jobname] : jobname
+                                 job = d[:workflow]._job(d[:task], _jobname, _inputs) 
                                  overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
                                  job
                                end
@@ -250,8 +272,8 @@ module Workflow
                        else
                          task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
                          _inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         job = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
-                         job = d[:workflow]._job(d[:task], d[:jobname], _inputs) 
+                         _jobname = dep.include?(:jobname) ? dep[:jobname] : jobname
+                         job = dep[:workflow]._job(dep[:task], _jobname, _inputs)
                          overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
                          job
                        end

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -24,8 +24,8 @@ class RemoteWorkflow
       workflow, task, job, *rest = path.split("/")
 
       workflow_name = begin
-                        workflow = Kernel.const_get(workflow) if String === workflow
-                        workflow.respond_to?(:complete_name) ? workflow.complete_name : workflow
+                        wf = Kernel.const_get(workflow) if String === workflow && ! workflow.empty?
+                        wf.respond_to?(:complete_name) ? (wf.complete_name || workflow) : workflow
                       rescue
                         workflow
                       end
@@ -188,9 +188,10 @@ job.clean
           #   Open.write(file, new)
           # end
 
-          paths = Dir.glob(File.join(dir, "*.as_step")).collect{|f| Open.read(f).strip }
+          files = Dir.glob(File.join(dir, "*.as_step"))
+          paths = files.collect{|f| Open.read(f).strip }
           new = Step.migrate(paths, :user, :target => server)
-          paths.zip(new).each{|file,new| Open.write(file, new) }
+          files.zip(new).each{|file,new| Open.write(file, new) }
 
           CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
@@ -229,10 +230,11 @@ job.clean
       job.input_dependencies.each do |dep|
         Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
         dep.produce
-      end if produce_dependencies
+      end 
 
       migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
-      Log.medium "Migrating #{migrate_dependencies.length} dependencies to #{ server }" 
+      migrate_dependencies += job.input_dependencies.select{|d| d.done? }.collect{|d| d.path }
+      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{job.path} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -52,6 +52,7 @@ class RemoteStep
     end
     
     def _run
+      RemoteWorkflow::SSH.upload_dependencies(self, @server)
       RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
     end
 
@@ -97,6 +98,18 @@ class RemoteStep
       _restart
     end
 
+    def abort
+      Log.warn "not implemented RemoteWorkflow::SSH.abort(@url, @input_id, @base_name)"
+    end
+
+    def input_dependencies
+      @input_dependencies ||= inputs.values.flatten.
+        select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
+        inputs.values.flatten.
+        select{|dep| Path === dep && Step === dep.resource }.
+        select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+        collect{|dep| dep.resource }
+    end
   end
 end
 

data/lib/rbbt/workflow/step/dependencies.rb

@@ -145,11 +145,12 @@ class Step
   end
 
   def input_dependencies
-    @input_dependencies ||= (recursive_inputs.flatten.select{|i| Step === i } + 
-                             recursive_inputs.flatten.
-                              select{|dep| Path === dep && Step === dep.resource }.
-                              select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
-                              collect{|dep| dep.resource })
+    @input_dependencies ||= recursive_inputs.flatten.
+      select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
+      recursive_inputs.flatten.
+      select{|dep| Path === dep && Step === dep.resource }.
+      select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
+      collect{|dep| dep.resource }
   end
 
   def execute_dependency(dependency, log = true)
@@ -260,6 +261,9 @@ class Step
           dep.produce
         rescue RbbtException
           Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last if dep.messages}"
+        rescue Exception
+          Log.warn "Not Allowing failing of #{dep.path} because #{$!.class} not RbbtException"
+          raise $!
         end
         nil
       end
@@ -447,13 +451,15 @@ class Step
     compute_simple_dependencies = {}
     compute_last_deps = {}
     seen_paths = Set.new
-    rec_dependencies.uniq.each do |step| 
+    rec_dependencies.uniq.reverse.each do |step| 
       next if seen_paths.include? step.path
       seen_paths << step.path
       next unless required_dep_paths.include? step.path
       required_seen_paths = seen_paths & required_dep_paths
 
-      internal = step.inputs.select{|i| Step == i && required_paths.include?(i.path) && seen_paths.include?(i.path) }
+      inputs = step.inputs
+      inputs = inputs.values if Hash === inputs
+      internal = inputs.select{|i| i.respond_to?(:path) && required_seen_paths.include?(i.path) }.any?
 
       if ComputeDependency === step 
         next if produced.include? step.path
@@ -466,9 +472,9 @@ class Step
         end
       else
         if internal
-          simple_dependencies.prepend(step)
-        else
           simple_dependencies << step
+        else
+          simple_dependencies.prepend(step)
         end
       end
     end

data/lib/rbbt/workflow/step/run.rb

@@ -98,8 +98,10 @@ class Step
       old = Signal.trap("INT"){ Thread.current.raise Aborted }
       if @task.respond_to?(:exec_in)
         @task.exec_in((bindings || self), *@inputs)
-      else
+      elsif @task
         (bindings || self).instance_exec *@inputs, &@task
+      else
+        raise DependencyError, "Dependency #{self.path} cannot be produced"
       end
     ensure
       Signal.trap("INT", old)

data/lib/rbbt/workflow.rb

@@ -490,8 +490,10 @@ module Workflow
       end
     end
 
+    overriden = true if dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
+
     input_values = task.take_input_values(inputs)
-    if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden 
+    if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden #&& ! dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
       step_path = step_path taskname, jobname, [], [], extension
     else
       step_path = step_path taskname, jobname, input_values, dependencies, extension

data/share/rbbt_commands/workflow/task

@@ -428,15 +428,19 @@ begin
 
   def match_dependencies(queries, dependencies)
     queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
-    dependencies.select do |dep|
-      queries.select do |q|
+
+    matched = []
+    queries.each do |q|
+      matched += dependencies.select do |dep|
         if Array === q
           q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
         else
           q.to_s == dep.task_name.to_s
         end
-      end.any?
+      end
     end
+
+    matched
   end
 
   if server = options.delete(:relay)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.36.0
+  version: 5.37.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-03-02 00:00:00.000000000 Z
+date: 2023-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: progress-monitor
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: lockfile
   requirement: !ruby/object:Gem::Requirement
@@ -52,34 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: spreadsheet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubyXL
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: highline
   requirement: !ruby/object:Gem::Requirement
@@ -122,20 +80,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: to_regexp
   requirement: !ruby/object:Gem::Requirement
@@ -613,113 +557,113 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_association.rb
-- test/rbbt/hpc/test_orchestrate.rb
-- test/rbbt/hpc/test_slurm.rb
-- test/rbbt/hpc/orchestrate/test_batches.rb
-- test/rbbt/hpc/orchestrate/test_rules.rb
-- test/rbbt/hpc/orchestrate/test_chains.rb
-- test/rbbt/hpc/test_batch.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/test_packed_index.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_fix_width_table.rb
 - test/rbbt/workflow/test_remote_workflow.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/step/test_save_load_inputs.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
 - test/rbbt/workflow/util/test_data.rb
-- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/util/test_migrate.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/test_procpath.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_concurrency.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/python/test_util.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/test_python.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_development.rb
+- test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_serialize.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_communication.rb
-- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
+- test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_R.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/test_python.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_migrate.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_marshal.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
-- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_marshal.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_manipulate.rb
 - test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/orchestrate/test_chains.rb
+- test/rbbt/hpc/orchestrate/test_rules.rb
+- test/rbbt/hpc/orchestrate/test_batches.rb
+- test/rbbt/hpc/test_slurm.rb
+- test/rbbt/hpc/test_orchestrate.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/test_helper.rb