scout-gear 7.3.0 → 8.0.0

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 7.3.0 ruby lib
+# stub: scout-gear 8.0.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "7.3.0"
+  s.version = "8.0.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-05-12"
+  s.date = "2023-05-23"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -45,6 +45,7 @@ Gem::Specification.new do |s|
     "lib/scout/log/progress.rb",
     "lib/scout/log/progress/report.rb",
     "lib/scout/log/progress/util.rb",
+    "lib/scout/log/trap.rb",
     "lib/scout/meta_extension.rb",
     "lib/scout/misc.rb",
     "lib/scout/misc/digest.rb",
@@ -57,6 +58,7 @@ Gem::Specification.new do |s|
     "lib/scout/named_array.rb",
     "lib/scout/open.rb",
     "lib/scout/open/lock.rb",
+    "lib/scout/open/lock/lockfile.rb",
     "lib/scout/open/remote.rb",
     "lib/scout/open/stream.rb",
     "lib/scout/open/util.rb",
@@ -84,6 +86,8 @@ Gem::Specification.new do |s|
     "lib/scout/simple_opt/setup.rb",
     "lib/scout/tmpfile.rb",
     "lib/scout/tsv.rb",
+    "lib/scout/tsv/attach.rb",
+    "lib/scout/tsv/change_id.rb",
     "lib/scout/tsv/dumper.rb",
     "lib/scout/tsv/index.rb",
     "lib/scout/tsv/open.rb",
@@ -94,11 +98,14 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/persist/fix_width_table.rb",
     "lib/scout/tsv/persist/serialize.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
+    "lib/scout/tsv/transformer.rb",
     "lib/scout/tsv/traverse.rb",
     "lib/scout/tsv/util.rb",
     "lib/scout/tsv/util/filter.rb",
     "lib/scout/tsv/util/process.rb",
+    "lib/scout/tsv/util/reorder.rb",
     "lib/scout/tsv/util/select.rb",
+    "lib/scout/tsv/util/unzip.rb",
     "lib/scout/work_queue.rb",
     "lib/scout/work_queue/socket.rb",
     "lib/scout/work_queue/worker.rb",
@@ -113,6 +120,7 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/step/load.rb",
     "lib/scout/workflow/step/progress.rb",
     "lib/scout/workflow/step/provenance.rb",
+    "lib/scout/workflow/step/status.rb",
     "lib/scout/workflow/task.rb",
     "lib/scout/workflow/task/inputs.rb",
     "lib/scout/workflow/usage.rb",
@@ -175,22 +183,28 @@ Gem::Specification.new do |s|
     "test/scout/tsv/persist/test_adapter.rb",
     "test/scout/tsv/persist/test_fix_width_table.rb",
     "test/scout/tsv/persist/test_tokyocabinet.rb",
+    "test/scout/tsv/test_attach.rb",
+    "test/scout/tsv/test_change_id.rb",
     "test/scout/tsv/test_dumper.rb",
     "test/scout/tsv/test_index.rb",
     "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
     "test/scout/tsv/test_persist.rb",
+    "test/scout/tsv/test_transformer.rb",
     "test/scout/tsv/test_traverse.rb",
     "test/scout/tsv/test_util.rb",
     "test/scout/tsv/util/test_filter.rb",
     "test/scout/tsv/util/test_process.rb",
+    "test/scout/tsv/util/test_reorder.rb",
     "test/scout/tsv/util/test_select.rb",
+    "test/scout/tsv/util/test_unzip.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/step/test_dependencies.rb",
     "test/scout/workflow/step/test_info.rb",
     "test/scout/workflow/step/test_load.rb",
     "test/scout/workflow/step/test_provenance.rb",
+    "test/scout/workflow/step/test_status.rb",
     "test/scout/workflow/task/test_inputs.rb",
     "test/scout/workflow/test_definition.rb",
     "test/scout/workflow/test_documentation.rb",
@@ -203,25 +217,15 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.2.15".freeze
+  s.rubygems_version = "3.4.11".freeze
   s.summary = "basic gear for scouts".freeze
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 4
-  end
+  s.specification_version = 4
 
-  if s.respond_to? :add_runtime_dependency then
-    s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
-    s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
-    s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
-    s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
-    s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
-  else
-    s.add_dependency(%q<term-ansicolor>.freeze, [">= 0"])
-    s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
-    s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
-    s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
-    s.add_dependency(%q<simplecov>.freeze, [">= 0"])
-  end
+  s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+  s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
+  s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+  s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
 end
 

data/scout_commands/workflow/task

@@ -4,6 +4,9 @@ require 'scout'
 
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
+ENV["SCOUT_NO_STREAM"] = "true" if ARGV.include? "--nostream"
+ENV["SCOUT_UPDATE"] = "true" if ARGV.include? "--update"
+
 options = SOPT.setup <<EOF
 
 Run a workflow job
@@ -11,11 +14,14 @@ Run a workflow job
 $ #{$0} [<options>] <workflow> <task>
 
 -h--help Print this help
+--nostream Disable job streaming
+--update Update jobs with newer dependencies
 -jn--job_name* Name to use as job identifier
 -pf--print_filepath Print the file path
 -prov--provenance Print the step provenance
 -cl--clean Clean the last step
 -rcl--recursive_clean Clean all steps
+-ct--clean_task* Clean a particular task
 EOF
 
 workflow_name, task_name = ARGV
@@ -28,7 +34,10 @@ task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
-if options[:help]
+help, provenance, clean, recursive_clean, clean_task, jobname, print_filepath = IndiferentHash.process_options options, 
+  :help, :provenance, :clean, :recursive_clean, :clean_task, :job_name, :print_filepath
+
+if help
   if defined? scout_usage
     scout_usage 
   else
@@ -40,17 +49,35 @@ if options[:help]
 end
 
 job_options = task.get_SOPT(task)
-job = workflow.job(task_name, options[:job_name], job_options)
+job = workflow.job(task_name, jobname, job_options)
+
+job.recursive_clean if recursive_clean
+job.clean if clean
+
+if clean_task 
+  ENV["SCOUT_UPDATE"] = 'true'
+  clean_task.split(",").each do |clean_task|
+    if clean_task.include? "#"
+      clean_workflow, clean_task = clean_task.split("#")
+    end
 
-job.recursive_clean if options[:recursive_clean]
-job.clean if options[:clean]
+    job.rec_dependencies.each do |dep|
+      next unless dep.task_name.to_s == clean_task.to_s
+      next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
+      dep.clean 
+      dep.set_info :status, :cleaned
+    end
+
+    job.clean if job.task_name.to_s == clean_task.to_s
+  end
+end
 
-if options[:provenance]
+if provenance
   puts Step.prov_report(job)
 else
-  job.run unless job.done?
+  job.run(true) unless job.done?
 
-  if options[:print_filepath]
+  if print_filepath
     job.join
     path = job.path
     path = path.find if Path === path

data/test/scout/open/test_stream.rb

@@ -334,63 +334,65 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
-  #  def test_collapse_stream
-  #    text=<<-EOF
-  #row1 A B C
-  #row1 a b c
-  #row2 AA BB CC
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    tsv = TSV.open Misc.collapse_stream(s,nil, " "), :sep => " " 
-  #    assert_equal ["A", "a"], tsv["row1"][0]
-  #    assert_equal ["BB", "bb"], tsv["row2"][1]
-  #  end
-  #
-  #  def test_collapse_sum
-  #    text=<<-EOF
-  #row1 12
-  #row1 4
-  #row2 10
-  #row2 6
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    stream = Misc.collapse_stream(s,nil, " ") do |parts|
-  #      next nil if parts.empty?
-  #      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
-  #    end
-  #    tsv = TSV.open  stream, :sep => " " 
-  #    ppp tsv.to_s
-  #  end
-  #
-  #  def test_collapse_stream_gap
-  #    text=<<-EOF
-  #row2 AA BB 
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
-  #    
-  #    text=<<-EOF
-  #row2 aa bb cc
-  #row2 AA BB 
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /cc\|$/
-  #
-  #    text=<<-EOF
-  #row2 AA BB
-  #row2 aa bb cc
-  #    EOF
-  #
-  #    s = StringIO.new text
-  #    assert Misc.collapse_stream(s, nil, " ").read =~  /\|cc$/
-  #
-  #  end
+  def test_collapse_stream
+    text=<<-EOF
+row1 A B C
+row1 a b c
+row2 AA BB CC
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    stream = Open.collapse_stream(s, sep: " ")
+    txt = stream.read
+    assert_include txt, "A|a"
+    assert_include txt, "B|b"
+    assert_include txt, "C|c"
+  end
+
+
+  def test_collapse_sum
+    text=<<-EOF
+row1 12
+row1 4
+row2 10
+row2 6
+    EOF
+
+    s = StringIO.new text
+    stream = Open.collapse_stream(s, sep: " ") do |parts|
+      next nil if parts.empty?
+      parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
+    end
+    assert_include stream.read, "row1 16"
+  end
+
+
+  def test_collapse_stream_gap
+    text=<<-EOF
+row2 AA BB 
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /\|cc$/
+    
+    text=<<-EOF
+row2 aa bb cc
+row2 AA BB 
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /cc\|$/
+
+    text=<<-EOF
+row2 AA BB
+row2 aa bb cc
+    EOF
+
+    s = StringIO.new text
+    assert Open.collapse_stream(s, sep: " ").read =~  /\|cc$/
+  end
   #
   #
   #  def test_paste_stream
@@ -433,7 +435,7 @@ row1 A B C
   #    s = StringIO.new text
   #    sorted = Misc.sort_stream(s)
   #    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
-#    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  #    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
 #  end
 #
 #  def test_sort_long_stream

data/test/scout/path/test_find.rb

@@ -68,7 +68,6 @@ class TestPathFind < Test::Unit::TestCase
     p = Path.setup("/tmp/foo/bar")
     assert p.located?
     assert_equal_path p, p.find
-    
   end
 
   def test_custom
@@ -106,5 +105,15 @@ class TestPathFind < Test::Unit::TestCase
       assert Open.exist?(tmpdir.somefile)
     end
   end
+
+  def test_with_extension
+    dir = tmpdir.directory[__method__]
+    list = %w(a b)
+    Misc.in_dir(dir) do
+      file = dir.foo
+      Open.write(file.set_extension('list'), list * "\n")
+      assert_equal list, file.list
+    end
+  end
 end
 

data/test/scout/resource/test_produce.rb

@@ -59,4 +59,19 @@ end
     assert_include File.open(TestResource.tmp.test.work.footest.bar.find).read, "OTHER"
     assert_include File.open(TestResource.tmp.test.work.footest.foo_bar.find).read, "OTHER"
   end
+
+  def test_produce_with_extension
+    dir = tmpdir.directory[__method__]
+    dir.pkgdir = Scout
+    list = %w(a b)
+
+    Scout.claim dir["foo.list"], :proc do
+      list
+    end
+
+    Misc.in_dir(dir) do
+      file = dir.foo
+      assert_equal list, file.list
+    end
+  end
 end

data/test/scout/test_meta_extension.rb

@@ -8,6 +8,13 @@ class TestMetaExtension < Test::Unit::TestCase
     extension_attr :code, :code2
   end
 
+  module ExtensionClass2
+    extend MetaExtension
+
+    extension_attr :code3, :code4
+  end
+
+
   def test_setup_annotate
     str = "String"
     ExtensionClass.setup(str, :code)
@@ -51,5 +58,23 @@ class TestMetaExtension < Test::Unit::TestCase
     assert o.extension_attr_hash.include?(:code)
     assert o.extension_attr_hash.include?(:code2)
   end
+
+  def test_twice
+    str = "String"
+
+    ExtensionClass.setup(str, :code2 => :code)
+    assert_equal :code, str.code2
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+
+    str.extend ExtensionClass2
+    str.code3 = :code_alt
+    assert_equal :code, str.code2
+    assert_equal :code_alt, str.code3
+    assert_include str.instance_variable_get(:@extension_attrs), :code
+    assert_include str.instance_variable_get(:@extension_attrs), :code3
+
+    assert_include str.extension_attr_hash, :code
+    assert_include str.extension_attr_hash, :code3
+  end
 end
 

data/test/scout/test_named_array.rb

@@ -15,5 +15,23 @@ ValueB (Entity type)
     assert_equal 1, NamedArray.identify_name(names, "ValueB")
     assert_equal 1, NamedArray.identify_name(names, 1)
   end
+
+  def test_missing_field
+    a = NamedArray.setup([1,2], [:a, :b])
+    assert_equal 1, a[:a]
+    assert_equal nil, a[:c]
+  end
+
+  def test_zip_fields
+    a = [%w(a b), %w(1 1)]
+    assert_equal [%w(a 1), %w(b 1)], NamedArray.zip_fields(a)
+  end
+
+  def test_add_zipped
+    a = [%w(a b), %w(1 1)]
+    NamedArray.add_zipped a, [%w(c), %w(1)]
+    NamedArray.add_zipped a, [%w(d), %w(1)]
+    assert_equal [%w(a b c d), %w(1 1 1 1)], a
+  end
 end
 

data/test/scout/test_persist.rb

@@ -156,6 +156,12 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_path_prefix
+    Persist.persist('foo', :tsv, :prefix => "TSV") do |filename|
+      assert File.basename(filename).start_with? "TSV"
+    end
+  end
+
   def __test_speed
     times = 100_000
     TmpFile.with_file do |tmpfile|