scout-gear 7.2.0 → 7.3.0

data/lib/scout/work_queue/worker.rb

@@ -27,7 +27,7 @@ class WorkQueue
         rescue Interrupt
         rescue Exception
           output.write WorkerException.new($!, Process.pid)
-          exit -1
+          Kernel.exit -1
         end
       end
     end

data/lib/scout/workflow/definition.rb

@@ -164,4 +164,12 @@ module Workflow
   end
 
   alias dep_task task_alias
+
+  def export(*args)
+  end
+
+  alias export_synchronous export
+  alias export_asynchronous export
+  alias export_exec export
+  alias export_stream export
 end

data/lib/scout/workflow/step/info.rb

@@ -101,4 +101,8 @@ class Step
   def running?
     ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
   end
+
+  def exception
+    info[:exception]
+  end
 end

data/lib/scout/workflow/step/progress.rb

@@ -0,0 +1,14 @@
+class Step
+  def progress_bar(msg = "Progress", options = nil)
+    if Hash === msg and options.nil?
+      options = msg
+      msg = nil
+    end
+    options = {} if options.nil?
+
+    max = options[:max]
+    Open.mkdir files_dir
+    Log::ProgressBar.new_bar(max, {:desc => msg, :file => (@exec ? nil : file(:progress))}.merge(options))
+  end
+end
+

data/lib/scout/workflow/step.rb

@@ -6,6 +6,7 @@ require_relative 'step/file'
 require_relative 'step/dependencies'
 require_relative 'step/provenance'
 require_relative 'step/config'
+require_relative 'step/progress'
 
 class Step 
 
@@ -78,7 +79,9 @@ class Step
           :inputs => inputs, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
-        exec
+        @result = exec
+        @result = @result.respond_to?(:stream) ? @result.stream : @result
+        @result
       rescue Exception => e
         merge_info :status => :error, :exception => e
         raise e
@@ -104,6 +107,7 @@ class Step
         end
       end
     end
+    @result
   end
 
   def done?
@@ -111,10 +115,10 @@ class Step
   end
 
   def streaming?
-    @take_stream || IO === @result || StringIO === @result
+    @take_stream || IO === @result || StringIO === @result 
   end
 
-  def get_stream
+  def stream
     synchronize do
       if streaming? && ! @result.nil?
         if @result.next
@@ -137,8 +141,9 @@ class Step
   end
 
   def join
-    stream = get_stream if streaming?
-    Open.consume_stream(stream, false) if stream
+    io = self.stream if streaming?
+    Open.consume_stream(io, false) if io
+    self
   end
 
   def produce

data/lib/scout/workflow/task.rb

@@ -1,4 +1,5 @@
 require_relative '../meta_extension'
+require_relative '../named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 
@@ -24,7 +25,7 @@ module Task
     return inputs if deps.nil?
     deps.inject(inputs) do |acc,dep|
       workflow, task = dep
-      next if workflow.nil?
+      next acc if workflow.nil? || task.nil?
       acc += workflow.tasks[task].recursive_inputs
     end
   end
@@ -144,13 +145,16 @@ module Task
 
     if non_default_inputs.any?
       hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
-      Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
+      Log.debug "Hash #{name} - #{hash}: #{Log.fingerprint(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
       id = [id, hash] * "_"
     end
 
     path = directory[id]
 
-    NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
-    Step.new path.find, inputs, dependencies, &self
+    Persist.memory(path) do 
+      Log.debug "Creating job #{path} #{Log.fingerprint inputs} #{Log.fingerprint dependencies}"
+      NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
+      Step.new path.find, inputs, dependencies, &self
+    end
   end
 end

data/lib/scout/workflow/usage.rb

@@ -20,6 +20,7 @@ module Task
       seen = inputs.collect{|name,_| name }
       dep_inputs = {}
       deps.each do |dep_workflow,task_name|
+        next if task_name.nil?
         task = dep_workflow.tasks[task_name]
         next if task.inputs.nil?
         inputs = task.inputs.reject{|name, _| seen.include? name }
@@ -122,6 +123,7 @@ module Workflow
       next if seen.include? dep
       seen << dep
       workflow, task, *rest = dep
+      next if task.nil?
 
       key = [workflow, task]
 

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 7.2.0 ruby lib
+# stub: scout-gear 7.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "7.2.0"
+  s.version = "7.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-05-09"
+  s.date = "2023-05-12"
   s.description = "Temporary files, logs, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -50,6 +50,7 @@ Gem::Specification.new do |s|
     "lib/scout/misc/digest.rb",
     "lib/scout/misc/filesystem.rb",
     "lib/scout/misc/format.rb",
+    "lib/scout/misc/helper.rb",
     "lib/scout/misc/insist.rb",
     "lib/scout/misc/monitor.rb",
     "lib/scout/misc/system.rb",
@@ -85,13 +86,19 @@ Gem::Specification.new do |s|
     "lib/scout/tsv.rb",
     "lib/scout/tsv/dumper.rb",
     "lib/scout/tsv/index.rb",
+    "lib/scout/tsv/open.rb",
     "lib/scout/tsv/parser.rb",
     "lib/scout/tsv/path.rb",
     "lib/scout/tsv/persist.rb",
     "lib/scout/tsv/persist/adapter.rb",
+    "lib/scout/tsv/persist/fix_width_table.rb",
+    "lib/scout/tsv/persist/serialize.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
     "lib/scout/tsv/traverse.rb",
     "lib/scout/tsv/util.rb",
+    "lib/scout/tsv/util/filter.rb",
+    "lib/scout/tsv/util/process.rb",
+    "lib/scout/tsv/util/select.rb",
     "lib/scout/work_queue.rb",
     "lib/scout/work_queue/socket.rb",
     "lib/scout/work_queue/worker.rb",
@@ -104,6 +111,7 @@ Gem::Specification.new do |s|
     "lib/scout/workflow/step/file.rb",
     "lib/scout/workflow/step/info.rb",
     "lib/scout/workflow/step/load.rb",
+    "lib/scout/workflow/step/progress.rb",
     "lib/scout/workflow/step/provenance.rb",
     "lib/scout/workflow/task.rb",
     "lib/scout/workflow/task/inputs.rb",
@@ -165,13 +173,18 @@ Gem::Specification.new do |s|
     "test/scout/test_work_queue.rb",
     "test/scout/test_workflow.rb",
     "test/scout/tsv/persist/test_adapter.rb",
+    "test/scout/tsv/persist/test_fix_width_table.rb",
     "test/scout/tsv/persist/test_tokyocabinet.rb",
     "test/scout/tsv/test_dumper.rb",
     "test/scout/tsv/test_index.rb",
+    "test/scout/tsv/test_open.rb",
     "test/scout/tsv/test_parser.rb",
     "test/scout/tsv/test_persist.rb",
     "test/scout/tsv/test_traverse.rb",
     "test/scout/tsv/test_util.rb",
+    "test/scout/tsv/util/test_filter.rb",
+    "test/scout/tsv/util/test_process.rb",
+    "test/scout/tsv/util/test_select.rb",
     "test/scout/work_queue/test_socket.rb",
     "test/scout/work_queue/test_worker.rb",
     "test/scout/workflow/step/test_dependencies.rb",
@@ -190,15 +203,25 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.4.11".freeze
+  s.rubygems_version = "3.2.15".freeze
   s.summary = "basic gear for scouts".freeze
 
-  s.specification_version = 4
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+  end
 
-  s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
-  s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
-  s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
-  s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
-  s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
+  if s.respond_to? :add_runtime_dependency then
+    s.add_runtime_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+    s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+    s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
+    s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+    s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
+  else
+    s.add_dependency(%q<term-ansicolor>.freeze, [">= 0"])
+    s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+    s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
+    s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+    s.add_dependency(%q<simplecov>.freeze, [">= 0"])
+  end
 end
 

data/scout_commands/workflow/task

@@ -24,7 +24,7 @@ raise MissingParameterException.new :workflow if workflow_name.nil?
 
 workflow = Workflow.require_workflow workflow_name
 task_name = task_name.to_sym if task_name
-task = workflow.tasks[task_name.to_sym] 
+task = workflow.tasks[task_name.to_sym] if task_name
 
 options[:help] = true if task.nil?
 
@@ -51,11 +51,12 @@ else
   job.run unless job.done?
 
   if options[:print_filepath]
+    job.join
     path = job.path
     path = path.find if Path === path
     puts path
   else
-    if ! ((c = Open.consume_stream(job.get_stream, false, STDOUT, false)) && c.end_with?("\n"))
+    if ! ((c = Open.consume_stream(job.stream, false, STDOUT, false)) && c.end_with?("\n"))
       puts
     end
   end

data/scout_commands/workflow/task_old

@@ -656,12 +656,12 @@ when (defined?(RemoteStep) and RemoteStep)
   end
 when Step
   if res.streaming?
-    io = TSV.get_stream res
+    io = TSV.stream res
     Misc.consume_stream(io, false, out)
     io.join if io.respond_to? :join
   elsif IO === res.result
     begin
-      io = res.get_stream
+      io = res.stream
       Misc.consume_stream(io, false, out)
       io.join if io.respond_to? :join
     rescue Aborted, Interrupt

data/test/scout/open/test_stream.rb

@@ -5,7 +5,7 @@ require 'scout/path'
 require 'scout/open'
 
 class TestOpenStream < Test::Unit::TestCase
-  def test_consume_stream
+  def test_stream
     content =<<-EOF
 1
 2

data/test/scout/test_persist.rb

@@ -95,6 +95,67 @@ class TestPersist < Test::Unit::TestCase
     end
   end
 
+  def test_concurrent
+    num = 10
+
+    s = 0.01
+    10.times do
+      TmpFile.with_file do |file|
+        output1 = file + '.output1'
+        output2 = file + '.output2'
+        pid1 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output1, io)
+          end
+        end
+        pid2 = Process.fork do
+          Open.purge_pipes
+          sleep rand/10.0
+          io = Persist.persist("test", :string, :path => file) do
+            Open.open_pipe do |sin|
+              num.times do |i|
+                sin.puts "line-#{i}-#{Process.pid}"
+                sleep s
+              end
+            end
+          end
+          if IO === io
+            Open.consume_stream(io, false)
+          else
+            Open.write(output2, io)
+          end
+        end
+        Process.wait
+        Process.wait
+
+        assert File.exist?(output1) || File.exist?(output2)
+        [pid1, pid2].zip([output2, output1]).each do |pid, found|
+          next unless File.exist?(found)
+          assert Open.read(found).include? "-#{pid}\n"
+        end
+        [pid1, pid2].zip([output1, output2]).each do |pid, found|
+          next unless File.exist?(found)
+          refute Open.read(found).include? "-#{pid}\n"
+        end
+        Open.rm file
+        Open.rm output1
+        Open.rm output2
+      end
+    end
+  end
+
   def __test_speed
     times = 100_000
     TmpFile.with_file do |tmpfile|

data/test/scout/test_tmpfile.rb

@@ -10,7 +10,7 @@ class TestTmpFile < Test::Unit::TestCase
   def test_do_tmp_file
     content = "Hello World!"
     TmpFile.with_file(content) do |file|
-      assert_equal_path content, File.open(file).read
+      assert_equal content, File.open(file).read
     end
   end
 

data/test/scout/test_tsv.rb

@@ -61,7 +61,15 @@ row2    a    a    id3
     EOF
 
     tsv = TmpFile.with_file(content) do |filename|
-      TSV.open(filename, :persist => true)
+      TSV.open(filename, :persist => false)
+    end
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true, :merge => true)
     end
 
     assert tsv.respond_to?(:persistence_class)
@@ -69,6 +77,7 @@ row2    a    a    id3
 
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
+    assert_equal %w(A a), tsv["row2"][0]
   end
 end
 

data/test/scout/test_work_queue.rb

@@ -93,6 +93,7 @@ class TestWorkQueue < Test::Unit::TestCase
     num = 100
     reps = 10_000
 
+    sss 0
     q = WorkQueue.new num do |obj|
       raise ScoutException if rand < 0.1
       [Process.pid.to_s, obj.to_s] * " "

data/test/scout/tsv/persist/test_adapter.rb

@@ -30,5 +30,15 @@ class TestTSVAdapter < Test::Unit::TestCase
     assert_equal [["a", %w(1 2 3)], ["b", %w(11 22 33)]], tsv.sort
     assert_equal [["b", %w(11 22 33)], ["a", %w(1 2 3)]], tsv.sort_by{|k,v| -v[0].to_i }
   end
+
+  def test_serializer
+    tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
+    tsv.type = :list
+    tsv.extend TSVAdapter
+    tsv.serializer = :integer_array
+    tsv["a"] = [1, 2, 3]
+
+    assert_equal [1, 2, 3], tsv["a"]
+  end
 end
 

data/test/scout/tsv/persist/test_fix_width_table.rb

@@ -0,0 +1,134 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestFixWidthTable < Test::Unit::TestCase
+  def load_data(data)
+    tsv = TSV.open(data, type: :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
+    tsv.add_field "Start" do |key, values|
+      values["Range"].first
+    end
+    tsv.add_field "End" do |key, values|
+      values["Range"].last
+    end
+
+    tsv = tsv.slice ["Start", "End"]
+
+    tsv
+  end
+
+  def test_options
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.close
+
+      f1 = FixWidthTable.new filename, 100, false
+
+      assert_equal true, f1.range
+    end
+  end
+
+  def test_add
+    TmpFile.with_file do |filename|
+      f = FixWidthTable.new filename, 100, true
+      f.add [1,2,0], "test1"
+      f.add [3,4,0], "test2"
+      f.read
+
+      assert_equal 1, f.idx_pos(0)
+      assert_equal 3, f.idx_pos(1)
+      assert_equal 2, f.idx_pos_end(0)
+      assert_equal 4, f.idx_pos_end(1)
+      assert_equal 0, f.idx_overlap(0)
+      assert_equal 0, f.idx_overlap(1)
+      assert_equal "test1", f.idx_value(0)
+      assert_equal "test2", f.idx_value(1)
+
+    end
+
+  end
+
+  def test_point
+    data =<<-EOF
+#: :sep=/\\s+/#:type=:single#:cast=:to_i
+#ID Pos
+a 1
+b 10
+c 20
+d 12
+e 26
+f 11
+g 25
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = TSV.open datafile
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, false
+        f.add_point tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[10].sort
+        assert_equal %w(a b c d f), f[(0..20)].sort
+      end
+    end
+  end
+
+  def test_range
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f[0].sort
+        assert_equal %w(b), f[1].sort
+        assert_equal %w(), f[20].sort
+        assert_equal %w(), f[(20..100)].sort
+        assert_equal %w(a b d), f[3].sort
+        assert_equal %w(a b c d e), f[(3..4)].sort
+        assert_equal %w(a c e), f[7].sort
+      end
+    end
+  end
+
+
+  def test_range_pos
+    data =<<-EOF
+##012345678901234567890
+#ID:Range
+a:   ______
+b: ______
+c:    _______
+d:  ____
+e:    ______
+f:             ___
+g:         ____
+    EOF
+    TmpFile.with_file(data) do |datafile|
+      tsv = load_data(datafile)
+      TmpFile.with_file do |filename|
+        f = FixWidthTable.new filename, 100, true
+        f.add_range tsv
+        f.read
+
+        assert_equal %w(), f.overlaps(0).sort
+        assert_equal %w(1:6), f.overlaps(1).sort
+        assert_equal %w(1:6:b), f.overlaps(1, true).sort
+      end
+    end
+  end
+end
+