scout-gear 7.2.0 → 7.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de499ccbdffd888298324a025b253f0dd0e359d279d6abe8331acd91fd00cc57
-  data.tar.gz: 9643053080da23a98209ebefbcd3b3043b7031374a405ffed112460f1ddcf411
+  metadata.gz: 7273030ac34bd180620d9075ee85b26d8a9883831a5925fc911915f1edeeecf3
+  data.tar.gz: 4e445acb896844240c0024b7754cffee93af71feb62f9c4251fda70aee79121a
 SHA512:
-  metadata.gz: 73afbf6aeb4032380fc45e2f60e2e9335d763c6837ef3bb8247c83bfaf2cd962a6fdfba767788b988e00f685d0e9bc19927a4e794fc092296cc088f23d4565ee
-  data.tar.gz: 309e86afe466c649800fd212fd22e9e183a5e92ba558d88ec0258690a15437952ead3e968c3693b89c036dc2fa9e27f4d129f6be6c001031dc38ab071361ba99
+  metadata.gz: '029a8eb23eda77c7e0c11ac2966e1e8ebf3eae479bbee0c76e113f5b839c655e97e0fec5025cc3c251795afcfc92ac3395a79b68c8eb99158d1cee8e2310f002'
+  data.tar.gz: 04ed4bcabfa7c41b80ba520074404bfb09c723ecdb8d02a2052f73fe86759bab7fdadff31f0f66284ed4938ebe5a1cad56fb94182b3edcc0de6325decdcb84f6

data/.vimproject

@@ -32,6 +32,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     filesystem.rb
     monitor.rb
     system.rb
+    helper.rb
    }
    named_array.rb
    indiferent_hash.rb
@@ -103,6 +104,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      config.rb
      dependencies.rb
      provenance.rb
+     progress.rb
     }
     task.rb
     task=task{
@@ -123,26 +125,58 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     parser.rb
     dumper.rb
     persist.rb
-    index.rb
     persist=persist{
      adapter.rb
+     serialize.rb
      tokyocabinet.rb
+     fix_width_table.rb
     }
+    index.rb
     traverse.rb
     util.rb
+    util=util{
+     process.rb
+     select.rb
+     filter.rb
+    }
     path.rb
+    open.rb
    }
   }
  }
  test=test {
   test_helper.rb
+  test_scout-gear.rb
   scout=scout{
+   test_cmd.rb
+   test_concurrent_stream.rb
+   test_config.rb
    test_indiferent_hash.rb
+   test_log.rb
+   test_meta_extension.rb
+   test_misc.rb
+   test_named_array.rb
+   test_open.rb
+   test_path.rb
+   test_persist.rb
+   test_resource.rb
+   test_semaphore.rb
+   test_tmpfile.rb
+   test_tsv.rb
+   test_work_queue.rb
+   test_workflow.rb
+   workflow=workflow{
+    test_definition.rb
+    test_documentation.rb
+    test_step.rb
+    test_task.rb
+    test_usage.rb
+    test_util.rb
+    }
    indiferent_hash=indiferent_hash{
+    test_case_insensitive.rb
     test_options.rb
    }
-   test_log.rb
-   test_tmpfile.rb
   }
  }
  share=share{

data/VERSION

@@ -1 +1 @@
-7.2.0
+7.3.0

data/lib/scout/concurrent_stream.rb

@@ -9,11 +9,11 @@ module AbortedStream
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
 
   def self.setup(stream, options = {}, &block)
     
-    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair, :next
+    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -54,7 +54,7 @@ module ConcurrentStream
 
     stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
 
-    stream.lockfile = lockfile unless lockfile.nil?
+    stream.lock = lock unless lock.nil?
 
     stream.aborted = false
 
@@ -62,7 +62,7 @@ module ConcurrentStream
   end
 
   def annotate(stream)
-    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
     stream
   end
 
@@ -136,11 +136,12 @@ module ConcurrentStream
     begin
       join_threads
       join_pids
+      raise stream_exception if stream_exception
       join_callback
       close unless closed?
     ensure
       @joined = true
-      lockfile.unlock if lockfile and lockfile.locked?
+      lock.unlock if lock and lock.locked?
       raise stream_exception if stream_exception
     end
   end
@@ -160,7 +161,7 @@ module ConcurrentStream
     @threads.each do |t|
       next if t == Thread.current
       begin
-        t.join unless t == Thread.current
+        t.join
       rescue Aborted
       rescue Exception
         Log.debug "Thread (#{name}) exception: #{$!.message}"
@@ -205,8 +206,8 @@ module ConcurrentStream
     ensure
       close unless closed?
 
-      if lockfile and lockfile.locked?
-        lockfile.unlock 
+      if lock and lock.locked?
+        lock.unlock 
       end
     end
   end

data/lib/scout/exceptions.rb

@@ -92,6 +92,7 @@ class WorkerException < ScoutException
   end
 end
 
+class SemaphoreInterrupted < TryAgain; end
 
 #class OpenGzipError < StandardError; end
 #

data/lib/scout/log/color.rb

@@ -2,7 +2,6 @@ require_relative 'color_class'
 require_relative '../indiferent_hash'
 
 require 'term/ansicolor'
-require 'colorist'
 
 module Colorize
   def self.colors=(colors)

data/lib/scout/log/progress/util.rb

@@ -96,6 +96,71 @@ module Log
         remove_bar(bar, error) if bar && ! keep
       end
     end
+
+    def self.guess_obj_max(obj)
+    begin
+      case obj
+      when (defined? Step and Step)
+        if obj.done?
+          path = obj.path
+          path = path.find if path.respond_to? :find
+          if File.exist? path
+            CMD.cmd("wc -l '#{path}'").read.to_i 
+          else
+            nil
+          end
+        else
+          nil
+        end
+      when TSV
+        obj.length
+      when Array, Hash
+        obj.size
+      when File
+        return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+        CMD.cmd("wc -l '#{obj.path}'").read.to_i
+      when Path, String
+        obj = obj.find if Path === obj
+        if File.exist? obj
+          return nil if Open.gzip?(obj) or Open.bgzip?(obj)
+          CMD.cmd("wc -l '#{obj}'").read.to_i
+        else
+          nil
+        end
+      end
+    rescue Exception
+      Log.exception $!
+      nil
+    end
+  end
+
+  def self.get_obj_bar(bar, obj)
+    case bar
+    when String
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(max, {:desc => bar}) 
+    when TrueClass
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(max, nil) 
+    when Numeric
+      max = guess_obj_max(obj)
+      Log::ProgressBar.new_bar(bar) 
+    when Hash
+      max = Misc.process_options(bar, :max) || max
+      Log::ProgressBar.new_bar(max, bar) 
+    when Log::ProgressBar
+      bar.max ||= guess_obj_max(obj)
+      bar
+    else
+      if (defined? Step and Step === bar)
+        max = guess_obj_max(obj)
+        Log::ProgressBar.new_bar(max, {:desc => bar.status, :file => bar.file(:progress)}) 
+      else
+        bar
+      end
+    end
+  end
   end
+
 end
 

data/lib/scout/misc/helper.rb

@@ -0,0 +1,31 @@
+module Misc
+  def self.intersect_sorted_arrays(a1, a2)
+    e1, e2 = a1.shift, a2.shift
+    intersect = []
+    while true
+      break if e1.nil? or e2.nil?
+      case e1 <=> e2
+      when 0
+        intersect << e1
+        e1, e2 = a1.shift, a2.shift
+      when -1
+        e1 = a1.shift while not e1.nil? and e1 < e2
+      when 1
+        e2 = a2.shift
+        e2 = a2.shift while not e2.nil? and e2 < e1
+      end
+    end
+    intersect
+  end
+
+  def self.counts(array)
+    counts = {}
+    array.each do |e|
+      counts[e] ||= 0
+      counts[e] += 1
+    end
+
+    counts
+  end
+
+end

data/lib/scout/misc/monitor.rb

@@ -14,7 +14,7 @@ module Misc
       if message
         puts "#{message }: #{ repeats } repeats"
       else
-        puts "Benchmark for #{ repeats } repeats"
+        puts "Benchmark for #{ repeats } repeats (#{caller.first})"
       end
       puts measure
     rescue Exception

data/lib/scout/misc.rb

@@ -4,6 +4,7 @@ require_relative 'misc/digest'
 require_relative 'misc/filesystem'
 require_relative 'misc/monitor'
 require_relative 'misc/system'
+require_relative 'misc/helper'
 
 module Misc
 end

data/lib/scout/open/stream.rb

@@ -54,11 +54,9 @@ module Open
         
         into_close = false unless into.respond_to? :close
 
-        begin
-          while c = io.readpartial(BLOCK_SIZE)
-            into << c if into
-          end
-        rescue EOFError
+        while c = io.read(BLOCK_SIZE)
+          into << c if into
+          break if io.closed?
         end
 
         io.join if io.respond_to? :join
@@ -120,12 +118,10 @@ module Open
           when (IO === content or StringIO === content or File === content)
             Open.write(tmp_path) do |f|
               #f.sync = true
-              begin
-                while block = content.readpartial(BLOCK_SIZE)
-                  f.write block
-                end 
-              rescue EOFError
-              end
+              while block = content.read(BLOCK_SIZE)
+                f.write block
+                break if content.closed?
+              end 
             end
           else
             File.open(tmp_path, 'wb') do |f|  end
@@ -290,22 +286,20 @@ module Open
         Thread.current["name"] = "Splitter #{Log.fingerprint stream}"
 
         skip = [false] * num
-        begin
-          while block = stream.readpartial(BLOCK_SIZE)
-
-            in_pipes.each_with_index do |sin,i|
-              begin 
-                sin.write block
-              rescue IOError
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
-                skip[i] = true
-              rescue
-                Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
-                raise $!
-              end unless skip[i] 
-            end
+        while block = stream.read(BLOCK_SIZE)
+
+          in_pipes.each_with_index do |sin,i|
+            begin 
+              sin.write block
+            rescue IOError
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} IOError: #{$!.message} (#{Log.fingerprint sin})");
+              skip[i] = true
+            rescue
+              Log.warn("Tee stream #{i} #{Log.fingerprint stream} Exception: #{$!.message} (#{Log.fingerprint sin})");
+              raise $!
+            end unless skip[i] 
           end
-        rescue IOError
+          break if stream.closed?
         end
 
         stream.join if stream.respond_to? :join
@@ -397,7 +391,7 @@ module Open
     end
     str
   end
-  
+
   def self.sort_stream(stream, header_hash = "#", cmd_args = "-u")
     Open.open_pipe do |sin|
       line = stream.gets

data/lib/scout/persist.rb

@@ -14,7 +14,7 @@ module Persist
 
     attr_writer :lock_dir
     def lock_dir
-      @lock_dir ||= Path.setup("var/cache/persist_locks")
+      @lock_dir ||= Path.setup("tmp/persist_locks").find
     end
   end
 
@@ -24,46 +24,60 @@ module Persist
     TmpFile.tmp_for_file(name, options, other_options)
   end
 
+  MEMORY_CACHE = {}
+  CONNECTIONS = {}
   def self.persist(name, type = :serializer, options = {}, &block)
     persist_options = IndiferentHash.pull_keys options, :persist 
     return yield if FalseClass === persist_options[:persist]
     file = persist_options[:path] || options[:path] || persistence_path(name, options)
 
+    lockfile = persist_options[:lockfile] || options[:lockfile] || Persist.persistence_path(file + '.persist', {:dir => Persist.lock_dir})
+
     update = options[:update] || persist_options[:update]
     update = Open.mtime(update) if Path === update
     update = Open.mtime(file) >= update ? false : true if Time === update
 
-    if Open.exist?(file) && ! update
-      Persist.load(file, type)
-    else
-      return yield(file) if block.arity == 1
-      res = yield
-      begin
-        Open.rm(file)
+    if type == :memory
+      repo = options[:memory] || options[:repo] || MEMORY_CACHE
+      repo[file] ||= yield
+      return repo[file]
+    end
+
+    Open.lock lockfile do |lock|
+      if Open.exist?(file) && ! update
+        Persist.load(file, type)
+      else
+        return yield(file) if block.arity == 1
+        res = yield
+        begin
+          Open.rm(file)
 
-        if IO === res || StringIO === res
-          tee_copies = options[:tee_copies] || 1
-          main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
-          t = Thread.new do
-            Thread.current.report_on_exception = false
-            Thread.current["name"] = "file saver: " + file
-            Open.sensible_write(file, main)
-          end
-          Thread.pass until t["name"]
-          copies.each_with_index do |copy,i|
-            next_stream = copies[i+1] if copies.length > i
-            ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+          if IO === res || StringIO === res
+            tee_copies = options[:tee_copies] || 1
+            main, *copies = Open.tee_stream_thread_multiple res, tee_copies + 1
+            main.lock = lock
+            t = Thread.new do
+              Thread.current.report_on_exception = false
+              Thread.current["name"] = "file saver: " + file
+              Open.sensible_write(file, main)
+            end
+            Thread.pass until t["name"]
+            copies.each_with_index do |copy,i|
+              next_stream = copies[i+1] if copies.length > i
+              ConcurrentStream.setup copy, :threads => t, :filename => file, :autojoin => true, :next => next_stream
+            end
+            res = copies.first
+            raise KeepLocked.new(res)
+          else
+            pres = Persist.save(res, file, type)
+            res = pres unless pres.nil?
           end
-          res = copies.first
-        else
-          pres = Persist.save(res, file, type)
-          res = pres unless pres.nil?
+        rescue
+          raise $! unless options[:canfail]
+          Log.debug "Could not persist #{type} on #{file}"
         end
-      rescue
-        raise $! unless options[:canfail]
-        Log.debug "Could not persist #{type} on #{file}"
+        res
       end
-      res
     end
   end
 

data/lib/scout/semaphore.rb

@@ -37,7 +37,13 @@ if continue
     int ret;
     sem_t* sem;
     sem = sem_open(name, 0);
+    if (sem == SEM_FAILED){
+      return(errno);
+    }
     ret = sem_wait(sem);
+    if (ret == -1){
+      return(errno);
+    }
     sem_close(sem);
     return(ret);
   }
@@ -51,6 +57,7 @@ if continue
     sem_close(sem);
   }
       EOF
+
     end
 
     SEM_MUTEX = Mutex.new
@@ -66,7 +73,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/" << Misc.digest(rand(1000000000000).to_s) if file.nil?
+        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end

data/lib/scout/tsv/dumper.rb

@@ -52,8 +52,8 @@ module TSV
         :sep => "\t", :type => :double
       @options = options
       @sout, @sin = Open.pipe
-      ConcurrentStream.setup(@sin, :pair => @sout)
-      ConcurrentStream.setup(@sout, :pair => @sin)
+      ConcurrentStream.setup(@sin, pair: @sout)
+      ConcurrentStream.setup(@sout, pair: @sin)
     end
 
     def init
@@ -88,16 +88,21 @@ module TSV
   end
 
   def stream
-    iii self.extension_attr_hash
     dumper = TSV::Dumper.new self.extension_attr_hash
     dumper.init
-    Thread.new do 
-      Thread.current["name"] = "Dumper thread"
-      self.each do |k,v|
-        dumper.add k, v
+    t = Thread.new do 
+      begin
+        Thread.current.report_on_exception = true
+        Thread.current["name"] = "Dumper thread"
+        self.each do |k,v|
+          dumper.add k, v
+        end
+        dumper.close
+      rescue
+        dumper.abort($!)
       end
-      dumper.close
     end
+    Thread.pass until t["name"]
     dumper.stream
   end