rbbt-util 5.12.2 → 5.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/lib/rbbt/entity.rb +1 -1
 - data/lib/rbbt/persist.rb +1 -1
 - data/lib/rbbt/tsv/attach.rb +63 -54
 - data/lib/rbbt/tsv/dumper.rb +1 -0
 - data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
 - data/lib/rbbt/tsv/util.rb +1 -3
 - data/lib/rbbt/util/cmd.rb +7 -8
 - data/lib/rbbt/util/concurrency/processes/worker.rb +2 -0
 - data/lib/rbbt/util/concurrency/processes.rb +5 -3
 - data/lib/rbbt/util/misc/concurrent_stream.rb +20 -3
 - data/lib/rbbt/util/misc/exceptions.rb +1 -0
 - data/lib/rbbt/util/open.rb +8 -3
 - data/lib/rbbt/workflow/step/run.rb +4 -4
 - data/lib/rbbt/workflow/task.rb +1 -5
 - data/test/rbbt/test_entity.rb +2 -2
 - data/test/rbbt/tsv/parallel/test_traverse.rb +5 -1
 - data/test/rbbt/tsv/test_util.rb +1 -1
 - data/test/rbbt/util/misc/test_pipes.rb +4 -4
 - data/test/rbbt/util/test_R.rb +2 -2
 - data/test/rbbt/util/test_cmd.rb +8 -11
 - data/test/rbbt/util/test_misc.rb +0 -9
 - data/test/rbbt/util/test_open.rb +0 -1
 - data/test/rbbt/util/test_tmpfile.rb +0 -1
 - data/test/rbbt/workflow/test_step.rb +10 -8
 - metadata +2 -2
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA1:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: b7e5eb68b9212f47cb33b251ea8d4d0f9c878df1
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 39ecf6d22cde5921db4011ca849d3b5c03ba2f13
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: ca4f1549b877b62b6ae97d29b214c6bff957d626819ff14147f0eadaf63b9d394d3bb3e2db31c08f77329ed823d61afba9f44704dc975767a23f5f7fad7be793
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 9e477fc7917df197279869c4bb4a754bf6e60be2d1695ca9f281f5406a1c51d29f311cebd8153430c8349abec29c529cac9e4481283387ef2929630cd5d9c914
         
     | 
    
        data/lib/rbbt/entity.rb
    CHANGED
    
    | 
         @@ -139,7 +139,7 @@ module Entity 
     | 
|
| 
       139 
139 
     | 
    
         
             
                      persist_options = options
         
     | 
| 
       140 
140 
     | 
    
         
             
                      persist_options = persist_options.merge(:other => {:args => args}) if args.any?
         
     | 
| 
       141 
141 
     | 
    
         | 
| 
       142 
     | 
    
         
            -
                      Persist.persist(persist_name, type, persist_options) do
         
     | 
| 
      
 142 
     | 
    
         
            +
                      Persist.persist(persist_name, type, persist_options.merge(:persist => true)) do
         
     | 
| 
       143 
143 
     | 
    
         
             
                        self.send(orig_name, *args)
         
     | 
| 
       144 
144 
     | 
    
         
             
                      end
         
     | 
| 
       145 
145 
     | 
    
         
             
                    end
         
     | 
    
        data/lib/rbbt/persist.rb
    CHANGED
    
    | 
         @@ -398,7 +398,7 @@ module Persist 
     | 
|
| 
       398 
398 
     | 
    
         
             
                    begin
         
     | 
| 
       399 
399 
     | 
    
         | 
| 
       400 
400 
     | 
    
         
             
                      lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
         
     | 
| 
       401 
     | 
    
         
            -
                      Misc.lock lock_filename 
     | 
| 
      
 401 
     | 
    
         
            +
                      Misc.lock lock_filename do |lockfile|
         
     | 
| 
       402 
402 
     | 
    
         | 
| 
       403 
403 
     | 
    
         
             
                        if is_persisted?(path, persist_options)
         
     | 
| 
       404 
404 
     | 
    
         
             
                          Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
         
     | 
    
        data/lib/rbbt/tsv/attach.rb
    CHANGED
    
    | 
         @@ -88,75 +88,84 @@ module TSV 
     | 
|
| 
       88 
88 
     | 
    
         
             
                  file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2.to_s(:sort, true), :pipe => true)
         
     | 
| 
       89 
89 
     | 
    
         
             
                end
         
     | 
| 
       90 
90 
     | 
    
         | 
| 
       91 
     | 
    
         
            -
                 
     | 
| 
      
 91 
     | 
    
         
            +
                begin
         
     | 
| 
      
 92 
     | 
    
         
            +
                  output = File.open(output, 'w') if String === output
         
     | 
| 
       92 
93 
     | 
    
         | 
| 
       93 
     | 
    
         
            -
             
     | 
| 
       94 
     | 
    
         
            -
             
     | 
| 
      
 94 
     | 
    
         
            +
                  cols1 = nil
         
     | 
| 
      
 95 
     | 
    
         
            +
                  cols2 = nil
         
     | 
| 
       95 
96 
     | 
    
         | 
| 
       96 
     | 
    
         
            -
             
     | 
| 
       97 
     | 
    
         
            -
             
     | 
| 
      
 97 
     | 
    
         
            +
                  done1 = false
         
     | 
| 
      
 98 
     | 
    
         
            +
                  done2 = false
         
     | 
| 
       98 
99 
     | 
    
         | 
| 
       99 
     | 
    
         
            -
             
     | 
| 
       100 
     | 
    
         
            -
             
     | 
| 
       101 
     | 
    
         
            -
             
     | 
| 
       102 
     | 
    
         
            -
             
     | 
| 
      
 100 
     | 
    
         
            +
                  key1 = key2 = nil
         
     | 
| 
      
 101 
     | 
    
         
            +
                  while key1.nil?
         
     | 
| 
      
 102 
     | 
    
         
            +
                    while (line1 = file1.gets) =~ /^#/
         
     | 
| 
      
 103 
     | 
    
         
            +
                      key_field1, *fields1 = line1.strip.sub('#','').split(sep)
         
     | 
| 
      
 104 
     | 
    
         
            +
                    end
         
     | 
| 
      
 105 
     | 
    
         
            +
                    key1, *parts1 = line1.sub("\n",'').split(sep, -1)
         
     | 
| 
      
 106 
     | 
    
         
            +
                    cols1 = parts1.length
         
     | 
| 
       103 
107 
     | 
    
         
             
                  end
         
     | 
| 
       104 
     | 
    
         
            -
                  key1, *parts1 = line1.sub("\n",'').split(sep, -1)
         
     | 
| 
       105 
     | 
    
         
            -
                  cols1 = parts1.length
         
     | 
| 
       106 
     | 
    
         
            -
                end
         
     | 
| 
       107 
108 
     | 
    
         | 
| 
       108 
     | 
    
         
            -
             
     | 
| 
       109 
     | 
    
         
            -
             
     | 
| 
       110 
     | 
    
         
            -
             
     | 
| 
      
 109 
     | 
    
         
            +
                  while key2.nil?
         
     | 
| 
      
 110 
     | 
    
         
            +
                    while (line2 = file2.gets) =~ /^#/
         
     | 
| 
      
 111 
     | 
    
         
            +
                      key_field2, *fields2 = line2.strip.sub('#','').split(sep)
         
     | 
| 
      
 112 
     | 
    
         
            +
                    end
         
     | 
| 
      
 113 
     | 
    
         
            +
                    key2, *parts2 = line2.sub("\n",'').split(sep, -1)
         
     | 
| 
      
 114 
     | 
    
         
            +
                    cols2 = parts2.length
         
     | 
| 
       111 
115 
     | 
    
         
             
                  end
         
     | 
| 
       112 
     | 
    
         
            -
                  key2, *parts2 = line2.sub("\n",'').split(sep, -1)
         
     | 
| 
       113 
     | 
    
         
            -
                  cols2 = parts2.length
         
     | 
| 
       114 
     | 
    
         
            -
                end
         
     | 
| 
       115 
116 
     | 
    
         | 
| 
       116 
     | 
    
         
            -
             
     | 
| 
      
 117 
     | 
    
         
            +
                  progress_monitor = Progress::Bar.new(size, 0, 100, "Merging fields") if monitor
         
     | 
| 
       117 
118 
     | 
    
         | 
| 
       118 
     | 
    
         
            -
             
     | 
| 
       119 
     | 
    
         
            -
             
     | 
| 
       120 
     | 
    
         
            -
             
     | 
| 
      
 119 
     | 
    
         
            +
                  entry_hash = options
         
     | 
| 
      
 120 
     | 
    
         
            +
                  entry_hash.delete :sep if entry_hash[:sep] == "\t"
         
     | 
| 
      
 121 
     | 
    
         
            +
                  output.puts TSV.header_lines key_field1, fields1 + fields2, entry_hash if key_field1 and fields1 and fields2
         
     | 
| 
       121 
122 
     | 
    
         | 
| 
       122 
     | 
    
         
            -
             
     | 
| 
       123 
     | 
    
         
            -
             
     | 
| 
       124 
     | 
    
         
            -
             
     | 
| 
       125 
     | 
    
         
            -
             
     | 
| 
       126 
     | 
    
         
            -
             
     | 
| 
       127 
     | 
    
         
            -
             
     | 
| 
       128 
     | 
    
         
            -
             
     | 
| 
       129 
     | 
    
         
            -
             
     | 
| 
       130 
     | 
    
         
            -
             
     | 
| 
       131 
     | 
    
         
            -
             
     | 
| 
       132 
     | 
    
         
            -
             
     | 
| 
       133 
     | 
    
         
            -
             
     | 
| 
       134 
     | 
    
         
            -
                  end
         
     | 
| 
       135 
     | 
    
         
            -
                  while (not done2 and key2 == key)
         
     | 
| 
       136 
     | 
    
         
            -
                    parts2.each_with_index do |part, i|
         
     | 
| 
       137 
     | 
    
         
            -
                      i += cols1
         
     | 
| 
       138 
     | 
    
         
            -
                      parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
         
     | 
| 
      
 123 
     | 
    
         
            +
                  key = key1 < key2 ? key1 : key2
         
     | 
| 
      
 124 
     | 
    
         
            +
                  parts = [""] * (cols1 + cols2)
         
     | 
| 
      
 125 
     | 
    
         
            +
                  while not (done1 and done2)
         
     | 
| 
      
 126 
     | 
    
         
            +
                    while (not done1 and key1 == key)
         
     | 
| 
      
 127 
     | 
    
         
            +
                      parts1.each_with_index do |part, i|
         
     | 
| 
      
 128 
     | 
    
         
            +
                        parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
         
     | 
| 
      
 129 
     | 
    
         
            +
                      end
         
     | 
| 
      
 130 
     | 
    
         
            +
                      key1 = nil
         
     | 
| 
      
 131 
     | 
    
         
            +
                      while key1.nil? and not done1
         
     | 
| 
      
 132 
     | 
    
         
            +
                        if file1.eof?; done1 = true; else key1, *parts1 = file1.gets.sub("\n",'').split(sep, -1) end
         
     | 
| 
      
 133 
     | 
    
         
            +
                      end
         
     | 
| 
      
 134 
     | 
    
         
            +
                      progress_monitor.tick if monitor
         
     | 
| 
       139 
135 
     | 
    
         
             
                    end
         
     | 
| 
       140 
     | 
    
         
            -
                    key2  
     | 
| 
       141 
     | 
    
         
            -
             
     | 
| 
       142 
     | 
    
         
            -
             
     | 
| 
      
 136 
     | 
    
         
            +
                    while (not done2 and key2 == key)
         
     | 
| 
      
 137 
     | 
    
         
            +
                      parts2.each_with_index do |part, i|
         
     | 
| 
      
 138 
     | 
    
         
            +
                        i += cols1
         
     | 
| 
      
 139 
     | 
    
         
            +
                        parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
         
     | 
| 
      
 140 
     | 
    
         
            +
                      end
         
     | 
| 
      
 141 
     | 
    
         
            +
                      key2 = nil
         
     | 
| 
      
 142 
     | 
    
         
            +
                      while key2.nil? and not done2
         
     | 
| 
      
 143 
     | 
    
         
            +
                        if file2.eof?; done2 = true; else key2, *parts2 = file2.gets.sub("\n",'').split(sep, -1) end
         
     | 
| 
      
 144 
     | 
    
         
            +
                      end
         
     | 
| 
       143 
145 
     | 
    
         
             
                    end
         
     | 
| 
       144 
     | 
    
         
            -
                  end
         
     | 
| 
       145 
146 
     | 
    
         | 
| 
       146 
     | 
    
         
            -
             
     | 
| 
       147 
     | 
    
         
            -
             
     | 
| 
      
 147 
     | 
    
         
            +
                    output.puts [key, parts].flatten * sep
         
     | 
| 
      
 148 
     | 
    
         
            +
                    parts = [""] * (cols1 + cols2)
         
     | 
| 
       148 
149 
     | 
    
         | 
| 
       149 
     | 
    
         
            -
             
     | 
| 
       150 
     | 
    
         
            -
             
     | 
| 
       151 
     | 
    
         
            -
             
     | 
| 
       152 
     | 
    
         
            -
             
     | 
| 
       153 
     | 
    
         
            -
             
     | 
| 
       154 
     | 
    
         
            -
             
     | 
| 
       155 
     | 
    
         
            -
             
     | 
| 
      
 150 
     | 
    
         
            +
                    case
         
     | 
| 
      
 151 
     | 
    
         
            +
                    when done1
         
     | 
| 
      
 152 
     | 
    
         
            +
                      key = key2
         
     | 
| 
      
 153 
     | 
    
         
            +
                    when done2
         
     | 
| 
      
 154 
     | 
    
         
            +
                      key = key1
         
     | 
| 
      
 155 
     | 
    
         
            +
                    else
         
     | 
| 
      
 156 
     | 
    
         
            +
                      key = key1 < key2 ? key1 : key2
         
     | 
| 
      
 157 
     | 
    
         
            +
                    end
         
     | 
| 
       156 
158 
     | 
    
         
             
                  end
         
     | 
| 
       157 
     | 
    
         
            -
                end
         
     | 
| 
       158 
159 
     | 
    
         | 
| 
       159 
     | 
    
         
            -
             
     | 
| 
      
 160 
     | 
    
         
            +
                  output.close
         
     | 
| 
      
 161 
     | 
    
         
            +
                  file1.join if file1.respond_to? :join
         
     | 
| 
      
 162 
     | 
    
         
            +
                  file2.join if file2.respond_to? :join
         
     | 
| 
      
 163 
     | 
    
         
            +
                rescue
         
     | 
| 
      
 164 
     | 
    
         
            +
                  file1.abort if file1.respond_to? :abort
         
     | 
| 
      
 165 
     | 
    
         
            +
                  file2.abort if file2.respond_to? :abort
         
     | 
| 
      
 166 
     | 
    
         
            +
                  file1.join if file1.respond_to? :join
         
     | 
| 
      
 167 
     | 
    
         
            +
                  file2.join if file2.respond_to? :join
         
     | 
| 
      
 168 
     | 
    
         
            +
                end
         
     | 
| 
       160 
169 
     | 
    
         
             
              end
         
     | 
| 
       161 
170 
     | 
    
         | 
| 
       162 
171 
     | 
    
         
             
              # Merge columns from different files
         
     | 
    
        data/lib/rbbt/tsv/dumper.rb
    CHANGED
    
    
| 
         @@ -283,7 +283,7 @@ module TSV 
     | 
|
| 
       283 
283 
     | 
    
         
             
              end
         
     | 
| 
       284 
284 
     | 
    
         | 
| 
       285 
285 
     | 
    
         
             
              def self.traverse_run(obj, threads, cpus, options = {}, &block)
         
     | 
| 
       286 
     | 
    
         
            -
                if threads.nil? and cpus.nil? 
     | 
| 
      
 286 
     | 
    
         
            +
                if ENV["RBBT_NO_MAP_REDUCE"] == "true" or (threads.nil? and cpus.nil?)
         
     | 
| 
       287 
287 
     | 
    
         
             
                  traverse_obj obj, options, &block
         
     | 
| 
       288 
288 
     | 
    
         
             
                else
         
     | 
| 
       289 
289 
     | 
    
         
             
                  if threads
         
     | 
    
        data/lib/rbbt/tsv/util.rb
    CHANGED
    
    | 
         @@ -40,7 +40,7 @@ module TSV 
     | 
|
| 
       40 
40 
     | 
    
         
             
                persist_options = Misc.pull_keys options, :persist
         
     | 
| 
       41 
41 
     | 
    
         | 
| 
       42 
42 
     | 
    
         
             
                filename = TSV === file ? file.filename : file
         
     | 
| 
       43 
     | 
    
         
            -
                 
     | 
| 
      
 43 
     | 
    
         
            +
                path = Persist.persist filename, :string, persist_options.merge(:no_load => true) do
         
     | 
| 
       44 
44 
     | 
    
         
             
                  tsv = TSV === file ? file : TSV.open(file)
         
     | 
| 
       45 
45 
     | 
    
         | 
| 
       46 
46 
     | 
    
         
             
                  text = ""
         
     | 
| 
         @@ -55,13 +55,11 @@ module TSV 
     | 
|
| 
       55 
55 
     | 
    
         
             
                  text
         
     | 
| 
       56 
56 
     | 
    
         
             
                end
         
     | 
| 
       57 
57 
     | 
    
         | 
| 
       58 
     | 
    
         
            -
                path = Persist.persistence_path(filename, persist_options)
         
     | 
| 
       59 
58 
     | 
    
         
             
                TmpFile.with_file(values.uniq * "\n") do |value_file|
         
     | 
| 
       60 
59 
     | 
    
         
             
                  cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' |cut -f 2 | sed 's/HEADERNOMATCH//' | sort|uniq -c|sed 's/^ *//;s/ /\t/'"
         
     | 
| 
       61 
60 
     | 
    
         
             
                  begin
         
     | 
| 
       62 
61 
     | 
    
         
             
                    TSV.open(CMD.cmd(cmd), :key_field => 1, :type => :single, :cast => :to_i)
         
     | 
| 
       63 
62 
     | 
    
         
             
                  rescue
         
     | 
| 
       64 
     | 
    
         
            -
                    Log.exception $!
         
     | 
| 
       65 
63 
     | 
    
         
             
                    TSV.setup({}, :type => :single, :cast => :to_i)
         
     | 
| 
       66 
64 
     | 
    
         
             
                  end
         
     | 
| 
       67 
65 
     | 
    
         
             
                end
         
     | 
    
        data/lib/rbbt/util/cmd.rb
    CHANGED
    
    | 
         @@ -3,8 +3,6 @@ require 'stringio' 
     | 
|
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
            module CMD
         
     | 
| 
       5 
5 
     | 
    
         | 
| 
       6 
     | 
    
         
            -
              class CMDError < StandardError; end
         
     | 
| 
       7 
     | 
    
         
            -
             
     | 
| 
       8 
6 
     | 
    
         
             
              module SmartIO 
         
     | 
| 
       9 
7 
     | 
    
         
             
                attr_accessor :pid, :cmd, :post, :in, :out, :err, :log
         
     | 
| 
       10 
8 
     | 
    
         
             
                def self.tie(io, pid = nil, cmd = "",  post = nil, sin = nil, out = nil, err = nil, log = true)
         
     | 
| 
         @@ -33,7 +31,7 @@ module CMD 
     | 
|
| 
       33 
31 
     | 
    
         | 
| 
       34 
32 
     | 
    
         
             
                    if $? and not $?.success?
         
     | 
| 
       35 
33 
     | 
    
         
             
                      Log.debug{"Raising exception" if log}
         
     | 
| 
       36 
     | 
    
         
            -
                      exception =  
     | 
| 
      
 34 
     | 
    
         
            +
                      exception = ProcessFailed.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
         
     | 
| 
       37 
35 
     | 
    
         
             
                      begin
         
     | 
| 
       38 
36 
     | 
    
         
             
                        original_close
         
     | 
| 
       39 
37 
     | 
    
         
             
                      ensure
         
     | 
| 
         @@ -160,9 +158,9 @@ module CMD 
     | 
|
| 
       160 
158 
     | 
    
         | 
| 
       161 
159 
     | 
    
         
             
                    exit(-1)
         
     | 
| 
       162 
160 
     | 
    
         
             
                  rescue Exception
         
     | 
| 
       163 
     | 
    
         
            -
                    Log.debug{ " 
     | 
| 
      
 161 
     | 
    
         
            +
                    Log.debug{ "ProcessFailed: #{$!.message}" } if log
         
     | 
| 
       164 
162 
     | 
    
         
             
                    Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
         
     | 
| 
       165 
     | 
    
         
            -
                    raise  
     | 
| 
      
 163 
     | 
    
         
            +
                    raise ProcessFailed, $!.message
         
     | 
| 
       166 
164 
     | 
    
         
             
                  end
         
     | 
| 
       167 
165 
     | 
    
         
             
                }
         
     | 
| 
       168 
166 
     | 
    
         | 
| 
         @@ -207,7 +205,8 @@ module CMD 
     | 
|
| 
       207 
205 
     | 
    
         
             
                    Thread.exit
         
     | 
| 
       208 
206 
     | 
    
         
             
                  end
         
     | 
| 
       209 
207 
     | 
    
         | 
| 
       210 
     | 
    
         
            -
                  SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
         
     | 
| 
      
 208 
     | 
    
         
            +
                  #SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
         
     | 
| 
      
 209 
     | 
    
         
            +
                  ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
         
     | 
| 
       211 
210 
     | 
    
         | 
| 
       212 
211 
     | 
    
         
             
                  sout
         
     | 
| 
       213 
212 
     | 
    
         
             
                else
         
     | 
| 
         @@ -222,12 +221,12 @@ module CMD 
     | 
|
| 
       222 
221 
     | 
    
         | 
| 
       223 
222 
     | 
    
         
             
                  out = StringIO.new sout.read
         
     | 
| 
       224 
223 
     | 
    
         
             
                  sout.close unless sout.closed?
         
     | 
| 
       225 
     | 
    
         
            -
                  SmartIO.tie out, pid, cmd, post, in_content, sin, serr
         
     | 
| 
      
 224 
     | 
    
         
            +
                  #SmartIO.tie out, pid, cmd, post, in_content, sin, serr
         
     | 
| 
       226 
225 
     | 
    
         | 
| 
       227 
226 
     | 
    
         
             
                  Process.waitpid pid
         
     | 
| 
       228 
227 
     | 
    
         | 
| 
       229 
228 
     | 
    
         
             
                  if not $?.success?
         
     | 
| 
       230 
     | 
    
         
            -
                    exception      =  
     | 
| 
      
 229 
     | 
    
         
            +
                    exception      = ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
         
     | 
| 
       231 
230 
     | 
    
         
             
                    raise exception
         
     | 
| 
       232 
231 
     | 
    
         
             
                  else
         
     | 
| 
       233 
232 
     | 
    
         
             
                    Log.log err, stderr if Integer === stderr and log
         
     | 
| 
         @@ -18,6 +18,7 @@ class RbbtProcessQueue 
     | 
|
| 
       18 
18 
     | 
    
         
             
                      end
         
     | 
| 
       19 
19 
     | 
    
         | 
| 
       20 
20 
     | 
    
         
             
                      Signal.trap(:INT){ raise Aborted; }
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
       21 
22 
     | 
    
         
             
                      loop do
         
     | 
| 
       22 
23 
     | 
    
         
             
                        p = @queue.pop
         
     | 
| 
       23 
24 
     | 
    
         
             
                        next if p.nil?
         
     | 
| 
         @@ -32,6 +33,7 @@ class RbbtProcessQueue 
     | 
|
| 
       32 
33 
     | 
    
         
             
                      Log.error "Worker #{Process.pid} aborted"
         
     | 
| 
       33 
34 
     | 
    
         
             
                      Kernel.exit! -1
         
     | 
| 
       34 
35 
     | 
    
         
             
                    rescue Exception
         
     | 
| 
      
 36 
     | 
    
         
            +
                      Log.exception $!
         
     | 
| 
       35 
37 
     | 
    
         
             
                      @callback_queue.push($!) if @callback_queue
         
     | 
| 
       36 
38 
     | 
    
         
             
                      Kernel.exit! -1
         
     | 
| 
       37 
39 
     | 
    
         
             
                    ensure
         
     | 
| 
         @@ -56,6 +56,7 @@ class RbbtProcessQueue 
     | 
|
| 
       56 
56 
     | 
    
         
             
                    end
         
     | 
| 
       57 
57 
     | 
    
         
             
                  rescue Aborted
         
     | 
| 
       58 
58 
     | 
    
         
             
                    @processes.each{|p| p.abort }
         
     | 
| 
      
 59 
     | 
    
         
            +
                    @processes.each{|p| p.join }
         
     | 
| 
       59 
60 
     | 
    
         
             
                    Log.error "Process monitor aborted"
         
     | 
| 
       60 
61 
     | 
    
         
             
                  rescue Exception
         
     | 
| 
       61 
62 
     | 
    
         
             
                    Log.error "Process monitor exception: #{$!.message}"
         
     | 
| 
         @@ -78,7 +79,8 @@ class RbbtProcessQueue 
     | 
|
| 
       78 
79 
     | 
    
         
             
              def join
         
     | 
| 
       79 
80 
     | 
    
         
             
                @processes.length.times do 
         
     | 
| 
       80 
81 
     | 
    
         
             
                  @queue.push ClosedStream.new
         
     | 
| 
       81 
     | 
    
         
            -
                end
         
     | 
| 
      
 82 
     | 
    
         
            +
                end if @process_monitor.alive?
         
     | 
| 
      
 83 
     | 
    
         
            +
             
     | 
| 
       82 
84 
     | 
    
         
             
                begin
         
     | 
| 
       83 
85 
     | 
    
         
             
                  @process_monitor.join
         
     | 
| 
       84 
86 
     | 
    
         
             
                  close_callback if @callback
         
     | 
| 
         @@ -104,8 +106,8 @@ class RbbtProcessQueue 
     | 
|
| 
       104 
106 
     | 
    
         
             
              end
         
     | 
| 
       105 
107 
     | 
    
         | 
| 
       106 
108 
     | 
    
         
             
              def abort
         
     | 
| 
       107 
     | 
    
         
            -
                @process_monitor.raise 
     | 
| 
       108 
     | 
    
         
            -
                @callback_thread.raise 
     | 
| 
      
 109 
     | 
    
         
            +
                @process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
         
     | 
| 
      
 110 
     | 
    
         
            +
                @callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
         
     | 
| 
       109 
111 
     | 
    
         
             
              end
         
     | 
| 
       110 
112 
     | 
    
         | 
| 
       111 
113 
     | 
    
         
             
              def process(*e)
         
     | 
| 
         @@ -1,14 +1,15 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            module ConcurrentStream
         
     | 
| 
       2 
     | 
    
         
            -
              attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
         
     | 
| 
      
 2 
     | 
    
         
            +
              attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :autojoin
         
     | 
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
              def self.setup(stream, options = {}, &block)
         
     | 
| 
       5 
     | 
    
         
            -
                threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
         
     | 
| 
      
 5 
     | 
    
         
            +
                threads, pids, callback, filename, autojoin = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin
         
     | 
| 
       6 
6 
     | 
    
         
             
                stream.extend ConcurrentStream unless ConcurrentStream === stream
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
8 
     | 
    
         
             
                stream.threads ||= []
         
     | 
| 
       9 
9 
     | 
    
         
             
                stream.pids ||= []
         
     | 
| 
       10 
10 
     | 
    
         
             
                stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
         
     | 
| 
       11 
11 
     | 
    
         
             
                stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
         
     | 
| 
      
 12 
     | 
    
         
            +
                stream.autojoin = autojoin
         
     | 
| 
       12 
13 
     | 
    
         | 
| 
       13 
14 
     | 
    
         
             
                callback = block if block_given?
         
     | 
| 
       14 
15 
     | 
    
         
             
                if stream.callback and callback
         
     | 
| 
         @@ -33,6 +34,7 @@ module ConcurrentStream 
     | 
|
| 
       33 
34 
     | 
    
         
             
                stream.callback = callback
         
     | 
| 
       34 
35 
     | 
    
         
             
                stream.abort_callback = abort_callback
         
     | 
| 
       35 
36 
     | 
    
         
             
                stream.filename = filename
         
     | 
| 
      
 37 
     | 
    
         
            +
                stream.autojoin = autojoin
         
     | 
| 
       36 
38 
     | 
    
         
             
                stream.joined = joined
         
     | 
| 
       37 
39 
     | 
    
         
             
              end
         
     | 
| 
       38 
40 
     | 
    
         | 
| 
         @@ -61,7 +63,7 @@ module ConcurrentStream 
     | 
|
| 
       61 
63 
     | 
    
         
             
                  @pids.each do |pid| 
         
     | 
| 
       62 
64 
     | 
    
         
             
                    begin
         
     | 
| 
       63 
65 
     | 
    
         
             
                      Process.waitpid(pid, Process::WUNTRACED)
         
     | 
| 
       64 
     | 
    
         
            -
                      raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
         
     | 
| 
      
 66 
     | 
    
         
            +
                      raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success?
         
     | 
| 
       65 
67 
     | 
    
         
             
                    rescue Errno::ECHILD
         
     | 
| 
       66 
68 
     | 
    
         
             
                    end
         
     | 
| 
       67 
69 
     | 
    
         
             
                  end 
         
     | 
| 
         @@ -102,4 +104,19 @@ module ConcurrentStream 
     | 
|
| 
       102 
104 
     | 
    
         
             
                @callback = nil
         
     | 
| 
       103 
105 
     | 
    
         
             
              end
         
     | 
| 
       104 
106 
     | 
    
         | 
| 
      
 107 
     | 
    
         
            +
              def read(*args)
         
     | 
| 
      
 108 
     | 
    
         
            +
                if autojoin
         
     | 
| 
      
 109 
     | 
    
         
            +
                  begin
         
     | 
| 
      
 110 
     | 
    
         
            +
                    super(*args)
         
     | 
| 
      
 111 
     | 
    
         
            +
                  rescue
         
     | 
| 
      
 112 
     | 
    
         
            +
                    self.abort
         
     | 
| 
      
 113 
     | 
    
         
            +
                    self.join 
         
     | 
| 
      
 114 
     | 
    
         
            +
                  ensure
         
     | 
| 
      
 115 
     | 
    
         
            +
                    self.join if self.closed? or self.eof? 
         
     | 
| 
      
 116 
     | 
    
         
            +
                  end
         
     | 
| 
      
 117 
     | 
    
         
            +
                else
         
     | 
| 
      
 118 
     | 
    
         
            +
                  super(*args)
         
     | 
| 
      
 119 
     | 
    
         
            +
                end
         
     | 
| 
      
 120 
     | 
    
         
            +
              end
         
     | 
| 
      
 121 
     | 
    
         
            +
             
     | 
| 
       105 
122 
     | 
    
         
             
            end
         
     | 
    
        data/lib/rbbt/util/open.rb
    CHANGED
    
    | 
         @@ -341,9 +341,14 @@ module Open 
     | 
|
| 
       341 
341 
     | 
    
         
             
                io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
         
     | 
| 
       342 
342 
     | 
    
         | 
| 
       343 
343 
     | 
    
         
             
                if block_given?
         
     | 
| 
       344 
     | 
    
         
            -
                   
     | 
| 
       345 
     | 
    
         
            -
             
     | 
| 
       346 
     | 
    
         
            -
                   
     | 
| 
      
 344 
     | 
    
         
            +
                  begin
         
     | 
| 
      
 345 
     | 
    
         
            +
                    return yield(io)
         
     | 
| 
      
 346 
     | 
    
         
            +
                  rescue
         
     | 
| 
      
 347 
     | 
    
         
            +
                    io.abort if io.respond_to? :abort
         
     | 
| 
      
 348 
     | 
    
         
            +
                    io.join if io.respond_to? :join
         
     | 
| 
      
 349 
     | 
    
         
            +
                  ensure
         
     | 
| 
      
 350 
     | 
    
         
            +
                    io.join if io.respond_to? :join
         
     | 
| 
      
 351 
     | 
    
         
            +
                  end
         
     | 
| 
       347 
352 
     | 
    
         
             
                else
         
     | 
| 
       348 
353 
     | 
    
         
             
                  io
         
     | 
| 
       349 
354 
     | 
    
         
             
                end
         
     | 
| 
         @@ -100,7 +100,6 @@ class Step 
     | 
|
| 
       100 
100 
     | 
    
         
             
                    dependency.relay_log self
         
     | 
| 
       101 
101 
     | 
    
         
             
                    dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
         
     | 
| 
       102 
102 
     | 
    
         
             
                    dependency.clean if dependency.streaming? and not dependency.running?
         
     | 
| 
       103 
     | 
    
         
            -
                    #dependency.run_dependencies(seen)
         
     | 
| 
       104 
103 
     | 
    
         
             
                    dependency.run(ENV["RBBT_NO_STREAM"] != 'true') unless dependency.result or dependency.done?
         
     | 
| 
       105 
104 
     | 
    
         
             
                    seen << dependency.path
         
     | 
| 
       106 
105 
     | 
    
         
             
                    seen.concat dependency.rec_dependencies.collect{|d| d.path} 
         
     | 
| 
         @@ -159,12 +158,13 @@ class Step 
     | 
|
| 
       159 
158 
     | 
    
         
             
                        raise $!
         
     | 
| 
       160 
159 
     | 
    
         
             
                      end
         
     | 
| 
       161 
160 
     | 
    
         | 
| 
       162 
     | 
    
         
            -
                       
     | 
| 
       163 
     | 
    
         
            -
             
     | 
| 
      
 161 
     | 
    
         
            +
                      if not no_load or ENV["RBBT_NO_STREAM"] == "true" 
         
     | 
| 
      
 162 
     | 
    
         
            +
                        result = prepare_result result, @task.description, info if IO === result 
         
     | 
| 
      
 163 
     | 
    
         
            +
                        result = prepare_result result.stream, @task.description, info if TSV::Dumper === result 
         
     | 
| 
      
 164 
     | 
    
         
            +
                      end
         
     | 
| 
       164 
165 
     | 
    
         | 
| 
       165 
166 
     | 
    
         
             
                      case result
         
     | 
| 
       166 
167 
     | 
    
         
             
                      when IO
         
     | 
| 
       167 
     | 
    
         
            -
                        result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
         
     | 
| 
       168 
168 
     | 
    
         | 
| 
       169 
169 
     | 
    
         
             
                        log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
         
     | 
| 
       170 
170 
     | 
    
         
             
                        ConcurrentStream.setup result do
         
     | 
    
        data/lib/rbbt/workflow/task.rb
    CHANGED
    
    | 
         @@ -44,11 +44,7 @@ module Task 
     | 
|
| 
       44 
44 
     | 
    
         
             
              def exec(*args)
         
     | 
| 
       45 
45 
     | 
    
         
             
                case
         
     | 
| 
       46 
46 
     | 
    
         
             
                when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
         
     | 
| 
       47 
     | 
    
         
            -
                   
     | 
| 
       48 
     | 
    
         
            -
                    self.call *take_input_values(IndiferentHash.setup(args.first))
         
     | 
| 
       49 
     | 
    
         
            -
                  ensure
         
     | 
| 
       50 
     | 
    
         
            -
                    purge_stream_cache
         
     | 
| 
       51 
     | 
    
         
            -
                  end
         
     | 
| 
      
 47 
     | 
    
         
            +
                  self.call *take_input_values(IndiferentHash.setup(args.first))
         
     | 
| 
       52 
48 
     | 
    
         
             
                else
         
     | 
| 
       53 
49 
     | 
    
         
             
                  self.call *args
         
     | 
| 
       54 
50 
     | 
    
         
             
                end
         
     | 
    
        data/test/rbbt/test_entity.rb
    CHANGED
    
    | 
         @@ -27,7 +27,7 @@ module ReversableString 
     | 
|
| 
       27 
27 
     | 
    
         
             
                self.reverse
         
     | 
| 
       28 
28 
     | 
    
         
             
              end
         
     | 
| 
       29 
29 
     | 
    
         | 
| 
       30 
     | 
    
         
            -
              property :reverse_text_ary_p => : 
     | 
| 
      
 30 
     | 
    
         
            +
              property :reverse_text_ary_p => :array2single do
         
     | 
| 
       31 
31 
     | 
    
         
             
                $count += 1
         
     | 
| 
       32 
32 
     | 
    
         
             
                self.collect{|s| s.reverse}
         
     | 
| 
       33 
33 
     | 
    
         
             
              end
         
     | 
| 
         @@ -103,7 +103,7 @@ class TestEntity < Test::Unit::TestCase 
     | 
|
| 
       103 
103 
     | 
    
         
             
                $count = 0
         
     | 
| 
       104 
104 
     | 
    
         | 
| 
       105 
105 
     | 
    
         
             
                assert_equal "2gnirtS", a.reverse_text_ary_p.last
         
     | 
| 
       106 
     | 
    
         
            -
                assert_equal "2gnirtS", a[1] 
     | 
| 
      
 106 
     | 
    
         
            +
                assert_equal "2gnirtS", a.collect{|e| e.reverse_text_ary_p }[1]
         
     | 
| 
       107 
107 
     | 
    
         
             
                assert_equal 1, $count
         
     | 
| 
       108 
108 
     | 
    
         
             
              end
         
     | 
| 
       109 
109 
     | 
    
         | 
| 
         @@ -317,9 +317,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase 
     | 
|
| 
       317 
317 
     | 
    
         
             
                    raise "STOP" if rand(100) < 1
         
     | 
| 
       318 
318 
     | 
    
         
             
                    [k,v]
         
     | 
| 
       319 
319 
     | 
    
         
             
                  end
         
     | 
| 
       320 
     | 
    
         
            -
                  dumper.stream 
     | 
| 
      
 320 
     | 
    
         
            +
                  stream = dumper.stream
         
     | 
| 
      
 321 
     | 
    
         
            +
                  stream.read
         
     | 
| 
      
 322 
     | 
    
         
            +
                  stream.join
         
     | 
| 
       321 
323 
     | 
    
         
             
                  rescue Exception
         
     | 
| 
       322 
324 
     | 
    
         
             
                    Log.exception $!
         
     | 
| 
      
 325 
     | 
    
         
            +
                    stream.abort if stream.respond_to? :abort
         
     | 
| 
      
 326 
     | 
    
         
            +
                    stream.join if stream.respond_to? :join
         
     | 
| 
       323 
327 
     | 
    
         
             
                    raise $!
         
     | 
| 
       324 
328 
     | 
    
         
             
                  end
         
     | 
| 
       325 
329 
     | 
    
         
             
                end
         
     | 
    
        data/test/rbbt/tsv/test_util.rb
    CHANGED
    
    
| 
         @@ -5,7 +5,7 @@ require 'rbbt/util/misc' 
     | 
|
| 
       5 
5 
     | 
    
         | 
| 
       6 
6 
     | 
    
         
             
            class TestMiscPipes < Test::Unit::TestCase
         
     | 
| 
       7 
7 
     | 
    
         | 
| 
       8 
     | 
    
         
            -
              def  
     | 
| 
      
 8 
     | 
    
         
            +
              def _test_collapse_stream
         
     | 
| 
       9 
9 
     | 
    
         
             
                text=<<-EOF
         
     | 
| 
       10 
10 
     | 
    
         
             
            row1 A B C
         
     | 
| 
       11 
11 
     | 
    
         
             
            row1 a b c
         
     | 
| 
         @@ -19,7 +19,7 @@ row2 aa bb cc 
     | 
|
| 
       19 
19 
     | 
    
         
             
                assert_equal ["BB", "bb"], tsv["row2"][1]
         
     | 
| 
       20 
20 
     | 
    
         
             
              end
         
     | 
| 
       21 
21 
     | 
    
         | 
| 
       22 
     | 
    
         
            -
              def  
     | 
| 
      
 22 
     | 
    
         
            +
              def _test_paste_stream
         
     | 
| 
       23 
23 
     | 
    
         
             
                text1=<<-EOF
         
     | 
| 
       24 
24 
     | 
    
         
             
            row1 A B C
         
     | 
| 
       25 
25 
     | 
    
         
             
            row2 AA BB CC
         
     | 
| 
         @@ -46,7 +46,7 @@ row3 ccc 
     | 
|
| 
       46 
46 
     | 
    
         
             
                assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
         
     | 
| 
       47 
47 
     | 
    
         
             
              end
         
     | 
| 
       48 
48 
     | 
    
         | 
| 
       49 
     | 
    
         
            -
              def  
     | 
| 
      
 49 
     | 
    
         
            +
              def _test_sort_stream
         
     | 
| 
       50 
50 
     | 
    
         
             
                text =<<-EOF
         
     | 
| 
       51 
51 
     | 
    
         
             
            #: :sep=" "
         
     | 
| 
       52 
52 
     | 
    
         
             
            #Row LabelA LabelB LabelC
         
     | 
| 
         @@ -60,7 +60,7 @@ row1 A B C 
     | 
|
| 
       60 
60 
     | 
    
         
             
                assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
         
     | 
| 
       61 
61 
     | 
    
         
             
              end
         
     | 
| 
       62 
62 
     | 
    
         | 
| 
       63 
     | 
    
         
            -
              def  
     | 
| 
      
 63 
     | 
    
         
            +
              def _test_dup_stream
         
     | 
| 
       64 
64 
     | 
    
         
             
                text =<<-EOF
         
     | 
| 
       65 
65 
     | 
    
         
             
            #: :sep=" "
         
     | 
| 
       66 
66 
     | 
    
         
             
            #Row LabelA LabelB LabelC
         
     | 
    
        data/test/rbbt/util/test_R.rb
    CHANGED
    
    | 
         @@ -2,11 +2,11 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe 
     | 
|
| 
       2 
2 
     | 
    
         
             
            require 'rbbt/util/R'
         
     | 
| 
       3 
3 
     | 
    
         | 
| 
       4 
4 
     | 
    
         
             
            class TestR < Test::Unit::TestCase
         
     | 
| 
       5 
     | 
    
         
            -
              def  
     | 
| 
      
 5 
     | 
    
         
            +
              def test_sum
         
     | 
| 
       6 
6 
     | 
    
         
             
                assert_equal "6", R.run('cat(3+3)').read.split(/\n/).last
         
     | 
| 
       7 
7 
     | 
    
         
             
              end
         
     | 
| 
       8 
8 
     | 
    
         | 
| 
       9 
     | 
    
         
            -
              def  
     | 
| 
      
 9 
     | 
    
         
            +
              def test_tsv_R
         
     | 
| 
       10 
10 
     | 
    
         
             
                tsv = TSV.setup({:a => 1, :b => 2})
         
     | 
| 
       11 
11 
     | 
    
         
             
                tsv2 = tsv.R <<-EOF
         
     | 
| 
       12 
12 
     | 
    
         
             
            data = data + 1
         
     | 
    
        data/test/rbbt/util/test_cmd.rb
    CHANGED
    
    | 
         @@ -29,17 +29,17 @@ class TestCmd < Test::Unit::TestCase 
     | 
|
| 
       29 
29 
     | 
    
         
             
              end
         
     | 
| 
       30 
30 
     | 
    
         | 
| 
       31 
31 
     | 
    
         
             
              def test_error
         
     | 
| 
       32 
     | 
    
         
            -
                assert_raise  
     | 
| 
       33 
     | 
    
         
            -
                assert_raise  
     | 
| 
      
 32 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('fake-command') end
         
     | 
| 
      
 33 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('ls -fake_option') end
         
     | 
| 
       34 
34 
     | 
    
         | 
| 
       35 
     | 
    
         
            -
                assert_raise  
     | 
| 
       36 
     | 
    
         
            -
                assert_raise  
     | 
| 
      
 35 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true) end
         
     | 
| 
      
 36 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true) end
         
     | 
| 
       37 
37 
     | 
    
         | 
| 
       38 
     | 
    
         
            -
                assert_nothing_raised  
     | 
| 
       39 
     | 
    
         
            -
                assert_nothing_raised  
     | 
| 
      
 38 
     | 
    
         
            +
                assert_nothing_raised ProcessFailed do CMD.cmd('fake-command', :stderr => false, :pipe => true) end
         
     | 
| 
      
 39 
     | 
    
         
            +
                assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :stderr => false, :pipe => true) end
         
     | 
| 
       40 
40 
     | 
    
         | 
| 
       41 
     | 
    
         
            -
                assert_raise  
     | 
| 
       42 
     | 
    
         
            -
                assert_raise  
     | 
| 
      
 41 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
         
     | 
| 
      
 42 
     | 
    
         
            +
                assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
         
     | 
| 
       43 
43 
     | 
    
         
             
              end
         
     | 
| 
       44 
44 
     | 
    
         | 
| 
       45 
45 
     | 
    
         
             
              def test_pipes
         
     | 
| 
         @@ -58,10 +58,7 @@ line33 
     | 
|
| 
       58 
58 
     | 
    
         
             
                  gz = CMD.cmd("gunzip", :in => File.open(file + '.gz'), :pipe => true)
         
     | 
| 
       59 
59 
     | 
    
         
             
                  io = CMD.cmd('tail -n 10', :in => gz, :pipe => true)
         
     | 
| 
       60 
60 
     | 
    
         
             
                  assert_equal 10, io.read.split(/\n/).length
         
     | 
| 
       61 
     | 
    
         
            -
             
     | 
| 
       62 
     | 
    
         
            -
             
     | 
| 
       63 
61 
     | 
    
         
             
                end
         
     | 
| 
       64 
     | 
    
         
            -
             
     | 
| 
       65 
62 
     | 
    
         
             
              end
         
     | 
| 
       66 
63 
     | 
    
         | 
| 
       67 
64 
     | 
    
         
             
            end
         
     | 
    
        data/test/rbbt/util/test_misc.rb
    CHANGED
    
    | 
         @@ -115,15 +115,6 @@ eum fugiat quo voluptas nulla pariatur?" 
     | 
|
| 
       115 
115 
     | 
    
         
             
                assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
         
     | 
| 
       116 
116 
     | 
    
         
             
              end
         
     | 
| 
       117 
117 
     | 
    
         | 
| 
       118 
     | 
    
         
            -
              def test_pipe
         
     | 
| 
       119 
     | 
    
         
            -
                sout, sin = Misc.pipe
         
     | 
| 
       120 
     | 
    
         
            -
                assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
         
     | 
| 
       121 
     | 
    
         
            -
                sin.close
         
     | 
| 
       122 
     | 
    
         
            -
                assert sout.eof?
         
     | 
| 
       123 
     | 
    
         
            -
                Misc.purge_pipes
         
     | 
| 
       124 
     | 
    
         
            -
                assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
         
     | 
| 
       125 
     | 
    
         
            -
              end
         
     | 
| 
       126 
     | 
    
         
            -
             
     | 
| 
       127 
118 
     | 
    
         
             
              def test_pipe_fork
         
     | 
| 
       128 
119 
     | 
    
         
             
                sout, sin = Misc.pipe
         
     | 
| 
       129 
120 
     | 
    
         
             
                pid = Process.fork do
         
     | 
    
        data/test/rbbt/util/test_open.rb
    CHANGED
    
    
| 
         @@ -62,7 +62,7 @@ class TestStep < Test::Unit::TestCase 
     | 
|
| 
       62 
62 
     | 
    
         
             
                  step2 = Step.new tmpfile + 'step2', task2, [], [step1]
         
     | 
| 
       63 
63 
     | 
    
         | 
| 
       64 
64 
     | 
    
         
             
                  step2.run
         
     | 
| 
       65 
     | 
    
         
            -
                  assert step2.messages.include? " 
     | 
| 
      
 65 
     | 
    
         
            +
                  assert step2.messages.include? "Starting Task1"
         
     | 
| 
       66 
66 
     | 
    
         
             
                end
         
     | 
| 
       67 
67 
     | 
    
         
             
              end
         
     | 
| 
       68 
68 
     | 
    
         | 
| 
         @@ -83,7 +83,8 @@ class TestStep < Test::Unit::TestCase 
     | 
|
| 
       83 
83 
     | 
    
         | 
| 
       84 
84 
     | 
    
         
             
                  Step.log_relay_step = step2
         
     | 
| 
       85 
85 
     | 
    
         
             
                  step2.run
         
     | 
| 
       86 
     | 
    
         
            -
             
     | 
| 
      
 86 
     | 
    
         
            +
             
     | 
| 
      
 87 
     | 
    
         
            +
                  assert step2.messages.include? "Starting Task1"
         
     | 
| 
       87 
88 
     | 
    
         
             
                end
         
     | 
| 
       88 
89 
     | 
    
         
             
              end
         
     | 
| 
       89 
90 
     | 
    
         | 
| 
         @@ -99,7 +100,7 @@ class TestStep < Test::Unit::TestCase 
     | 
|
| 
       99 
100 
     | 
    
         
             
              end
         
     | 
| 
       100 
101 
     | 
    
         | 
| 
       101 
102 
     | 
    
         | 
| 
       102 
     | 
    
         
            -
              def  
     | 
| 
      
 103 
     | 
    
         
            +
              def __test_fork
         
     | 
| 
       103 
104 
     | 
    
         
             
                TmpFile.with_file do |lock|
         
     | 
| 
       104 
105 
     | 
    
         
             
                  task  = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
         
     | 
| 
       105 
106 
     | 
    
         
             
                  TmpFile.with_file do |tmp|
         
     | 
| 
         @@ -115,19 +116,20 @@ class TestStep < Test::Unit::TestCase 
     | 
|
| 
       115 
116 
     | 
    
         
             
                end
         
     | 
| 
       116 
117 
     | 
    
         
             
              end
         
     | 
| 
       117 
118 
     | 
    
         | 
| 
       118 
     | 
    
         
            -
              def  
     | 
| 
      
 119 
     | 
    
         
            +
              def __test_abort
         
     | 
| 
       119 
120 
     | 
    
         
             
                TmpFile.with_file do |lock|
         
     | 
| 
       120 
121 
     | 
    
         
             
                  task  = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
         
     | 
| 
       121 
122 
     | 
    
         
             
                  TmpFile.with_file do |tmp|
         
     | 
| 
       122 
123 
     | 
    
         
             
                    step = Step.new tmp, task
         
     | 
| 
       123 
124 
     | 
    
         
             
                    job = step.fork
         
     | 
| 
       124 
125 
     | 
    
         
             
                    assert !job.done?
         
     | 
| 
       125 
     | 
    
         
            -
                     
     | 
| 
       126 
     | 
    
         
            -
                     
     | 
| 
       127 
     | 
    
         
            -
                     
     | 
| 
      
 126 
     | 
    
         
            +
                    step.clean.fork 
         
     | 
| 
      
 127 
     | 
    
         
            +
                    job.abort 
         
     | 
| 
      
 128 
     | 
    
         
            +
                    assert_equal :aborted, job.status
         
     | 
| 
       128 
129 
     | 
    
         
             
                    Open.write(lock, "open")
         
     | 
| 
      
 130 
     | 
    
         
            +
                    job.clean.fork 
         
     | 
| 
       129 
131 
     | 
    
         
             
                    job.join
         
     | 
| 
       130 
     | 
    
         
            -
                    assert job. 
     | 
| 
      
 132 
     | 
    
         
            +
                    assert job.done?
         
     | 
| 
       131 
133 
     | 
    
         
             
                  end
         
     | 
| 
       132 
134 
     | 
    
         
             
                end
         
     | 
| 
       133 
135 
     | 
    
         
             
              end
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: rbbt-util
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 5.12. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 5.12.3
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - Miguel Vazquez
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire: 
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: bin
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date: 2014-04- 
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2014-04-24 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies:
         
     | 
| 
       13 
13 
     | 
    
         
             
            - !ruby/object:Gem::Dependency
         
     | 
| 
       14 
14 
     | 
    
         
             
              name: rake
         
     |