scout-gear 7.2.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +51 -6
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +33 -29
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/exceptions.rb +1 -0
  10. data/lib/scout/log/color.rb +4 -2
  11. data/lib/scout/log/progress/report.rb +1 -1
  12. data/lib/scout/log/progress/util.rb +71 -2
  13. data/lib/scout/log/progress.rb +1 -1
  14. data/lib/scout/log/trap.rb +107 -0
  15. data/lib/scout/log.rb +56 -21
  16. data/lib/scout/meta_extension.rb +13 -6
  17. data/lib/scout/misc/digest.rb +1 -1
  18. data/lib/scout/misc/format.rb +12 -0
  19. data/lib/scout/misc/helper.rb +31 -0
  20. data/lib/scout/misc/insist.rb +1 -1
  21. data/lib/scout/misc/monitor.rb +12 -1
  22. data/lib/scout/misc/system.rb +10 -0
  23. data/lib/scout/misc.rb +1 -0
  24. data/lib/scout/named_array.rb +65 -3
  25. data/lib/scout/open/lock/lockfile.rb +587 -0
  26. data/lib/scout/open/lock.rb +28 -2
  27. data/lib/scout/open/remote.rb +4 -0
  28. data/lib/scout/open/stream.rb +111 -42
  29. data/lib/scout/open/util.rb +13 -3
  30. data/lib/scout/path/find.rb +9 -1
  31. data/lib/scout/path/util.rb +35 -0
  32. data/lib/scout/persist/serialize.rb +18 -5
  33. data/lib/scout/persist.rb +60 -30
  34. data/lib/scout/resource/path.rb +53 -0
  35. data/lib/scout/resource/produce.rb +0 -8
  36. data/lib/scout/resource/util.rb +2 -1
  37. data/lib/scout/semaphore.rb +8 -1
  38. data/lib/scout/tmpfile.rb +7 -8
  39. data/lib/scout/tsv/attach.rb +177 -0
  40. data/lib/scout/tsv/change_id.rb +40 -0
  41. data/lib/scout/tsv/dumper.rb +85 -54
  42. data/lib/scout/tsv/index.rb +188 -20
  43. data/lib/scout/tsv/open.rb +182 -0
  44. data/lib/scout/tsv/parser.rb +200 -118
  45. data/lib/scout/tsv/path.rb +5 -6
  46. data/lib/scout/tsv/persist/adapter.rb +26 -37
  47. data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
  48. data/lib/scout/tsv/persist/serialize.rb +117 -0
  49. data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
  50. data/lib/scout/tsv/persist.rb +4 -2
  51. data/lib/scout/tsv/transformer.rb +141 -0
  52. data/lib/scout/tsv/traverse.rb +136 -37
  53. data/lib/scout/tsv/util/filter.rb +312 -0
  54. data/lib/scout/tsv/util/process.rb +73 -0
  55. data/lib/scout/tsv/util/reorder.rb +81 -0
  56. data/lib/scout/tsv/util/select.rb +265 -0
  57. data/lib/scout/tsv/util/unzip.rb +86 -0
  58. data/lib/scout/tsv/util.rb +126 -19
  59. data/lib/scout/tsv.rb +28 -5
  60. data/lib/scout/work_queue/socket.rb +6 -1
  61. data/lib/scout/work_queue/worker.rb +5 -2
  62. data/lib/scout/work_queue.rb +15 -8
  63. data/lib/scout/workflow/definition.rb +29 -2
  64. data/lib/scout/workflow/step/dependencies.rb +24 -4
  65. data/lib/scout/workflow/step/info.rb +40 -5
  66. data/lib/scout/workflow/step/progress.rb +14 -0
  67. data/lib/scout/workflow/step/provenance.rb +8 -7
  68. data/lib/scout/workflow/step/status.rb +45 -0
  69. data/lib/scout/workflow/step.rb +104 -33
  70. data/lib/scout/workflow/task/inputs.rb +14 -20
  71. data/lib/scout/workflow/task.rb +86 -47
  72. data/lib/scout/workflow/usage.rb +10 -6
  73. data/scout-gear.gemspec +30 -3
  74. data/scout_commands/workflow/task +37 -9
  75. data/scout_commands/workflow/task_old +2 -2
  76. data/test/scout/open/test_stream.rb +61 -59
  77. data/test/scout/path/test_find.rb +10 -1
  78. data/test/scout/resource/test_produce.rb +15 -0
  79. data/test/scout/test_meta_extension.rb +25 -0
  80. data/test/scout/test_named_array.rb +18 -0
  81. data/test/scout/test_persist.rb +67 -0
  82. data/test/scout/test_tmpfile.rb +1 -1
  83. data/test/scout/test_tsv.rb +222 -3
  84. data/test/scout/test_work_queue.rb +21 -18
  85. data/test/scout/tsv/persist/test_adapter.rb +11 -1
  86. data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
  87. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  88. data/test/scout/tsv/test_attach.rb +227 -0
  89. data/test/scout/tsv/test_change_id.rb +98 -0
  90. data/test/scout/tsv/test_dumper.rb +1 -1
  91. data/test/scout/tsv/test_index.rb +127 -3
  92. data/test/scout/tsv/test_open.rb +167 -0
  93. data/test/scout/tsv/test_parser.rb +45 -3
  94. data/test/scout/tsv/test_persist.rb +9 -0
  95. data/test/scout/tsv/test_transformer.rb +108 -0
  96. data/test/scout/tsv/test_traverse.rb +195 -3
  97. data/test/scout/tsv/test_util.rb +24 -0
  98. data/test/scout/tsv/util/test_filter.rb +188 -0
  99. data/test/scout/tsv/util/test_process.rb +47 -0
  100. data/test/scout/tsv/util/test_reorder.rb +94 -0
  101. data/test/scout/tsv/util/test_select.rb +58 -0
  102. data/test/scout/tsv/util/test_unzip.rb +112 -0
  103. data/test/scout/work_queue/test_socket.rb +0 -1
  104. data/test/scout/work_queue/test_worker.rb +63 -6
  105. data/test/scout/workflow/step/test_load.rb +3 -3
  106. data/test/scout/workflow/step/test_status.rb +31 -0
  107. data/test/scout/workflow/task/test_inputs.rb +14 -14
  108. data/test/scout/workflow/test_step.rb +13 -13
  109. data/test/scout/workflow/test_task.rb +168 -32
  110. data/test/scout/workflow/test_usage.rb +33 -6
  111. data/test/test_helper.rb +3 -1
  112. metadata +29 -2
@@ -0,0 +1,94 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+
6
+ class TestReorder < Test::Unit::TestCase
7
+ def test_reorder
8
+ content =<<-'EOF'
9
+ #: :sep=/\s+/#:type=:double
10
+ #Id ValueA ValueB OtherID
11
+ row1 a1|a2 b1|b2 Id1|Id2
12
+ row2 A1|A3 B1|B3 Id1|Id3
13
+ EOF
14
+
15
+ tsv = TmpFile.with_file(content) do |filename|
16
+ TSV.open(filename)
17
+ end
18
+
19
+ r = tsv.reorder "OtherID", %w(ValueB Id)
20
+
21
+ assert_equal %w(row1 row2), r["Id1"]["Id"]
22
+ assert_equal %w(row2), r["Id3"]["Id"]
23
+ end
24
+
25
+ def test_reorder_list
26
+ content =<<-'EOF'
27
+ #: :sep=/\s+/#:type=:list
28
+ #Id ValueA ValueB OtherID
29
+ row1 a1 b1 Id1
30
+ row2 A1 B1 Id1
31
+ EOF
32
+
33
+ tsv = TmpFile.with_file(content) do |filename|
34
+ TSV.open(filename)
35
+ end
36
+
37
+ r = tsv.reorder "ValueB"
38
+
39
+ assert_equal "row1", r["b1"]["Id"]
40
+ assert_equal "row2", r["B1"]["Id"]
41
+ end
42
+
43
+ def test_reorder_single
44
+ content =<<-'EOF'
45
+ #: :sep=/\s+/#:type=:single
46
+ #Id ValueA
47
+ row1 a1
48
+ row2 A1
49
+ EOF
50
+
51
+ tsv = TmpFile.with_file(content) do |filename|
52
+ TSV.open(filename)
53
+ end
54
+
55
+ r = tsv.reorder "ValueA"
56
+
57
+ assert_equal "row1", r["a1"]
58
+ assert_equal "row2", r["A1"]
59
+ end
60
+
61
+ def test_transpose
62
+ content =<<-EOF
63
+ #: :type=:list
64
+ #Row vA vB vID
65
+ row1 a b Id1
66
+ row2 A B Id3
67
+ row3 a C Id4
68
+ EOF
69
+
70
+ TmpFile.with_file(content) do |filename|
71
+ tsv = TSV.open(filename, :sep => /\s+/)
72
+
73
+ assert_equal %w(vA vB vID), tsv.transpose("Values").keys
74
+ assert_equal %w(Id1 Id3 Id4), tsv.transpose("Values")["vID"]
75
+ end
76
+ end
77
+
78
+ def test_column
79
+ content =<<-EOF
80
+ #Id ValueA ValueB ValueC
81
+ rowA A|AA B|BB C|CC
82
+ rowa a|aa b|BB C|CC
83
+ EOF
84
+
85
+ TmpFile.with_file(content) do |filename|
86
+ tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
87
+ tsv = tsv.column("ValueA", cast: :downcase)
88
+ assert_equal %w(a aa), tsv["rowA"]
89
+ assert_equal %w(a aa), tsv["rowa"]
90
+ end
91
+ end
92
+
93
+ end
94
+
@@ -0,0 +1,58 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestTSVSelect < Test::Unit::TestCase
6
+ def test_select
7
+ content =<<-'EOF'
8
+ #: :sep=/\s+/#:type=:double
9
+ #Id ValueA ValueB OtherID
10
+ row1 a|aa|aaa b Id1|Id2
11
+ row2 A B Id3
12
+ row2 AA BB Id33
13
+ EOF
14
+
15
+ tsv = TmpFile.with_file(content) do |filename|
16
+ TSV.open(filename, :persist => true)
17
+ end
18
+
19
+ s = tsv.select do |k,v|
20
+ k.include? "2"
21
+ end
22
+
23
+ assert_equal ['row2'], s.keys
24
+ end
25
+
26
+ def test_select_values
27
+ content =<<-'EOF'
28
+ #: :sep=" "
29
+ #ID ValueA ValueB
30
+ row1 A1|A11 B1|B11
31
+ row2 A2|A22 B2|B22
32
+ EOF
33
+
34
+ tsv = TSV.open(content)
35
+
36
+ assert TSV.select("row1", tsv["row1"], "A1")
37
+ refute TSV.select("row2", tsv["row2"], "A1")
38
+ assert TSV.select("row2", tsv["row2"], "A2")
39
+
40
+ assert TSV.select("row1", tsv["row1"], "B1")
41
+ refute TSV.select("row2", tsv["row2"], "B1")
42
+
43
+ refute TSV.select("row1", tsv["row1"], "B1", invert: true)
44
+ assert TSV.select("row2", tsv["row2"], "B1", invert: true)
45
+
46
+ assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"})
47
+ refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"})
48
+
49
+ tsv.with_unnamed do
50
+ assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"}, fields: tsv.fields)
51
+ refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"}, fields: tsv.fields)
52
+ end
53
+
54
+ assert TSV.select("row1", tsv["row1"], "B1", field: "ValueB")
55
+ refute TSV.select("row2", tsv["row2"], "B1", field: "ValueB")
56
+ end
57
+ end
58
+
@@ -0,0 +1,112 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestTSVUnzip < Test::Unit::TestCase
6
+ def test_unzip
7
+ content1 =<<-EOF
8
+ #: :sep=" "
9
+ #ID ValueA ValueB
10
+ row1 A1|A11 B1|B11
11
+ row2 A2|A22 B2|B22
12
+ EOF
13
+
14
+ tsv = TSV.open(content1)
15
+ unzip = tsv.unzip("ValueA", delete: true)
16
+ assert_equal "B1", unzip["row1:A1"]["ValueB"]
17
+ assert_equal "B22", unzip["row2:A22"]["ValueB"]
18
+ end
19
+
20
+ def test_unzip_list
21
+ content1 =<<-EOF
22
+ #: :sep=" "#:type=:list
23
+ #ID ValueA ValueB
24
+ row1 A1 B1
25
+ row2 A2 B2
26
+ EOF
27
+
28
+ tsv = TSV.open(content1)
29
+ unzip = tsv.unzip("ValueA", delete: true)
30
+ assert_equal "B1", unzip["row1:A1"]["ValueB"]
31
+ assert_equal "B2", unzip["row2:A2"]["ValueB"]
32
+ end
33
+
34
+ def test_unzip_merge
35
+ content1 =<<-EOF
36
+ #: :sep=" "
37
+ #ID ValueA ValueB
38
+ row1 A1|A11|A11 B1|B11|B11.1
39
+ row2 A2|A22|A22 B2|B22|B22.2
40
+ EOF
41
+
42
+ tsv = TSV.open(content1)
43
+ unzip = tsv.unzip("ValueA", delete: true, merge: true)
44
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
45
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
46
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
47
+ end
48
+
49
+ def test_unzip_merge_stream
50
+ content1 =<<-EOF
51
+ #: :sep=" "
52
+ #ID ValueA ValueB
53
+ row1 A1|A11|A11 B1|B11|B11.1
54
+ row2 A2|A22|A22 B2|B22|B22.2
55
+ EOF
56
+
57
+ tsv = TSV.open(content1)
58
+ unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
59
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
60
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
61
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
62
+ end
63
+
64
+ def test_unzip_merge_stream_parser
65
+ content1 =<<-EOF
66
+ #: :sep=" "
67
+ #ID ValueA ValueB
68
+ row1 A1|A11|A11 B1|B11|B11.1
69
+ row2 A2|A22|A22 B2|B22|B22.2
70
+ EOF
71
+
72
+ TmpFile.with_file(content1) do |filename|
73
+ unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
74
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
75
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
76
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
77
+ end
78
+ end
79
+
80
+ def test_unzip_merge_stream_parser_one2many
81
+ content1 =<<-EOF
82
+ #: :sep=" "
83
+ #ID ValueA ValueB
84
+ row1 A1|A11|A11 B1|B11|B11.1
85
+ row2 A2|A22|A22 B2|B22|B22.2
86
+ EOF
87
+
88
+ TmpFile.with_file(content1) do |filename|
89
+ unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
90
+ assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
91
+ assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
92
+ end
93
+ end
94
+
95
+
96
+ def test_unzip_target
97
+ content1 =<<-EOF
98
+ #: :sep=" "
99
+ #ID ValueA ValueB
100
+ row1 A1|A11|A11 B1|B11|B11.1
101
+ row2 A2|A22|A22 B2|B22|B22.2
102
+ EOF
103
+
104
+ tsv = TSV.open(content1)
105
+ unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
106
+ tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
107
+ assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
108
+ assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
109
+ assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
110
+ end
111
+ end
112
+
@@ -22,7 +22,6 @@ class TestSocket < Test::Unit::TestCase
22
22
  end
23
23
 
24
24
  def __test_speed
25
- sss 0
26
25
  socket = WorkQueue::Socket.new
27
26
 
28
27
  num = 10_000
@@ -16,27 +16,84 @@ class TestQueueWorker < Test::Unit::TestCase
16
16
  end
17
17
  end
18
18
 
19
- def test_semaphore
20
- ScoutSemaphore.with_semaphore 1 do |sem|
19
+ def test_semaphore_pipe
20
+
21
+ 2.times do
22
+ num_lines = 10
23
+ num_workers = 100
21
24
 
22
25
  TmpFile.with_file do |outfile|
23
- 2.times do
26
+ Open.rm(outfile)
27
+ ScoutSemaphore.with_semaphore 1 do |sem|
24
28
  sout = Open.open_pipe do |sin|
25
- workers = 100.times.collect{ WorkQueue::Worker.new }
29
+ workers = num_workers.times.collect{ WorkQueue::Worker.new }
26
30
  workers.each do |w|
27
31
  w.run do
28
32
  ScoutSemaphore.synchronize(sem) do
29
- 10.times do
30
- sin.puts Process.pid
33
+ sin.puts "Start - #{Process.pid}"
34
+ num_lines.times do |i|
35
+ sin.puts "line-#{i}-#{Process.pid}"
31
36
  end
37
+ sin.puts "End - #{Process.pid}"
32
38
  end
33
39
  end
34
40
  end
41
+ sin.close
35
42
 
36
43
  WorkQueue::Worker.join(workers)
37
44
  end
45
+
38
46
  Open.consume_stream(sout, false, outfile)
47
+ txt = Open.read(outfile)
48
+ pid_list = txt.split("\n")
49
+
50
+ assert_equal (num_lines + 2) * num_workers, pid_list.length
51
+
52
+ assert_nothing_raised do
53
+ seen = []
54
+ current = nil
55
+ pid_list.each do |pid|
56
+ if pid != current
57
+ raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
58
+ end
59
+ current = pid
60
+ seen << pid
61
+ end
62
+ end
63
+ end
64
+ end
65
+ end
66
+ end
67
+ def test_semaphore
68
+
69
+ 2.times do
70
+ num_lines = 10
71
+ num_workers = 500
72
+
73
+ TmpFile.with_file do |outfile|
74
+ Open.rm(outfile)
75
+ ScoutSemaphore.with_semaphore 1 do |sem|
76
+ workers = num_workers.times.collect{ WorkQueue::Worker.new }
77
+ Open.touch(outfile)
78
+ workers.each do |w|
79
+ w.run do
80
+ ScoutSemaphore.synchronize(sem) do
81
+ sin = Open.open(outfile, :mode => 'a')
82
+ sin.puts "Start - #{Process.pid}"
83
+ num_lines.times do |i|
84
+ sin.puts "line-#{i}-#{Process.pid}"
85
+ end
86
+ sin.puts "End - #{Process.pid}"
87
+ sin.close
88
+ end
89
+ end
90
+ end
91
+
92
+ WorkQueue::Worker.join(workers)
93
+
94
+
39
95
  pid_list = Open.read(outfile).split("\n")
96
+ assert_equal (num_lines + 2) * num_workers, pid_list.length
40
97
 
41
98
  assert_nothing_raised do
42
99
  seen = []
@@ -41,7 +41,7 @@ class TestStepLoad < Test::Unit::TestCase
41
41
  end
42
42
  end
43
43
 
44
- step2 = wf.job(:step2, :input1 => "TEST")
44
+ step2 = wf.job(:step2, nil, :input1 => "TEST_STRING")
45
45
  step1 = step2.step(:step1)
46
46
 
47
47
  step2.run
@@ -53,8 +53,8 @@ class TestStepLoad < Test::Unit::TestCase
53
53
  Open.mv step1.info_file, dir.var.jobs.RelocateWorkflow.step1[File.basename(step1.info_file)]
54
54
 
55
55
  new_step2 = Step.load(step2.path)
56
- assert_equal "TEST".reverse, new_step2.load
57
- assert_equal "TEST", new_step2.dependencies.first.load
56
+ assert_equal "TEST_STRING".reverse, new_step2.load
57
+ assert_equal "TEST_STRING", new_step2.dependencies.first.load
58
58
  end
59
59
  end
60
60
 
@@ -0,0 +1,31 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestStepStatus < Test::Unit::TestCase
5
+ def test_dependency
6
+ tmpfile = tmpdir.test_step
7
+ step1 = Step.new tmpfile.step1, ["12"] do |s|
8
+ s.length
9
+ end
10
+
11
+ step2 = Step.new tmpfile.step2 do
12
+ step1 = dependencies.first
13
+ step1.inputs.first + " has " + step1.load.to_s + " characters"
14
+ end
15
+
16
+ step2.dependencies = [step1]
17
+
18
+ sss 0
19
+ Misc.with_env "SCOUT_UPDATE", "true" do
20
+ step2.run
21
+ assert step2.updated?
22
+
23
+ sleep 0.1
24
+ step1.clean
25
+ step1.run
26
+ refute step2.updated?
27
+ end
28
+ end
29
+
30
+ end
31
+
@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
71
71
  task = self.example_task
72
72
 
73
73
  TmpFile.with_file("2\n3") do |integer_array_file|
74
- assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
75
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
74
+ assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
75
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
76
76
  end
77
77
  end
78
78
 
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
83
83
 
84
84
  TmpFile.with_file("1\n2") do |integer_array_file|
85
85
  hash1 = Open.open(integer_array_file) do |f|
86
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
86
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
87
87
  end
88
88
  hash2 = Open.open(integer_array_file) do |f|
89
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
89
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
90
90
  end
91
91
  assert_equal hash1, hash2
92
92
  end
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
98
98
  task = self.example_task
99
99
 
100
100
  TmpFile.with_file("2\n3") do |integer_array_file|
101
- assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
102
- task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
101
+ assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
102
+ task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
103
103
  end
104
104
  end
105
105
 
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
108
108
 
109
109
  TmpFile.with_file("2\n3") do |integer_array_file|
110
110
  inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
111
- original_digest = task.digest_inputs(inputs)
111
+ original_digest = task.process_inputs(inputs).last
112
112
 
113
113
  TmpFile.with_file do |save_directory|
114
114
  task.save_inputs(save_directory, inputs)
115
115
  new_inputs = task.load_inputs(save_directory)
116
- new_digest = task.digest_inputs(new_inputs)
116
+ new_digest = task.process_inputs(new_inputs).last
117
117
  assert_equal original_digest, new_digest
118
118
  end
119
119
  end
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
124
124
 
125
125
  TmpFile.with_file("TEST") do |somefile|
126
126
  inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
127
- original_digest = task.digest_inputs(inputs)
127
+ original_digest = task.process_inputs(inputs).last
128
128
 
129
129
  TmpFile.with_file do |save_directory|
130
130
  task.save_inputs(save_directory, inputs)
131
131
  Open.rm somefile
132
132
  new_inputs = task.load_inputs(save_directory)
133
- new_digest = task.digest_inputs(new_inputs)
133
+ new_digest = task.process_inputs(new_inputs).last
134
134
  assert_equal original_digest, new_digest
135
135
  end
136
136
  end
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
141
141
 
142
142
  TmpFile.with_file("TEST") do |somefile|
143
143
  inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
144
- original_digest = task.digest_inputs(inputs)
144
+ original_digest = task.process_inputs(inputs).last
145
145
 
146
146
  TmpFile.with_file do |save_directory|
147
147
  task.save_inputs(save_directory, inputs)
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
149
149
  Open.cp save_directory, copy_directory
150
150
  Open.rm_rf save_directory
151
151
  new_inputs = task.load_inputs(copy_directory)
152
- new_digest = task.digest_inputs(new_inputs)
152
+ new_digest = task.process_inputs(new_inputs).last
153
153
  assert_equal original_digest, new_digest
154
154
  end
155
155
  end
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
167
167
  Open.write(file1, "TEST1")
168
168
  Open.write(file2, "TEST2")
169
169
  inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
170
- original_digest = task.digest_inputs(inputs)
170
+ original_digest = task.process_inputs(inputs).last
171
171
 
172
172
  TmpFile.with_file do |save_directory|
173
173
  task.save_inputs(save_directory, inputs)
174
174
  Open.rm(file1)
175
175
  Open.rm(file2)
176
176
  new_inputs = task.load_inputs(save_directory)
177
- new_digest = task.digest_inputs(new_inputs)
177
+ new_digest = task.process_inputs(new_inputs).last
178
178
  assert_equal original_digest, new_digest
179
179
  end
180
180
  end
@@ -27,7 +27,6 @@ class TestWorkflowStep < Test::Unit::TestCase
27
27
 
28
28
  step2.dependencies = [step1]
29
29
 
30
-
31
30
  assert_equal "12 has 2 characters", step2.run
32
31
  assert_equal "12 has 2 characters", step2.run
33
32
  end
@@ -50,7 +49,7 @@ class TestWorkflowStep < Test::Unit::TestCase
50
49
 
51
50
  step2 = Step.new tmpfile.step2 do
52
51
  step1 = dependencies.first
53
- stream = step1.get_stream
52
+ stream = step1.stream
54
53
 
55
54
  Open.open_pipe do |sin|
56
55
  while line = stream.gets
@@ -76,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
76
75
  end
77
76
  assert_equal times/2, lines.length
78
77
 
79
- stream = step2.run
78
+ stream = step2.run(true)
80
79
  assert step1.streaming?
81
80
  assert step2.streaming?
82
81
 
@@ -87,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
87
86
 
88
87
  assert step1.path.read.end_with? "line-#{times-1}\n"
89
88
  assert_equal times/2, lines.length
90
- assert_equal times/2, step2.path.read.split("\n").length
89
+ assert_equal times/2, step2.join.path.read.split("\n").length
91
90
  end
92
91
 
93
92
  def test_streaming_duplicate
@@ -108,7 +107,7 @@ class TestWorkflowStep < Test::Unit::TestCase
108
107
 
109
108
  step2 = Step.new tmpfile.step2 do
110
109
  step1 = dependencies.first
111
- stream = step1.get_stream
110
+ stream = step1.stream
112
111
 
113
112
  Open.open_pipe do |sin|
114
113
  while line = stream.gets
@@ -123,7 +122,7 @@ class TestWorkflowStep < Test::Unit::TestCase
123
122
 
124
123
  step3 = Step.new tmpfile.step3 do
125
124
  step1, step2 = dependencies
126
- stream = step2.get_stream
125
+ stream = step2.stream
127
126
 
128
127
  Open.open_pipe do |sin|
129
128
  while line = stream.gets
@@ -138,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
138
137
 
139
138
  step3.recursive_clean
140
139
 
141
- stream = step3.run
140
+ stream = step3.run(true)
142
141
  out = []
143
142
  while l = stream.gets
144
143
  out << l
@@ -164,7 +163,7 @@ class TestWorkflowStep < Test::Unit::TestCase
164
163
 
165
164
  step2 = Step.new tmpfile.step2 do
166
165
  step1 = dependencies.first
167
- stream = step1.get_stream
166
+ stream = step1.stream
168
167
 
169
168
  Open.open_pipe do |sin|
170
169
  while line = stream.gets
@@ -179,7 +178,7 @@ class TestWorkflowStep < Test::Unit::TestCase
179
178
 
180
179
  step3 = Step.new tmpfile.step3 do
181
180
  step1 = dependencies.first
182
- stream = step1.get_stream
181
+ stream = step1.stream
183
182
 
184
183
  Open.open_pipe do |sin|
185
184
  while line = stream.gets
@@ -199,14 +198,14 @@ class TestWorkflowStep < Test::Unit::TestCase
199
198
  mutex = Mutex.new
200
199
  Open.open_pipe do |sin|
201
200
  t2 = Thread.new do
202
- stream2 = step2.get_stream
201
+ stream2 = step2.stream
203
202
  while line = stream2.gets
204
203
  sin.puts line
205
204
  end
206
205
  end
207
206
 
208
207
  t3 = Thread.new do
209
- stream3 = step3.get_stream
208
+ stream3 = step3.stream
210
209
  while line = stream3.gets
211
210
  sin.puts line
212
211
  end
@@ -219,13 +218,14 @@ class TestWorkflowStep < Test::Unit::TestCase
219
218
  step4.dependencies = [step2, step3]
220
219
 
221
220
  lines = []
222
- io = step4.run
223
- Log::ProgressBar.with_bar do |b|
221
+ io = step4.run(true)
222
+ Log::ProgressBar.with_bar severity: 0 do |b|
224
223
  while line = io.gets
225
224
  b.tick
226
225
  lines << line.strip
227
226
  end
228
227
  end
228
+ io.close
229
229
 
230
230
  assert_equal times, lines.length
231
231
  end