scout-gear 7.2.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +51 -6
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +33 -29
- data/lib/scout/config.rb +1 -1
- data/lib/scout/exceptions.rb +1 -0
- data/lib/scout/log/color.rb +4 -2
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +71 -2
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/helper.rb +31 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +12 -1
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +111 -42
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +60 -30
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/semaphore.rb +8 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +85 -54
- data/lib/scout/tsv/index.rb +188 -20
- data/lib/scout/tsv/open.rb +182 -0
- data/lib/scout/tsv/parser.rb +200 -118
- data/lib/scout/tsv/path.rb +5 -6
- data/lib/scout/tsv/persist/adapter.rb +26 -37
- data/lib/scout/tsv/persist/fix_width_table.rb +327 -0
- data/lib/scout/tsv/persist/serialize.rb +117 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +6 -3
- data/lib/scout/tsv/persist.rb +4 -2
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +136 -37
- data/lib/scout/tsv/util/filter.rb +312 -0
- data/lib/scout/tsv/util/process.rb +73 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +265 -0
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +126 -19
- data/lib/scout/tsv.rb +28 -5
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +29 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +40 -5
- data/lib/scout/workflow/step/progress.rb +14 -0
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +104 -33
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +86 -47
- data/lib/scout/workflow/usage.rb +10 -6
- data/scout-gear.gemspec +30 -3
- data/scout_commands/workflow/task +37 -9
- data/scout_commands/workflow/task_old +2 -2
- data/test/scout/open/test_stream.rb +61 -59
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +67 -0
- data/test/scout/test_tmpfile.rb +1 -1
- data/test/scout/test_tsv.rb +222 -3
- data/test/scout/test_work_queue.rb +21 -18
- data/test/scout/tsv/persist/test_adapter.rb +11 -1
- data/test/scout/tsv/persist/test_fix_width_table.rb +134 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +127 -3
- data/test/scout/tsv/test_open.rb +167 -0
- data/test/scout/tsv/test_parser.rb +45 -3
- data/test/scout/tsv/test_persist.rb +9 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +195 -3
- data/test/scout/tsv/test_util.rb +24 -0
- data/test/scout/tsv/util/test_filter.rb +188 -0
- data/test/scout/tsv/util/test_process.rb +47 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +58 -0
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/work_queue/test_worker.rb +63 -6
- data/test/scout/workflow/step/test_load.rb +3 -3
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- data/test/test_helper.rb +3 -1
- metadata +29 -2
@@ -0,0 +1,94 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
|
6
|
+
class TestReorder < Test::Unit::TestCase
|
7
|
+
def test_reorder
|
8
|
+
content =<<-'EOF'
|
9
|
+
#: :sep=/\s+/#:type=:double
|
10
|
+
#Id ValueA ValueB OtherID
|
11
|
+
row1 a1|a2 b1|b2 Id1|Id2
|
12
|
+
row2 A1|A3 B1|B3 Id1|Id3
|
13
|
+
EOF
|
14
|
+
|
15
|
+
tsv = TmpFile.with_file(content) do |filename|
|
16
|
+
TSV.open(filename)
|
17
|
+
end
|
18
|
+
|
19
|
+
r = tsv.reorder "OtherID", %w(ValueB Id)
|
20
|
+
|
21
|
+
assert_equal %w(row1 row2), r["Id1"]["Id"]
|
22
|
+
assert_equal %w(row2), r["Id3"]["Id"]
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_reorder_list
|
26
|
+
content =<<-'EOF'
|
27
|
+
#: :sep=/\s+/#:type=:list
|
28
|
+
#Id ValueA ValueB OtherID
|
29
|
+
row1 a1 b1 Id1
|
30
|
+
row2 A1 B1 Id1
|
31
|
+
EOF
|
32
|
+
|
33
|
+
tsv = TmpFile.with_file(content) do |filename|
|
34
|
+
TSV.open(filename)
|
35
|
+
end
|
36
|
+
|
37
|
+
r = tsv.reorder "ValueB"
|
38
|
+
|
39
|
+
assert_equal "row1", r["b1"]["Id"]
|
40
|
+
assert_equal "row2", r["B1"]["Id"]
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_reorder_single
|
44
|
+
content =<<-'EOF'
|
45
|
+
#: :sep=/\s+/#:type=:single
|
46
|
+
#Id ValueA
|
47
|
+
row1 a1
|
48
|
+
row2 A1
|
49
|
+
EOF
|
50
|
+
|
51
|
+
tsv = TmpFile.with_file(content) do |filename|
|
52
|
+
TSV.open(filename)
|
53
|
+
end
|
54
|
+
|
55
|
+
r = tsv.reorder "ValueA"
|
56
|
+
|
57
|
+
assert_equal "row1", r["a1"]
|
58
|
+
assert_equal "row2", r["A1"]
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_transpose
|
62
|
+
content =<<-EOF
|
63
|
+
#: :type=:list
|
64
|
+
#Row vA vB vID
|
65
|
+
row1 a b Id1
|
66
|
+
row2 A B Id3
|
67
|
+
row3 a C Id4
|
68
|
+
EOF
|
69
|
+
|
70
|
+
TmpFile.with_file(content) do |filename|
|
71
|
+
tsv = TSV.open(filename, :sep => /\s+/)
|
72
|
+
|
73
|
+
assert_equal %w(vA vB vID), tsv.transpose("Values").keys
|
74
|
+
assert_equal %w(Id1 Id3 Id4), tsv.transpose("Values")["vID"]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_column
|
79
|
+
content =<<-EOF
|
80
|
+
#Id ValueA ValueB ValueC
|
81
|
+
rowA A|AA B|BB C|CC
|
82
|
+
rowa a|aa b|BB C|CC
|
83
|
+
EOF
|
84
|
+
|
85
|
+
TmpFile.with_file(content) do |filename|
|
86
|
+
tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
|
87
|
+
tsv = tsv.column("ValueA", cast: :downcase)
|
88
|
+
assert_equal %w(a aa), tsv["rowA"]
|
89
|
+
assert_equal %w(a aa), tsv["rowa"]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestTSVSelect < Test::Unit::TestCase
|
6
|
+
def test_select
|
7
|
+
content =<<-'EOF'
|
8
|
+
#: :sep=/\s+/#:type=:double
|
9
|
+
#Id ValueA ValueB OtherID
|
10
|
+
row1 a|aa|aaa b Id1|Id2
|
11
|
+
row2 A B Id3
|
12
|
+
row2 AA BB Id33
|
13
|
+
EOF
|
14
|
+
|
15
|
+
tsv = TmpFile.with_file(content) do |filename|
|
16
|
+
TSV.open(filename, :persist => true)
|
17
|
+
end
|
18
|
+
|
19
|
+
s = tsv.select do |k,v|
|
20
|
+
k.include? "2"
|
21
|
+
end
|
22
|
+
|
23
|
+
assert_equal ['row2'], s.keys
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_select_values
|
27
|
+
content =<<-'EOF'
|
28
|
+
#: :sep=" "
|
29
|
+
#ID ValueA ValueB
|
30
|
+
row1 A1|A11 B1|B11
|
31
|
+
row2 A2|A22 B2|B22
|
32
|
+
EOF
|
33
|
+
|
34
|
+
tsv = TSV.open(content)
|
35
|
+
|
36
|
+
assert TSV.select("row1", tsv["row1"], "A1")
|
37
|
+
refute TSV.select("row2", tsv["row2"], "A1")
|
38
|
+
assert TSV.select("row2", tsv["row2"], "A2")
|
39
|
+
|
40
|
+
assert TSV.select("row1", tsv["row1"], "B1")
|
41
|
+
refute TSV.select("row2", tsv["row2"], "B1")
|
42
|
+
|
43
|
+
refute TSV.select("row1", tsv["row1"], "B1", invert: true)
|
44
|
+
assert TSV.select("row2", tsv["row2"], "B1", invert: true)
|
45
|
+
|
46
|
+
assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"})
|
47
|
+
refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"})
|
48
|
+
|
49
|
+
tsv.with_unnamed do
|
50
|
+
assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"}, fields: tsv.fields)
|
51
|
+
refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"}, fields: tsv.fields)
|
52
|
+
end
|
53
|
+
|
54
|
+
assert TSV.select("row1", tsv["row1"], "B1", field: "ValueB")
|
55
|
+
refute TSV.select("row2", tsv["row2"], "B1", field: "ValueB")
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestTSVUnzip < Test::Unit::TestCase
|
6
|
+
def test_unzip
|
7
|
+
content1 =<<-EOF
|
8
|
+
#: :sep=" "
|
9
|
+
#ID ValueA ValueB
|
10
|
+
row1 A1|A11 B1|B11
|
11
|
+
row2 A2|A22 B2|B22
|
12
|
+
EOF
|
13
|
+
|
14
|
+
tsv = TSV.open(content1)
|
15
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
16
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
17
|
+
assert_equal "B22", unzip["row2:A22"]["ValueB"]
|
18
|
+
end
|
19
|
+
|
20
|
+
def test_unzip_list
|
21
|
+
content1 =<<-EOF
|
22
|
+
#: :sep=" "#:type=:list
|
23
|
+
#ID ValueA ValueB
|
24
|
+
row1 A1 B1
|
25
|
+
row2 A2 B2
|
26
|
+
EOF
|
27
|
+
|
28
|
+
tsv = TSV.open(content1)
|
29
|
+
unzip = tsv.unzip("ValueA", delete: true)
|
30
|
+
assert_equal "B1", unzip["row1:A1"]["ValueB"]
|
31
|
+
assert_equal "B2", unzip["row2:A2"]["ValueB"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_unzip_merge
|
35
|
+
content1 =<<-EOF
|
36
|
+
#: :sep=" "
|
37
|
+
#ID ValueA ValueB
|
38
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
39
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
40
|
+
EOF
|
41
|
+
|
42
|
+
tsv = TSV.open(content1)
|
43
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true)
|
44
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
45
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
46
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_unzip_merge_stream
|
50
|
+
content1 =<<-EOF
|
51
|
+
#: :sep=" "
|
52
|
+
#ID ValueA ValueB
|
53
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
54
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
55
|
+
EOF
|
56
|
+
|
57
|
+
tsv = TSV.open(content1)
|
58
|
+
unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
|
59
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
60
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
61
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_unzip_merge_stream_parser
|
65
|
+
content1 =<<-EOF
|
66
|
+
#: :sep=" "
|
67
|
+
#ID ValueA ValueB
|
68
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
69
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
70
|
+
EOF
|
71
|
+
|
72
|
+
TmpFile.with_file(content1) do |filename|
|
73
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
|
74
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
75
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
76
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_unzip_merge_stream_parser_one2many
|
81
|
+
content1 =<<-EOF
|
82
|
+
#: :sep=" "
|
83
|
+
#ID ValueA ValueB
|
84
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
85
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
86
|
+
EOF
|
87
|
+
|
88
|
+
TmpFile.with_file(content1) do |filename|
|
89
|
+
unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
|
90
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
|
91
|
+
assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def test_unzip_target
|
97
|
+
content1 =<<-EOF
|
98
|
+
#: :sep=" "
|
99
|
+
#ID ValueA ValueB
|
100
|
+
row1 A1|A11|A11 B1|B11|B11.1
|
101
|
+
row2 A2|A22|A22 B2|B22|B22.2
|
102
|
+
EOF
|
103
|
+
|
104
|
+
tsv = TSV.open(content1)
|
105
|
+
unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
|
106
|
+
tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
|
107
|
+
assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
|
108
|
+
assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
|
109
|
+
assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
@@ -16,27 +16,84 @@ class TestQueueWorker < Test::Unit::TestCase
|
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
19
|
-
def
|
20
|
-
|
19
|
+
def test_semaphore_pipe
|
20
|
+
|
21
|
+
2.times do
|
22
|
+
num_lines = 10
|
23
|
+
num_workers = 100
|
21
24
|
|
22
25
|
TmpFile.with_file do |outfile|
|
23
|
-
|
26
|
+
Open.rm(outfile)
|
27
|
+
ScoutSemaphore.with_semaphore 1 do |sem|
|
24
28
|
sout = Open.open_pipe do |sin|
|
25
|
-
workers =
|
29
|
+
workers = num_workers.times.collect{ WorkQueue::Worker.new }
|
26
30
|
workers.each do |w|
|
27
31
|
w.run do
|
28
32
|
ScoutSemaphore.synchronize(sem) do
|
29
|
-
|
30
|
-
|
33
|
+
sin.puts "Start - #{Process.pid}"
|
34
|
+
num_lines.times do |i|
|
35
|
+
sin.puts "line-#{i}-#{Process.pid}"
|
31
36
|
end
|
37
|
+
sin.puts "End - #{Process.pid}"
|
32
38
|
end
|
33
39
|
end
|
34
40
|
end
|
41
|
+
sin.close
|
35
42
|
|
36
43
|
WorkQueue::Worker.join(workers)
|
37
44
|
end
|
45
|
+
|
38
46
|
Open.consume_stream(sout, false, outfile)
|
47
|
+
txt = Open.read(outfile)
|
48
|
+
pid_list = txt.split("\n")
|
49
|
+
|
50
|
+
assert_equal (num_lines + 2) * num_workers, pid_list.length
|
51
|
+
|
52
|
+
assert_nothing_raised do
|
53
|
+
seen = []
|
54
|
+
current = nil
|
55
|
+
pid_list.each do |pid|
|
56
|
+
if pid != current
|
57
|
+
raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
|
58
|
+
end
|
59
|
+
current = pid
|
60
|
+
seen << pid
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
def test_semaphore
|
68
|
+
|
69
|
+
2.times do
|
70
|
+
num_lines = 10
|
71
|
+
num_workers = 500
|
72
|
+
|
73
|
+
TmpFile.with_file do |outfile|
|
74
|
+
Open.rm(outfile)
|
75
|
+
ScoutSemaphore.with_semaphore 1 do |sem|
|
76
|
+
workers = num_workers.times.collect{ WorkQueue::Worker.new }
|
77
|
+
Open.touch(outfile)
|
78
|
+
workers.each do |w|
|
79
|
+
w.run do
|
80
|
+
ScoutSemaphore.synchronize(sem) do
|
81
|
+
sin = Open.open(outfile, :mode => 'a')
|
82
|
+
sin.puts "Start - #{Process.pid}"
|
83
|
+
num_lines.times do |i|
|
84
|
+
sin.puts "line-#{i}-#{Process.pid}"
|
85
|
+
end
|
86
|
+
sin.puts "End - #{Process.pid}"
|
87
|
+
sin.close
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
WorkQueue::Worker.join(workers)
|
93
|
+
|
94
|
+
|
39
95
|
pid_list = Open.read(outfile).split("\n")
|
96
|
+
assert_equal (num_lines + 2) * num_workers, pid_list.length
|
40
97
|
|
41
98
|
assert_nothing_raised do
|
42
99
|
seen = []
|
@@ -41,7 +41,7 @@ class TestStepLoad < Test::Unit::TestCase
|
|
41
41
|
end
|
42
42
|
end
|
43
43
|
|
44
|
-
step2 = wf.job(:step2, :input1 => "
|
44
|
+
step2 = wf.job(:step2, nil, :input1 => "TEST_STRING")
|
45
45
|
step1 = step2.step(:step1)
|
46
46
|
|
47
47
|
step2.run
|
@@ -53,8 +53,8 @@ class TestStepLoad < Test::Unit::TestCase
|
|
53
53
|
Open.mv step1.info_file, dir.var.jobs.RelocateWorkflow.step1[File.basename(step1.info_file)]
|
54
54
|
|
55
55
|
new_step2 = Step.load(step2.path)
|
56
|
-
assert_equal "
|
57
|
-
assert_equal "
|
56
|
+
assert_equal "TEST_STRING".reverse, new_step2.load
|
57
|
+
assert_equal "TEST_STRING", new_step2.dependencies.first.load
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestStepStatus < Test::Unit::TestCase
|
5
|
+
def test_dependency
|
6
|
+
tmpfile = tmpdir.test_step
|
7
|
+
step1 = Step.new tmpfile.step1, ["12"] do |s|
|
8
|
+
s.length
|
9
|
+
end
|
10
|
+
|
11
|
+
step2 = Step.new tmpfile.step2 do
|
12
|
+
step1 = dependencies.first
|
13
|
+
step1.inputs.first + " has " + step1.load.to_s + " characters"
|
14
|
+
end
|
15
|
+
|
16
|
+
step2.dependencies = [step1]
|
17
|
+
|
18
|
+
sss 0
|
19
|
+
Misc.with_env "SCOUT_UPDATE", "true" do
|
20
|
+
step2.run
|
21
|
+
assert step2.updated?
|
22
|
+
|
23
|
+
sleep 0.1
|
24
|
+
step1.clean
|
25
|
+
step1.run
|
26
|
+
refute step2.updated?
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
71
71
|
task = self.example_task
|
72
72
|
|
73
73
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
74
|
-
assert_equal task.
|
75
|
-
task.
|
74
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
75
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
76
76
|
end
|
77
77
|
end
|
78
78
|
|
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
|
|
83
83
|
|
84
84
|
TmpFile.with_file("1\n2") do |integer_array_file|
|
85
85
|
hash1 = Open.open(integer_array_file) do |f|
|
86
|
-
task.
|
86
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
87
87
|
end
|
88
88
|
hash2 = Open.open(integer_array_file) do |f|
|
89
|
-
task.
|
89
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
|
90
90
|
end
|
91
91
|
assert_equal hash1, hash2
|
92
92
|
end
|
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
|
|
98
98
|
task = self.example_task
|
99
99
|
|
100
100
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
101
|
-
assert_equal task.
|
102
|
-
task.
|
101
|
+
assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
|
102
|
+
task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
|
103
103
|
end
|
104
104
|
end
|
105
105
|
|
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
|
|
108
108
|
|
109
109
|
TmpFile.with_file("2\n3") do |integer_array_file|
|
110
110
|
inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
|
111
|
-
original_digest = task.
|
111
|
+
original_digest = task.process_inputs(inputs).last
|
112
112
|
|
113
113
|
TmpFile.with_file do |save_directory|
|
114
114
|
task.save_inputs(save_directory, inputs)
|
115
115
|
new_inputs = task.load_inputs(save_directory)
|
116
|
-
new_digest = task.
|
116
|
+
new_digest = task.process_inputs(new_inputs).last
|
117
117
|
assert_equal original_digest, new_digest
|
118
118
|
end
|
119
119
|
end
|
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
|
|
124
124
|
|
125
125
|
TmpFile.with_file("TEST") do |somefile|
|
126
126
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
127
|
-
original_digest = task.
|
127
|
+
original_digest = task.process_inputs(inputs).last
|
128
128
|
|
129
129
|
TmpFile.with_file do |save_directory|
|
130
130
|
task.save_inputs(save_directory, inputs)
|
131
131
|
Open.rm somefile
|
132
132
|
new_inputs = task.load_inputs(save_directory)
|
133
|
-
new_digest = task.
|
133
|
+
new_digest = task.process_inputs(new_inputs).last
|
134
134
|
assert_equal original_digest, new_digest
|
135
135
|
end
|
136
136
|
end
|
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
141
141
|
|
142
142
|
TmpFile.with_file("TEST") do |somefile|
|
143
143
|
inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
|
144
|
-
original_digest = task.
|
144
|
+
original_digest = task.process_inputs(inputs).last
|
145
145
|
|
146
146
|
TmpFile.with_file do |save_directory|
|
147
147
|
task.save_inputs(save_directory, inputs)
|
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
|
|
149
149
|
Open.cp save_directory, copy_directory
|
150
150
|
Open.rm_rf save_directory
|
151
151
|
new_inputs = task.load_inputs(copy_directory)
|
152
|
-
new_digest = task.
|
152
|
+
new_digest = task.process_inputs(new_inputs).last
|
153
153
|
assert_equal original_digest, new_digest
|
154
154
|
end
|
155
155
|
end
|
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
|
|
167
167
|
Open.write(file1, "TEST1")
|
168
168
|
Open.write(file2, "TEST2")
|
169
169
|
inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
|
170
|
-
original_digest = task.
|
170
|
+
original_digest = task.process_inputs(inputs).last
|
171
171
|
|
172
172
|
TmpFile.with_file do |save_directory|
|
173
173
|
task.save_inputs(save_directory, inputs)
|
174
174
|
Open.rm(file1)
|
175
175
|
Open.rm(file2)
|
176
176
|
new_inputs = task.load_inputs(save_directory)
|
177
|
-
new_digest = task.
|
177
|
+
new_digest = task.process_inputs(new_inputs).last
|
178
178
|
assert_equal original_digest, new_digest
|
179
179
|
end
|
180
180
|
end
|
@@ -27,7 +27,6 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
27
27
|
|
28
28
|
step2.dependencies = [step1]
|
29
29
|
|
30
|
-
|
31
30
|
assert_equal "12 has 2 characters", step2.run
|
32
31
|
assert_equal "12 has 2 characters", step2.run
|
33
32
|
end
|
@@ -50,7 +49,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
50
49
|
|
51
50
|
step2 = Step.new tmpfile.step2 do
|
52
51
|
step1 = dependencies.first
|
53
|
-
stream = step1.
|
52
|
+
stream = step1.stream
|
54
53
|
|
55
54
|
Open.open_pipe do |sin|
|
56
55
|
while line = stream.gets
|
@@ -76,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
76
75
|
end
|
77
76
|
assert_equal times/2, lines.length
|
78
77
|
|
79
|
-
stream = step2.run
|
78
|
+
stream = step2.run(true)
|
80
79
|
assert step1.streaming?
|
81
80
|
assert step2.streaming?
|
82
81
|
|
@@ -87,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
87
86
|
|
88
87
|
assert step1.path.read.end_with? "line-#{times-1}\n"
|
89
88
|
assert_equal times/2, lines.length
|
90
|
-
assert_equal times/2, step2.path.read.split("\n").length
|
89
|
+
assert_equal times/2, step2.join.path.read.split("\n").length
|
91
90
|
end
|
92
91
|
|
93
92
|
def test_streaming_duplicate
|
@@ -108,7 +107,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
108
107
|
|
109
108
|
step2 = Step.new tmpfile.step2 do
|
110
109
|
step1 = dependencies.first
|
111
|
-
stream = step1.
|
110
|
+
stream = step1.stream
|
112
111
|
|
113
112
|
Open.open_pipe do |sin|
|
114
113
|
while line = stream.gets
|
@@ -123,7 +122,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
123
122
|
|
124
123
|
step3 = Step.new tmpfile.step3 do
|
125
124
|
step1, step2 = dependencies
|
126
|
-
stream = step2.
|
125
|
+
stream = step2.stream
|
127
126
|
|
128
127
|
Open.open_pipe do |sin|
|
129
128
|
while line = stream.gets
|
@@ -138,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
138
137
|
|
139
138
|
step3.recursive_clean
|
140
139
|
|
141
|
-
stream = step3.run
|
140
|
+
stream = step3.run(true)
|
142
141
|
out = []
|
143
142
|
while l = stream.gets
|
144
143
|
out << l
|
@@ -164,7 +163,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
164
163
|
|
165
164
|
step2 = Step.new tmpfile.step2 do
|
166
165
|
step1 = dependencies.first
|
167
|
-
stream = step1.
|
166
|
+
stream = step1.stream
|
168
167
|
|
169
168
|
Open.open_pipe do |sin|
|
170
169
|
while line = stream.gets
|
@@ -179,7 +178,7 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
179
178
|
|
180
179
|
step3 = Step.new tmpfile.step3 do
|
181
180
|
step1 = dependencies.first
|
182
|
-
stream = step1.
|
181
|
+
stream = step1.stream
|
183
182
|
|
184
183
|
Open.open_pipe do |sin|
|
185
184
|
while line = stream.gets
|
@@ -199,14 +198,14 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
199
198
|
mutex = Mutex.new
|
200
199
|
Open.open_pipe do |sin|
|
201
200
|
t2 = Thread.new do
|
202
|
-
stream2 = step2.
|
201
|
+
stream2 = step2.stream
|
203
202
|
while line = stream2.gets
|
204
203
|
sin.puts line
|
205
204
|
end
|
206
205
|
end
|
207
206
|
|
208
207
|
t3 = Thread.new do
|
209
|
-
stream3 = step3.
|
208
|
+
stream3 = step3.stream
|
210
209
|
while line = stream3.gets
|
211
210
|
sin.puts line
|
212
211
|
end
|
@@ -219,13 +218,14 @@ class TestWorkflowStep < Test::Unit::TestCase
|
|
219
218
|
step4.dependencies = [step2, step3]
|
220
219
|
|
221
220
|
lines = []
|
222
|
-
io = step4.run
|
223
|
-
Log::ProgressBar.with_bar do |b|
|
221
|
+
io = step4.run(true)
|
222
|
+
Log::ProgressBar.with_bar severity: 0 do |b|
|
224
223
|
while line = io.gets
|
225
224
|
b.tick
|
226
225
|
lines << line.strip
|
227
226
|
end
|
228
227
|
end
|
228
|
+
io.close
|
229
229
|
|
230
230
|
assert_equal times, lines.length
|
231
231
|
end
|