scout-gear 2.0.0 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +65 -2
- data/Rakefile +2 -0
- data/VERSION +1 -1
- data/bin/scout +233 -24
- data/lib/scout/cmd.rb +344 -0
- data/lib/scout/concurrent_stream.rb +259 -0
- data/lib/scout/exceptions.rb +15 -8
- data/lib/scout/indiferent_hash/options.rb +8 -26
- data/lib/scout/log/color.rb +2 -2
- data/lib/scout/log/fingerprint.rb +11 -1
- data/lib/scout/log/progress/report.rb +0 -1
- data/lib/scout/log/progress/util.rb +1 -1
- data/lib/scout/log/progress.rb +4 -4
- data/lib/scout/log.rb +10 -2
- data/lib/scout/meta_extension.rb +19 -3
- data/lib/scout/misc/digest.rb +56 -0
- data/lib/scout/misc/filesystem.rb +26 -0
- data/lib/scout/misc/format.rb +17 -6
- data/lib/scout/misc/insist.rb +56 -0
- data/lib/scout/misc/monitor.rb +23 -0
- data/lib/scout/misc.rb +5 -11
- data/lib/scout/open/lock.rb +61 -0
- data/lib/scout/open/remote.rb +120 -0
- data/lib/scout/open/stream.rb +373 -0
- data/lib/scout/open/util.rb +225 -0
- data/lib/scout/open.rb +169 -0
- data/lib/scout/path/find.rb +68 -21
- data/lib/scout/path/tmpfile.rb +8 -0
- data/lib/scout/path/util.rb +14 -1
- data/lib/scout/path.rb +6 -30
- data/lib/scout/persist/open.rb +17 -0
- data/lib/scout/persist/path.rb +15 -0
- data/lib/scout/persist/serialize.rb +151 -0
- data/lib/scout/persist.rb +54 -0
- data/lib/scout/resource/path.rb +20 -0
- data/lib/scout/resource/produce/rake.rb +69 -0
- data/lib/scout/resource/produce.rb +246 -0
- data/lib/scout/resource/scout.rb +3 -0
- data/lib/scout/resource/util.rb +48 -0
- data/lib/scout/resource.rb +39 -0
- data/lib/scout/simple_opt/accessor.rb +1 -1
- data/lib/scout/simple_opt/doc.rb +29 -23
- data/lib/scout/simple_opt/parse.rb +4 -3
- data/lib/scout/tmpfile.rb +39 -1
- data/lib/scout/workflow/definition.rb +78 -0
- data/lib/scout/workflow/documentation.rb +83 -0
- data/lib/scout/workflow/step/info.rb +77 -0
- data/lib/scout/workflow/step/load.rb +18 -0
- data/lib/scout/workflow/step.rb +132 -0
- data/lib/scout/workflow/task/inputs.rb +114 -0
- data/lib/scout/workflow/task.rb +155 -0
- data/lib/scout/workflow/usage.rb +314 -0
- data/lib/scout/workflow/util.rb +11 -0
- data/lib/scout/workflow.rb +40 -0
- data/lib/scout-gear.rb +4 -0
- data/lib/scout.rb +1 -0
- data/lib/workflow-scout.rb +2 -0
- data/scout-gear.gemspec +77 -5
- data/scout_commands/alias +48 -0
- data/scout_commands/find +83 -0
- data/scout_commands/glob +0 -0
- data/scout_commands/rbbt +23 -0
- data/scout_commands/workflow/info +29 -0
- data/scout_commands/workflow/list +27 -0
- data/scout_commands/workflow/task +58 -0
- data/scout_commands/workflow/task_old +706 -0
- data/test/scout/indiferent_hash/test_options.rb +11 -1
- data/test/scout/misc/test_digest.rb +30 -0
- data/test/scout/misc/test_filesystem.rb +30 -0
- data/test/scout/misc/test_insist.rb +13 -0
- data/test/scout/open/test_lock.rb +52 -0
- data/test/scout/open/test_remote.rb +25 -0
- data/test/scout/open/test_stream.rb +515 -0
- data/test/scout/open/test_util.rb +73 -0
- data/test/scout/path/test_find.rb +28 -0
- data/test/scout/persist/test_open.rb +37 -0
- data/test/scout/persist/test_path.rb +37 -0
- data/test/scout/persist/test_serialize.rb +114 -0
- data/test/scout/resource/test_path.rb +40 -0
- data/test/scout/resource/test_produce.rb +62 -0
- data/test/scout/resource/test_util.rb +27 -0
- data/test/scout/simple_opt/test_doc.rb +16 -0
- data/test/scout/test_cmd.rb +85 -0
- data/test/scout/test_concurrent_stream.rb +29 -0
- data/test/scout/test_meta_extension.rb +9 -0
- data/test/scout/test_misc.rb +0 -7
- data/test/scout/test_open.rb +146 -0
- data/test/scout/test_path.rb +3 -1
- data/test/scout/test_persist.rb +83 -0
- data/test/scout/test_resource.rb +26 -0
- data/test/scout/test_workflow.rb +87 -0
- data/test/scout/workflow/step/test_info.rb +30 -0
- data/test/scout/workflow/step/test_load.rb +65 -0
- data/test/scout/workflow/task/test_inputs.rb +182 -0
- data/test/scout/workflow/test_definition.rb +0 -0
- data/test/scout/workflow/test_documentation.rb +30 -0
- data/test/scout/workflow/test_step.rb +36 -0
- data/test/scout/workflow/test_task.rb +179 -0
- data/test/scout/workflow/test_usage.rb +35 -0
- data/test/scout/workflow/test_util.rb +17 -0
- data/test/test_helper.rb +17 -0
- data/test/test_scout-gear.rb +0 -0
- metadata +75 -3
@@ -3,6 +3,16 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
|
|
3
3
|
|
4
4
|
class TestOptions < Test::Unit::TestCase
|
5
5
|
|
6
|
+
|
7
|
+
def test_add_defaults
|
8
|
+
options = {:a => 1, "b" => 2}
|
9
|
+
assert_equal 2, IndiferentHash.add_defaults(options, :b => 3)["b"]
|
10
|
+
assert_equal 2, IndiferentHash.add_defaults(options, "b" => 3)["b"]
|
11
|
+
assert_equal 3, IndiferentHash.add_defaults(options, :c => 3)["c"]
|
12
|
+
assert_equal 3, IndiferentHash.add_defaults(options, "c" => 4)[:c]
|
13
|
+
assert_equal 3, IndiferentHash.add_defaults(options, "c" => 4)["c"]
|
14
|
+
end
|
15
|
+
|
6
16
|
def test_positions2hash
|
7
17
|
inputs = IndiferentHash.positional2hash([:one, :two, :three], 1, :two => 2, :four => 4)
|
8
18
|
assert_equal 1, inputs[:one]
|
@@ -13,7 +23,7 @@ class TestOptions < Test::Unit::TestCase
|
|
13
23
|
|
14
24
|
def test_process_to_hash
|
15
25
|
list = [1,2,3,4]
|
16
|
-
assert_equal 4, IndiferentHash.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
|
26
|
+
assert_equal 4, IndiferentHash.process_to_hash(list){|l| l.collect{|e| e * 2 } }[2]
|
17
27
|
end
|
18
28
|
|
19
29
|
def test_hash2string
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestMiscDigest < Test::Unit::TestCase
|
5
|
+
def test_digest_str
|
6
|
+
o = {:a => [1,2,3], :b => [1.1, 0.00001, 'hola']}
|
7
|
+
assert Misc.digest_str(o).include? "hola"
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_digest_stream_located
|
11
|
+
TmpFile.with_file("TEST") do |filename|
|
12
|
+
Open.open(filename) do |f|
|
13
|
+
assert_equal 32, Misc.digest_str(f).length
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_digest_stream_unlocated
|
19
|
+
TmpFile.with_file do |directory|
|
20
|
+
Path.setup(directory)
|
21
|
+
Open.write(directory.share.file, "TEST")
|
22
|
+
Misc.in_dir directory do
|
23
|
+
Open.open(Path.setup('share/file')) do |f|
|
24
|
+
assert_equal '\'share/file\'', Misc.digest_str(Path.setup('share/file'))
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestFilesystem < Test::Unit::TestCase
|
5
|
+
def test_in_dir
|
6
|
+
TmpFile.with_file do |tmpdir|
|
7
|
+
Misc.in_dir tmpdir do
|
8
|
+
assert_equal tmpdir, FileUtils.pwd
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def test_relative
|
14
|
+
assert_equal 'bar', Misc.path_relative_to("/tmp/foo", "/tmp/foo/bar")
|
15
|
+
assert_equal 'bar/other', Misc.path_relative_to("/tmp/foo", "/tmp/foo/bar/other")
|
16
|
+
|
17
|
+
refute Misc.path_relative_to("/tmp/bar", "/tmp/foo/bar/other")
|
18
|
+
refute Misc.path_relative_to("/tmp/foo", "tmp/foo/bar/other")
|
19
|
+
|
20
|
+
TmpFile.with_file do |tmpdir|
|
21
|
+
Misc.in_dir tmpdir do
|
22
|
+
assert Misc.path_relative_to(tmpdir, "foo")
|
23
|
+
assert Misc.path_relative_to(tmpdir, File.join(tmpdir, "foo"))
|
24
|
+
end
|
25
|
+
assert Misc.path_relative_to(tmpdir, File.join(tmpdir, "foo"))
|
26
|
+
assert Misc.path_relative_to(File.dirname(tmpdir), File.join(tmpdir, "foo"))
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestMiscInsist < Test::Unit::TestCase
|
5
|
+
def test_insist
|
6
|
+
i = 0
|
7
|
+
Misc.insist do
|
8
|
+
i += 1
|
9
|
+
raise "Not yet" if i < 3
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tmpfile'
|
5
|
+
|
6
|
+
class TestLock < Test::Unit::TestCase
|
7
|
+
def test_locks
|
8
|
+
num = 10
|
9
|
+
TmpFile.with_file do |lockfile|
|
10
|
+
TmpFile.with_file do |output|
|
11
|
+
f = File.open(output, 'w')
|
12
|
+
thrs = num.times.collect do
|
13
|
+
Thread.new do
|
14
|
+
Open.lock lockfile, true, min_sleep: 0.01, max_sleep: 0.05, sleep_inc: 0.001 do
|
15
|
+
f.write "["
|
16
|
+
sleep 0.01
|
17
|
+
f.write "]"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
thrs.each{|t| t.join }
|
22
|
+
f.close
|
23
|
+
assert_equal "[]" * num, File.open(output).read
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_keep_locked
|
29
|
+
num = 10
|
30
|
+
TmpFile.with_file do |lockfile|
|
31
|
+
TmpFile.with_file do |output|
|
32
|
+
f = File.open(output, 'w')
|
33
|
+
thrs = num.times.collect do
|
34
|
+
Thread.new do
|
35
|
+
lock = Lockfile.new(lockfile, min_sleep: 0.01, max_sleep: 0.05, sleep_inc: 0.001)
|
36
|
+
res = Open.lock lock do
|
37
|
+
f.write "["
|
38
|
+
raise KeepLocked, "1"
|
39
|
+
end
|
40
|
+
f.write res
|
41
|
+
f.write "]"
|
42
|
+
lock.unlock
|
43
|
+
end
|
44
|
+
end
|
45
|
+
thrs.each{|t| t.join }
|
46
|
+
f.close
|
47
|
+
assert_equal "[1]" * num, File.open(output).read
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/open'
|
5
|
+
|
6
|
+
class TestOpenRemote < Test::Unit::TestCase
|
7
|
+
def test_wget
|
8
|
+
teardown
|
9
|
+
5.times do
|
10
|
+
assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true, :nocache => true).read) =~ /html/)
|
11
|
+
assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true).read) =~ /html/)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_ssh
|
16
|
+
TmpFile.with_file("TEST") do |f|
|
17
|
+
begin
|
18
|
+
assert_equal "TEST", Open.ssh("ssh://localhost:#{f}").read
|
19
|
+
rescue ConcurrentStreamProcessFailed
|
20
|
+
raise $! unless $!.message.include? "Connection refused"
|
21
|
+
end
|
22
|
+
end if false
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,515 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/path'
|
5
|
+
require 'scout/open'
|
6
|
+
|
7
|
+
class TestOpenStream < Test::Unit::TestCase
|
8
|
+
def test_consume_stream
|
9
|
+
content =<<-EOF
|
10
|
+
1
|
11
|
+
2
|
12
|
+
3
|
13
|
+
4
|
14
|
+
EOF
|
15
|
+
TmpFile.with_file(content) do |file|
|
16
|
+
TmpFile.with_file do |target|
|
17
|
+
f = File.open(file)
|
18
|
+
Open.consume_stream(f, false, target)
|
19
|
+
assert_equal content, File.open(target).read
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def test_sensible_write
|
25
|
+
content =<<-EOF
|
26
|
+
1
|
27
|
+
2
|
28
|
+
3
|
29
|
+
4
|
30
|
+
EOF
|
31
|
+
TmpFile.with_file(content) do |file|
|
32
|
+
TmpFile.with_file do |target|
|
33
|
+
f = File.open(file)
|
34
|
+
Open.sensible_write(target, f)
|
35
|
+
assert_equal content, File.open(target).read
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_open_pipe
|
41
|
+
sout = Open.open_pipe do |sin|
|
42
|
+
10.times do |i|
|
43
|
+
sin.puts "line #{i}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
TmpFile.with_file do |tmp|
|
48
|
+
Open.consume_stream(sout, false, tmp)
|
49
|
+
assert_equal 10, Open.read(tmp).split("\n").length
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_open_pipe_and_close
|
54
|
+
|
55
|
+
10000.times do
|
56
|
+
sout = Open.open_pipe do |sin|
|
57
|
+
sin.puts "hola"
|
58
|
+
end
|
59
|
+
sout.read
|
60
|
+
sout.close
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
def test_open_pipe_error
|
66
|
+
sout = Open.open_pipe do |sin|
|
67
|
+
10.times do |i|
|
68
|
+
sin.puts "line #{i}"
|
69
|
+
end
|
70
|
+
raise ScoutException
|
71
|
+
end
|
72
|
+
|
73
|
+
TmpFile.with_file do |tmp|
|
74
|
+
assert_raise ScoutException do
|
75
|
+
Open.consume_stream(sout, false, tmp)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_tee_stream
|
81
|
+
num = 2000
|
82
|
+
sout = Open.open_pipe do |sin|
|
83
|
+
num.times do |i|
|
84
|
+
sin.puts "line #{i} - #{rand(100000).to_s * 100}"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
TmpFile.with_file do |tmp|
|
89
|
+
Path.setup tmp
|
90
|
+
|
91
|
+
s1, s2 = Open.tee_stream_thread sout
|
92
|
+
|
93
|
+
t1 = Open.consume_stream(s1, true, tmp.file1)
|
94
|
+
t2 = Open.consume_stream(s2, true, tmp.file2)
|
95
|
+
t1.join
|
96
|
+
t2.join
|
97
|
+
assert_equal num, Open.read(tmp.file1).split("\n").length
|
98
|
+
assert_equal num, Open.read(tmp.file2).split("\n").length
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def test_tee_stream_source_error
|
103
|
+
5.times do |i|
|
104
|
+
num = 2000
|
105
|
+
sout = Open.open_pipe do |sin|
|
106
|
+
num.times do |i|
|
107
|
+
sin.puts "line #{i} - #{rand(100000).to_s * 100}"
|
108
|
+
end
|
109
|
+
raise ScoutException
|
110
|
+
end
|
111
|
+
|
112
|
+
TmpFile.with_file do |tmp|
|
113
|
+
Path.setup tmp
|
114
|
+
|
115
|
+
s1, s2 = Open.tee_stream_thread sout
|
116
|
+
|
117
|
+
t1 = Open.consume_stream(s1, true, tmp.file1)
|
118
|
+
t2 = Open.consume_stream(s2, true, tmp.file2)
|
119
|
+
assert_raise ScoutException do
|
120
|
+
begin
|
121
|
+
t1.join
|
122
|
+
t2.join
|
123
|
+
rescue
|
124
|
+
raise
|
125
|
+
ensure
|
126
|
+
[t1, t2].each do |t|
|
127
|
+
begin
|
128
|
+
t.join if t.alive?
|
129
|
+
rescue ScoutException
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
refute Open.exist?(tmp.file1)
|
135
|
+
refute Open.exist?(tmp.file2)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def test_tee_stream_save_error
|
141
|
+
Log.with_severity 6 do
|
142
|
+
5.times do |i|
|
143
|
+
TmpFile.with_file do |tmp|
|
144
|
+
Path.setup tmp
|
145
|
+
assert_raise ScoutException do
|
146
|
+
num = 2000
|
147
|
+
begin
|
148
|
+
sout = Open.open_pipe do |sin|
|
149
|
+
num.times do |i|
|
150
|
+
sin.puts "line #{i} - #{rand(100000).to_s * 100}"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
begin
|
155
|
+
s1, s2 = Open.tee_stream_thread sout
|
156
|
+
|
157
|
+
s2.abort ScoutException.new
|
158
|
+
t = Open.consume_stream(s1, true, tmp.file)
|
159
|
+
|
160
|
+
t.join
|
161
|
+
ensure
|
162
|
+
s1.close if s1.respond_to?(:close) && ! s1.closed?
|
163
|
+
s1.join if s1.respond_to?(:join) && ! s1.joined?
|
164
|
+
s2.close if s2.respond_to?(:close) && ! s2.closed?
|
165
|
+
s2.join if s2.respond_to?(:join) && ! s2.joined?
|
166
|
+
end
|
167
|
+
ensure
|
168
|
+
sout.close if sout.respond_to?(:close) && ! sout.closed?
|
169
|
+
sout.join if sout.respond_to?(:join) && ! sout.joined?
|
170
|
+
end
|
171
|
+
end
|
172
|
+
refute Open.exist?(tmp.file)
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
178
|
+
# def test_collapse_stream
|
179
|
+
# text=<<-EOF
|
180
|
+
#row1 A B C
|
181
|
+
#row1 a b c
|
182
|
+
#row2 AA BB CC
|
183
|
+
#row2 aa bb cc
|
184
|
+
# EOF
|
185
|
+
#
|
186
|
+
# s = StringIO.new text
|
187
|
+
# tsv = TSV.open Misc.collapse_stream(s,nil, " "), :sep => " "
|
188
|
+
# assert_equal ["A", "a"], tsv["row1"][0]
|
189
|
+
# assert_equal ["BB", "bb"], tsv["row2"][1]
|
190
|
+
# end
|
191
|
+
#
|
192
|
+
# def test_collapse_sum
|
193
|
+
# text=<<-EOF
|
194
|
+
#row1 12
|
195
|
+
#row1 4
|
196
|
+
#row2 10
|
197
|
+
#row2 6
|
198
|
+
# EOF
|
199
|
+
#
|
200
|
+
# s = StringIO.new text
|
201
|
+
# stream = Misc.collapse_stream(s,nil, " ") do |parts|
|
202
|
+
# next nil if parts.empty?
|
203
|
+
# parts.first.split("|").collect{|p| p.to_f}.inject(0){|acc,e| acc += e}.to_s
|
204
|
+
# end
|
205
|
+
# tsv = TSV.open stream, :sep => " "
|
206
|
+
# ppp tsv.to_s
|
207
|
+
# end
|
208
|
+
#
|
209
|
+
# def test_collapse_stream_gap
|
210
|
+
# text=<<-EOF
|
211
|
+
#row2 AA BB
|
212
|
+
#row2 aa bb cc
|
213
|
+
# EOF
|
214
|
+
#
|
215
|
+
# s = StringIO.new text
|
216
|
+
# assert Misc.collapse_stream(s, nil, " ").read =~ /\|cc$/
|
217
|
+
#
|
218
|
+
# text=<<-EOF
|
219
|
+
#row2 aa bb cc
|
220
|
+
#row2 AA BB
|
221
|
+
# EOF
|
222
|
+
#
|
223
|
+
# s = StringIO.new text
|
224
|
+
# assert Misc.collapse_stream(s, nil, " ").read =~ /cc\|$/
|
225
|
+
#
|
226
|
+
# text=<<-EOF
|
227
|
+
#row2 AA BB
|
228
|
+
#row2 aa bb cc
|
229
|
+
# EOF
|
230
|
+
#
|
231
|
+
# s = StringIO.new text
|
232
|
+
# assert Misc.collapse_stream(s, nil, " ").read =~ /\|cc$/
|
233
|
+
#
|
234
|
+
# end
|
235
|
+
#
|
236
|
+
#
|
237
|
+
# def test_paste_stream
|
238
|
+
# text1=<<-EOF
|
239
|
+
#row1 A B C
|
240
|
+
#row2 AA BB CC
|
241
|
+
#row3 AAA BBB CCC
|
242
|
+
# EOF
|
243
|
+
#
|
244
|
+
# text2=<<-EOF
|
245
|
+
#row1 a b
|
246
|
+
#row2 aa bb
|
247
|
+
# EOF
|
248
|
+
#
|
249
|
+
# text3=<<-EOF
|
250
|
+
#row1 c
|
251
|
+
#row2 cc
|
252
|
+
#row3 ccc
|
253
|
+
# EOF
|
254
|
+
#
|
255
|
+
# s1 = StringIO.new text1
|
256
|
+
# s2 = StringIO.new text2
|
257
|
+
# s3 = StringIO.new text3
|
258
|
+
# tsv = TSV.open Misc.paste_streams([s1,s2,s3], nil, " "), :sep => " ", :type => :list
|
259
|
+
# assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
|
260
|
+
# assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
|
261
|
+
# assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
|
262
|
+
# end
|
263
|
+
#
|
264
|
+
# def test_sort_stream
|
265
|
+
# text =<<-EOF
|
266
|
+
###
|
267
|
+
###
|
268
|
+
###
|
269
|
+
##Row LabelA LabelB LabelC
|
270
|
+
#row2 AA BB CC
|
271
|
+
#row3 AAA BBB CCC
|
272
|
+
#row1 A B C
|
273
|
+
# EOF
|
274
|
+
# s = StringIO.new text
|
275
|
+
# sorted = Misc.sort_stream(s)
|
276
|
+
# assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
|
277
|
+
# assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
278
|
+
# end
|
279
|
+
#
|
280
|
+
# def test_sort_long_stream
|
281
|
+
# text =<<-EOF
|
282
|
+
###
|
283
|
+
###
|
284
|
+
###
|
285
|
+
##Row LabelA LabelB LabelC
|
286
|
+
#row2 AA BB CC
|
287
|
+
#row3 AAA BBB CCC
|
288
|
+
#row1 A B C
|
289
|
+
# EOF
|
290
|
+
#
|
291
|
+
# s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
|
292
|
+
# sorted = Misc.sort_stream(s)
|
293
|
+
# assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
|
294
|
+
# assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
295
|
+
# end
|
296
|
+
#
|
297
|
+
# def test_sort_stream2
|
298
|
+
# text =<<-EOF
|
299
|
+
###
|
300
|
+
###
|
301
|
+
###
|
302
|
+
##Row LabelA LabelB LabelC
|
303
|
+
#row2 AA BB CC
|
304
|
+
#row3 AAA BBB CCC
|
305
|
+
#row1 A B C
|
306
|
+
# EOF
|
307
|
+
# s = StringIO.new text
|
308
|
+
# sorted = Misc.sort_stream(Misc.sort_stream(s))
|
309
|
+
# assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
|
310
|
+
# assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
311
|
+
# end
|
312
|
+
#
|
313
|
+
# def test_dup_stream
|
314
|
+
# text =<<-EOF
|
315
|
+
##: :sep=" "
|
316
|
+
##Row LabelA LabelB LabelC
|
317
|
+
#row2 AA BB CC
|
318
|
+
#row3 AAA BBB CCC
|
319
|
+
#row1 A B C
|
320
|
+
# EOF
|
321
|
+
#
|
322
|
+
# text = text * 10000
|
323
|
+
# TmpFile.with_file(text) do |tmp|
|
324
|
+
# io = Open.open(tmp)
|
325
|
+
# dup = Misc.dup_stream(io)
|
326
|
+
# new_text = StringIO.new ""
|
327
|
+
# Misc.consume_stream dup, true, new_text
|
328
|
+
# Misc.consume_stream io, false
|
329
|
+
# new_text.rewind
|
330
|
+
# assert_equal text, new_text.read
|
331
|
+
# end
|
332
|
+
#
|
333
|
+
#
|
334
|
+
# TmpFile.with_file(text) do |tmp|
|
335
|
+
# io = Open.open(tmp)
|
336
|
+
# dup = Misc.dup_stream(io)
|
337
|
+
# Misc.consume_stream dup, true
|
338
|
+
# assert_equal text, io.read
|
339
|
+
# end
|
340
|
+
# end
|
341
|
+
#
|
342
|
+
# def test_dup_stream_multiple
|
343
|
+
# text =<<-EOF
|
344
|
+
#row2 AA BB CC
|
345
|
+
#row3 AAA BBB CCC
|
346
|
+
#row1 A B C
|
347
|
+
# EOF
|
348
|
+
#
|
349
|
+
# text = text * 10000
|
350
|
+
# num = 5
|
351
|
+
# str_io = StringIO.new
|
352
|
+
# strs = []
|
353
|
+
# num.times{ strs << StringIO.new }
|
354
|
+
# TmpFile.with_file(text) do |tmp|
|
355
|
+
# io = Open.open(tmp)
|
356
|
+
# copies = Misc.dup_stream_multiple(io, num)
|
357
|
+
#
|
358
|
+
#
|
359
|
+
# copies.each_with_index do |cio,i|
|
360
|
+
# cio.add_callback do
|
361
|
+
# str = strs[i]
|
362
|
+
# str.rewind
|
363
|
+
# assert_equal text, str.read
|
364
|
+
# end
|
365
|
+
# Misc.consume_stream cio, true, strs[i], false
|
366
|
+
# end
|
367
|
+
#
|
368
|
+
#
|
369
|
+
# Misc.consume_stream io, false, str_io, false
|
370
|
+
# str_io.rewind
|
371
|
+
# assert_equal text, str_io.read
|
372
|
+
#
|
373
|
+
# end
|
374
|
+
# end
|
375
|
+
#
|
376
|
+
# def test_remove_lines
|
377
|
+
# text1 =<<-EOF
|
378
|
+
#line1
|
379
|
+
#line2
|
380
|
+
#line3
|
381
|
+
#line4
|
382
|
+
# EOF
|
383
|
+
# text2 =<<-EOF
|
384
|
+
#line3
|
385
|
+
#line1
|
386
|
+
# EOF
|
387
|
+
#
|
388
|
+
# TmpFile.with_file(text1) do |file1|
|
389
|
+
# TmpFile.with_file(text2) do |file2|
|
390
|
+
# assert ! Misc.remove_lines(file1, file2, true).read.split("\n").include?("line1")
|
391
|
+
# assert Misc.remove_lines(file1, file2, true).read.split("\n").include?("line2")
|
392
|
+
# end
|
393
|
+
# end
|
394
|
+
# end
|
395
|
+
#
|
396
|
+
#
|
397
|
+
#
|
398
|
+
# def test_select_lines
|
399
|
+
# text1 =<<-EOF
|
400
|
+
#line1
|
401
|
+
#line2
|
402
|
+
#line3
|
403
|
+
#line4
|
404
|
+
# EOF
|
405
|
+
# text2 =<<-EOF
|
406
|
+
#line3
|
407
|
+
#line1
|
408
|
+
#line5
|
409
|
+
# EOF
|
410
|
+
#
|
411
|
+
# TmpFile.with_file(text1) do |file1|
|
412
|
+
# TmpFile.with_file(text2) do |file2|
|
413
|
+
# assert Misc.select_lines(file1, file2, true).read.split("\n").include?("line1")
|
414
|
+
# assert ! Misc.select_lines(file1, file2, true).read.split("\n").include?("line2")
|
415
|
+
# end
|
416
|
+
# end
|
417
|
+
# end
|
418
|
+
#
|
419
|
+
# def test_consume_into_string_io
|
420
|
+
# text =<<-EOF
|
421
|
+
#line1
|
422
|
+
#line2
|
423
|
+
#line3
|
424
|
+
#line4
|
425
|
+
# EOF
|
426
|
+
#
|
427
|
+
# TmpFile.with_file(text) do |file|
|
428
|
+
# out = StringIO.new
|
429
|
+
# io = Open.open(file)
|
430
|
+
# Misc.consume_stream(io, false, out, false)
|
431
|
+
# out.rewind
|
432
|
+
# assert_equal text, out.read
|
433
|
+
# end
|
434
|
+
# end
|
435
|
+
#
|
436
|
+
# def test_monitor
|
437
|
+
# text =<<-EOF
|
438
|
+
#line1
|
439
|
+
#line2
|
440
|
+
#line3
|
441
|
+
#line4
|
442
|
+
# EOF
|
443
|
+
#
|
444
|
+
# TmpFile.with_file(text) do |file|
|
445
|
+
# io = Open.open(file)
|
446
|
+
# lines = Set.new
|
447
|
+
# io2 = Misc.line_monitor_stream io do |line|
|
448
|
+
# lines << line
|
449
|
+
# end
|
450
|
+
# Misc.consume_stream(io2, false)
|
451
|
+
# assert_equal text, lines.to_a * ""
|
452
|
+
# end
|
453
|
+
# end
|
454
|
+
#
|
455
|
+
# def test_sort
|
456
|
+
# assert_raise RbbtException do
|
457
|
+
# io = Misc.open_pipe do |sin|
|
458
|
+
# sin.puts "#START"
|
459
|
+
# 20.times do
|
460
|
+
# sin.puts rand(1000).to_s
|
461
|
+
# sleep 0.1
|
462
|
+
# end
|
463
|
+
# raise RbbtException
|
464
|
+
# end
|
465
|
+
#
|
466
|
+
# sio = Misc.sort_stream(io)
|
467
|
+
# begin
|
468
|
+
# Misc.consume_stream(sio, false, STDOUT)
|
469
|
+
# rescue
|
470
|
+
# raise $!
|
471
|
+
# end
|
472
|
+
# end
|
473
|
+
# end
|
474
|
+
#
|
475
|
+
# def test_gz_pipe
|
476
|
+
# text =<<-EOF
|
477
|
+
#line1
|
478
|
+
#line2
|
479
|
+
#line3
|
480
|
+
#line4
|
481
|
+
# EOF
|
482
|
+
#
|
483
|
+
# TmpFile.with_file nil, :extension => 'txt.gz' do |file|
|
484
|
+
# sout = Misc.open_gz_pipe do |sin|
|
485
|
+
# text.split("\n").each do |line|
|
486
|
+
# sin.puts line
|
487
|
+
# end
|
488
|
+
# end
|
489
|
+
#
|
490
|
+
# Open.mkdir File.basename(file)
|
491
|
+
# thr1 = Misc.consume_stream(sout, true, file)
|
492
|
+
# thr1.join
|
493
|
+
# assert Open.gzip?(file)
|
494
|
+
# assert_equal text, Open.read(file)
|
495
|
+
# end
|
496
|
+
# end
|
497
|
+
#
|
498
|
+
# def test_open_pipe_error
|
499
|
+
# sout = Misc.open_pipe do |sin|
|
500
|
+
# 10.times do |i|
|
501
|
+
# sin.puts "line #{i}"
|
502
|
+
# end
|
503
|
+
# raise
|
504
|
+
# end
|
505
|
+
#
|
506
|
+
# TmpFile.with_file do |tmp|
|
507
|
+
# #Misc.consume_stream(sout, false, tmp)
|
508
|
+
# assert_raise do
|
509
|
+
# Open.write(tmp, sout)
|
510
|
+
# end
|
511
|
+
# end
|
512
|
+
# end
|
513
|
+
#end
|
514
|
+
end
|
515
|
+
|