rbbt-util 5.10.1 → 5.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +39 -45
- data/lib/rbbt/resource/path.rb +6 -0
- data/lib/rbbt/tsv/dumper.rb +29 -6
- data/lib/rbbt/tsv/parallel/traverse.rb +151 -50
- data/lib/rbbt/tsv/parser.rb +7 -3
- data/lib/rbbt/tsv/util.rb +4 -1
- data/lib/rbbt/util/cmd.rb +1 -1
- data/lib/rbbt/util/concurrency/processes.rb +31 -24
- data/lib/rbbt/util/concurrency/processes/socket.rb +21 -13
- data/lib/rbbt/util/concurrency/processes/worker.rb +23 -13
- data/lib/rbbt/util/log.rb +18 -4
- data/lib/rbbt/util/misc.rb +198 -18
- data/lib/rbbt/workflow/definition.rb +2 -0
- data/lib/rbbt/workflow/step.rb +16 -8
- data/share/rbbt_commands/benchmark/throughput +41 -0
- data/share/rbbt_commands/system/report +12 -7
- data/share/rbbt_commands/workflow/task +5 -4
- data/test/rbbt/test_persist.rb +1 -2
- data/test/rbbt/tsv/parallel/test_traverse.rb +153 -13
- data/test/rbbt/util/concurrency/test_processes.rb +0 -1
- data/test/rbbt/util/test_misc.rb +148 -31
- metadata +3 -2
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -16,6 +16,7 @@ module TSV
|
|
16
16
|
|
17
17
|
# Get line
|
18
18
|
|
19
|
+
Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
|
19
20
|
line = stream.gets
|
20
21
|
raise "Empty content" if line.nil?
|
21
22
|
line = Misc.fixutf8 line
|
@@ -40,6 +41,7 @@ module TSV
|
|
40
41
|
@key_field = @fields.shift
|
41
42
|
@key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
|
42
43
|
|
44
|
+
Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
|
43
45
|
line = @header_hash != "" ? Misc.fixutf8(stream.gets) : nil
|
44
46
|
end
|
45
47
|
|
@@ -348,7 +350,8 @@ module TSV
|
|
348
350
|
|
349
351
|
options = header_options.merge options
|
350
352
|
|
351
|
-
@type
|
353
|
+
@type ||= Misc.process_options(options, :type) || :double
|
354
|
+
@type ||= :double
|
352
355
|
|
353
356
|
@filename = Misc.process_options(options, :filename)
|
354
357
|
@filename ||= stream.filename if stream.respond_to? :filename
|
@@ -497,7 +500,10 @@ module TSV
|
|
497
500
|
|
498
501
|
yield key, values
|
499
502
|
|
503
|
+
Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
|
504
|
+
|
500
505
|
line = stream.gets
|
506
|
+
|
501
507
|
line_num += 1
|
502
508
|
raise END_PARSING if head and line_num > head.to_i
|
503
509
|
rescue SKIP_LINE
|
@@ -514,8 +520,6 @@ module TSV
|
|
514
520
|
break
|
515
521
|
end
|
516
522
|
end
|
517
|
-
#ensure
|
518
|
-
# stream.close unless stream.closed?
|
519
523
|
end
|
520
524
|
|
521
525
|
self
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -98,7 +98,10 @@ module TSV
|
|
98
98
|
when Path
|
99
99
|
file.open(open_options)
|
100
100
|
when IO, StringIO
|
101
|
-
|
101
|
+
begin
|
102
|
+
file.rewind if file.respond_to?(:rewind) and file.eof?
|
103
|
+
rescue
|
104
|
+
end
|
102
105
|
file
|
103
106
|
when String
|
104
107
|
raise "Could not open file given by String: #{Misc.fingerprint file}" unless Open.remote?(file) or File.exists? file
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
require 'rbbt/util/concurrency/processes/worker'
|
2
2
|
require 'rbbt/util/concurrency/processes/socket'
|
3
3
|
|
4
|
-
|
5
4
|
class RbbtProcessQueue
|
6
5
|
#{{{ RbbtProcessQueue
|
7
6
|
|
8
|
-
attr_accessor :num_processes, :processes, :queue, :process_monitor
|
9
|
-
def initialize(num_processes)
|
7
|
+
attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup
|
8
|
+
def initialize(num_processes, cleanup = nil)
|
10
9
|
@num_processes = num_processes
|
11
10
|
@processes = []
|
11
|
+
@cleanup = cleanup
|
12
12
|
@queue = RbbtProcessSocket.new
|
13
13
|
end
|
14
14
|
|
@@ -24,13 +24,16 @@ class RbbtProcessQueue
|
|
24
24
|
loop do
|
25
25
|
p = @callback_queue.pop
|
26
26
|
raise p if Exception === p
|
27
|
+
raise p.first if Array === p and Exception === p.first
|
27
28
|
@callback.call p
|
28
29
|
end
|
29
30
|
rescue ClosedStream
|
30
31
|
rescue Exception
|
31
|
-
Log.
|
32
|
+
Log.exception $!
|
33
|
+
sleep 1
|
32
34
|
parent.raise $!
|
33
|
-
|
35
|
+
ensure
|
36
|
+
@callback_queue.sread.close unless @callback_queue.sread.closed?
|
34
37
|
end
|
35
38
|
end
|
36
39
|
else
|
@@ -40,47 +43,51 @@ class RbbtProcessQueue
|
|
40
43
|
|
41
44
|
def init(&block)
|
42
45
|
num_processes.times do |i|
|
43
|
-
@processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, &block)
|
46
|
+
@processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, &block)
|
44
47
|
end
|
45
|
-
@queue.
|
46
|
-
@callback_queue.swrite.close if @callback_queue
|
48
|
+
@queue.close_read
|
47
49
|
|
48
50
|
@process_monitor = Thread.new(Thread.current) do |parent|
|
49
51
|
begin
|
50
|
-
while @processes.any?
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
52
|
+
while @processes.any?
|
53
|
+
@processes[0].join
|
54
|
+
@processes.shift
|
55
|
+
end
|
56
|
+
rescue Exception
|
57
|
+
@processes.each do |p|
|
58
|
+
begin
|
59
|
+
Process.kill :INT, p
|
60
|
+
rescue
|
58
61
|
end
|
59
62
|
end
|
60
|
-
|
63
|
+
Log.exception $!
|
61
64
|
parent.raise $!
|
62
|
-
ensure
|
63
|
-
Thread.exit
|
64
65
|
end
|
65
66
|
end
|
66
67
|
end
|
67
68
|
|
68
69
|
def close_callback
|
69
|
-
@
|
70
|
+
@callback_queue.push ClosedStream.new if @callback_thread.alive?
|
71
|
+
@callback_queue.swrite.close
|
72
|
+
@callback_thread.join
|
70
73
|
end
|
71
74
|
|
72
75
|
def join
|
73
|
-
@
|
74
|
-
|
76
|
+
@processes.length.times do
|
77
|
+
@queue.push ClosedStream.new
|
78
|
+
end
|
75
79
|
begin
|
76
80
|
@process_monitor.join
|
77
|
-
ensure
|
78
81
|
close_callback if @callback
|
82
|
+
rescue Exception
|
83
|
+
Log.exception $!
|
84
|
+
ensure
|
85
|
+
@queue.swrite.close
|
79
86
|
end
|
80
87
|
end
|
81
88
|
|
82
89
|
def clean
|
83
|
-
@processes.each{|p| p.abort }
|
90
|
+
@processes.each{|p| p.abort }
|
84
91
|
@callback_thread.raise Aborted if @callback_thread and @callback_thread.alive?
|
85
92
|
end
|
86
93
|
|
@@ -7,7 +7,7 @@ class RbbtProcessQueue
|
|
7
7
|
|
8
8
|
attr_accessor :sread, :swrite, :write_sem, :read_sem
|
9
9
|
def initialize
|
10
|
-
@sread, @swrite =
|
10
|
+
@sread, @swrite = Misc.pipe
|
11
11
|
|
12
12
|
key = rand(100000).to_s;
|
13
13
|
@write_sem = key + '.in'
|
@@ -62,25 +62,33 @@ class RbbtProcessQueue
|
|
62
62
|
end
|
63
63
|
end
|
64
64
|
|
65
|
+
def closed_read?
|
66
|
+
@sread.closed?
|
67
|
+
end
|
68
|
+
|
69
|
+
def closed_write?
|
70
|
+
@swrite.closed?
|
71
|
+
end
|
72
|
+
|
73
|
+
def close_write
|
74
|
+
@swrite.close
|
75
|
+
end
|
76
|
+
|
77
|
+
def close_read
|
78
|
+
@sread.close
|
79
|
+
end
|
65
80
|
#{{{ ACCESSOR
|
81
|
+
|
66
82
|
|
67
83
|
def push(obj)
|
68
|
-
|
69
|
-
|
70
|
-
self.dump(obj, @swrite)
|
71
|
-
end
|
72
|
-
rescue
|
73
|
-
return ClosedStream.new
|
84
|
+
RbbtSemaphore.synchronize(@write_sem) do
|
85
|
+
self.dump(obj, @swrite)
|
74
86
|
end
|
75
87
|
end
|
76
88
|
|
77
89
|
def pop
|
78
|
-
|
79
|
-
|
80
|
-
self.load(@sread)
|
81
|
-
end
|
82
|
-
rescue IOError, ClosedStream
|
83
|
-
return ClosedStream.new
|
90
|
+
RbbtSemaphore.synchronize(@read_sem) do
|
91
|
+
self.load(@sread)
|
84
92
|
end
|
85
93
|
end
|
86
94
|
end
|
@@ -1,14 +1,21 @@
|
|
1
1
|
require 'rbbt/util/concurrency/processes/socket'
|
2
2
|
class RbbtProcessQueue
|
3
3
|
class RbbtProcessQueueWorker
|
4
|
-
|
5
|
-
def initialize(queue, callback_queue = nil, &block)
|
6
|
-
@queue, @callback_queue, @block = queue, callback_queue, block
|
4
|
+
attr_reader :pid, :queue, :callback_queue, :cleanup, :block
|
5
|
+
def initialize(queue, callback_queue = nil, cleanup = nil, &block)
|
6
|
+
@queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
|
7
7
|
|
8
8
|
@pid = Process.fork do
|
9
9
|
begin
|
10
|
-
@
|
11
|
-
@
|
10
|
+
@cleanup.call if @cleanup
|
11
|
+
@queue.close_write
|
12
|
+
|
13
|
+
if @callback_queue
|
14
|
+
Misc.purge_pipes(@callback_queue.swrite)
|
15
|
+
@callback_queue.close_read
|
16
|
+
else
|
17
|
+
Misc.purge_pipes
|
18
|
+
end
|
12
19
|
|
13
20
|
Signal.trap(:INT){ raise Aborted; }
|
14
21
|
loop do
|
@@ -19,26 +26,29 @@ class RbbtProcessQueue
|
|
19
26
|
@callback_queue.push res if @callback_queue
|
20
27
|
end
|
21
28
|
|
22
|
-
exit 0
|
23
29
|
rescue ClosedStream
|
24
|
-
exit 0
|
25
30
|
rescue Aborted
|
26
|
-
exit -1
|
27
|
-
rescue Exception
|
28
31
|
Log.exception $!
|
32
|
+
rescue Exception
|
29
33
|
@callback_queue.push($!) if @callback_queue
|
30
|
-
|
34
|
+
ensure
|
35
|
+
@callback_queue.close_write if @callback_queue
|
31
36
|
end
|
32
|
-
|
33
37
|
end
|
34
38
|
end
|
35
39
|
|
36
40
|
def join
|
37
|
-
|
41
|
+
begin
|
42
|
+
joined_pid = Process.waitpid @pid
|
43
|
+
rescue
|
44
|
+
end
|
38
45
|
end
|
39
46
|
|
40
47
|
def abort
|
41
|
-
|
48
|
+
begin
|
49
|
+
Process.kill :INT, @pid
|
50
|
+
rescue
|
51
|
+
end
|
42
52
|
end
|
43
53
|
|
44
54
|
def done?
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -17,8 +17,7 @@ module Log
|
|
17
17
|
end
|
18
18
|
self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
|
19
19
|
require "highline/system_extensions.rb"
|
20
|
-
self.tty_size = HighLine::SystemExtensions.terminal_size.first
|
21
|
-
|
20
|
+
self.tty_size = HighLine::SystemExtensions.terminal_size.first
|
22
21
|
|
23
22
|
def self.with_severity(level)
|
24
23
|
orig = Log.severity
|
@@ -60,7 +59,7 @@ module Log
|
|
60
59
|
end
|
61
60
|
|
62
61
|
def self.clear_line(out = STDOUT)
|
63
|
-
out.puts Log.return_line << " " * Log.tty_size << Log.return_line unless nocolor
|
62
|
+
out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
|
64
63
|
end
|
65
64
|
|
66
65
|
def self.highlight(str = nil)
|
@@ -74,7 +73,7 @@ module Log
|
|
74
73
|
end
|
75
74
|
|
76
75
|
def self.log(message = nil, severity = MEDIUM, &block)
|
77
|
-
return if severity < self.severity
|
76
|
+
return if severity < self.severity
|
78
77
|
message ||= block.call if block_given?
|
79
78
|
return if message.nil?
|
80
79
|
|
@@ -195,6 +194,21 @@ def iii(message, file = $stdout)
|
|
195
194
|
Log.info{""}
|
196
195
|
end
|
197
196
|
|
197
|
+
def www(message, file = $stdout)
|
198
|
+
stack = caller
|
199
|
+
Log.warn{"#{Log.color :cyan, "INFO:"} " << stack.first}
|
200
|
+
Log.warn{""}
|
201
|
+
Log.warn{"=> " << message.inspect}
|
202
|
+
Log.warn{""}
|
203
|
+
end
|
204
|
+
|
205
|
+
def eee(message, file = $stdout)
|
206
|
+
stack = caller
|
207
|
+
Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}
|
208
|
+
Log.error{""}
|
209
|
+
Log.error{"=> " << message.inspect}
|
210
|
+
Log.error{""}
|
211
|
+
end
|
198
212
|
|
199
213
|
if __FILE__ == $0
|
200
214
|
Log.severity = 0
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -28,10 +28,201 @@ module LaterString
|
|
28
28
|
end
|
29
29
|
end
|
30
30
|
|
31
|
+
module ConcurrentStream
|
32
|
+
attr_accessor :threads, :pids, :callback, :filename
|
33
|
+
|
34
|
+
def consume
|
35
|
+
Thread.pass while IO.select([self], nil, nil).nil? #if IO === self
|
36
|
+
while block = self.read(2048)
|
37
|
+
Thread.pass while IO.select([self], nil, nil).nil? # if IO === self
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def join
|
42
|
+
filename = self.respond_to?(:filename)? self.filename : :none
|
43
|
+
if @callback
|
44
|
+
@callback.call
|
45
|
+
end
|
46
|
+
@threads.each{|t| t.join } if @threads
|
47
|
+
|
48
|
+
@pids.each do |pid|
|
49
|
+
begin
|
50
|
+
Process.waitpid(pid, Process::WUNTRACED)
|
51
|
+
raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
|
52
|
+
rescue Errno::ECHILD
|
53
|
+
end
|
54
|
+
end if @pids
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.setup(stream, options = {}, &block)
|
58
|
+
threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
|
59
|
+
stream.extend ConcurrentStream unless ConcurrentStream === stream
|
60
|
+
|
61
|
+
stream.threads ||= []
|
62
|
+
stream.pids ||= []
|
63
|
+
stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
|
64
|
+
stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
|
65
|
+
|
66
|
+
callback = block if block_given?
|
67
|
+
if stream.callback and callback
|
68
|
+
old_callback = stream.callback
|
69
|
+
stream.callback = Proc.new do
|
70
|
+
old_callback.call
|
71
|
+
callback.call
|
72
|
+
end
|
73
|
+
else
|
74
|
+
stream.callback = callback
|
75
|
+
end
|
76
|
+
|
77
|
+
stream.filename = filename unless filename.nil?
|
78
|
+
|
79
|
+
stream
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
|
31
84
|
Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
|
32
85
|
module Misc
|
33
86
|
|
34
87
|
|
88
|
+
PIPE_MUTEX = Mutex.new
|
89
|
+
|
90
|
+
OPEN_PIPE_IN = []
|
91
|
+
def self.pipe
|
92
|
+
OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
|
93
|
+
PIPE_MUTEX.synchronize do
|
94
|
+
sout, sin = IO.pipe
|
95
|
+
OPEN_PIPE_IN << sin
|
96
|
+
|
97
|
+
[sout, sin]
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.release_pipes(*pipes)
|
102
|
+
PIPE_MUTEX.synchronize do
|
103
|
+
pipes.flatten.each do |pipe|
|
104
|
+
pipe.close unless pipe.closed?
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
def self.purge_pipes(*save)
|
111
|
+
PIPE_MUTEX.synchronize do
|
112
|
+
OPEN_PIPE_IN.each do |pipe|
|
113
|
+
next if save.include? pipe
|
114
|
+
pipe.close unless pipe.closed?
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.open_pipe(do_fork = false, other_stream = nil)
|
120
|
+
raise "No block given" unless block_given?
|
121
|
+
sout, sin = Misc.pipe
|
122
|
+
if do_fork
|
123
|
+
parent_pid = Process.pid
|
124
|
+
pid = Process.fork {
|
125
|
+
purge_pipes(sin)
|
126
|
+
sout.close
|
127
|
+
begin
|
128
|
+
yield sin
|
129
|
+
rescue
|
130
|
+
Log.exception $!
|
131
|
+
Process.kill :INT, parent_pid
|
132
|
+
Kernel.exit! -1
|
133
|
+
ensure
|
134
|
+
Misc.release_pipes(sin)
|
135
|
+
end
|
136
|
+
Kernel.exit! 0
|
137
|
+
}
|
138
|
+
Misc.release_pipes(sin)
|
139
|
+
else
|
140
|
+
thread = Thread.new(Thread.current) do |parent|
|
141
|
+
begin
|
142
|
+
yield sin
|
143
|
+
rescue
|
144
|
+
Log.exception $!
|
145
|
+
parent.raise $!
|
146
|
+
ensure
|
147
|
+
Misc.release_pipes(sin)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
sout
|
152
|
+
end
|
153
|
+
|
154
|
+
def self.tee_stream_fork(stream)
|
155
|
+
stream_out1, stream_in1 = Misc.pipe
|
156
|
+
stream_out2, stream_in2 = Misc.pipe
|
157
|
+
|
158
|
+
splitter_pid = Process.fork do
|
159
|
+
Misc.purge_pipes(stream_in1, stream_in2)
|
160
|
+
stream_out1.close
|
161
|
+
stream_out2.close
|
162
|
+
begin
|
163
|
+
filename = stream.respond_to?(:filename)? stream.filename : nil
|
164
|
+
while block = stream.read(2048)
|
165
|
+
begin stream_in1.write block; rescue Exception; Log.exception $! end
|
166
|
+
begin stream_in2.write block; rescue Exception; Log.exception $! end
|
167
|
+
end
|
168
|
+
rescue IOError
|
169
|
+
Log.exception $!
|
170
|
+
rescue Exception
|
171
|
+
Log.exception $!
|
172
|
+
ensure
|
173
|
+
if stream.respond_to? :join
|
174
|
+
stream.join
|
175
|
+
end
|
176
|
+
Misc.release_pipes(stream_in1)
|
177
|
+
Misc.release_pipes(stream_in2)
|
178
|
+
end
|
179
|
+
end
|
180
|
+
stream.close
|
181
|
+
stream_in1.close
|
182
|
+
stream_in2.close
|
183
|
+
#stream.join if stream.respond_to? :join
|
184
|
+
|
185
|
+
ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
|
186
|
+
ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
|
187
|
+
|
188
|
+
[stream_out1, stream_out2]
|
189
|
+
end
|
190
|
+
|
191
|
+
def self.tee_stream_thread(stream)
|
192
|
+
stream_out1, stream_in1 = Misc.pipe
|
193
|
+
stream_out2, stream_in2 = Misc.pipe
|
194
|
+
|
195
|
+
splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
|
196
|
+
begin
|
197
|
+
filename = stream.respond_to?(:filename)? stream.filename : nil
|
198
|
+
while block = stream.read(2048)
|
199
|
+
begin stream_in1.write block; rescue Exception; Log.exception $! end
|
200
|
+
begin stream_in2.write block; rescue Exception; Log.exception $! end
|
201
|
+
end
|
202
|
+
rescue IOError
|
203
|
+
Log.exception $!
|
204
|
+
rescue Exception
|
205
|
+
Log.exception $!
|
206
|
+
parent.raise $!
|
207
|
+
ensure
|
208
|
+
if stream.respond_to? :join
|
209
|
+
stream.join
|
210
|
+
end
|
211
|
+
Misc.release_pipes(stream_in1)
|
212
|
+
Misc.release_pipes(stream_in2)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
|
216
|
+
ConcurrentStream.setup stream_out1, :threads => splitter_thread
|
217
|
+
ConcurrentStream.setup stream_out2, :threads => splitter_thread
|
218
|
+
|
219
|
+
[stream_out1, stream_out2]
|
220
|
+
end
|
221
|
+
|
222
|
+
class << self
|
223
|
+
alias tee_stream tee_stream_fork
|
224
|
+
end
|
225
|
+
|
35
226
|
def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
|
36
227
|
i = 0
|
37
228
|
re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
|
@@ -89,8 +280,10 @@ module Misc
|
|
89
280
|
|
90
281
|
def self.read_stream(stream, size)
|
91
282
|
str = nil
|
283
|
+
Thread.pass while IO.select([stream],nil,nil,1).nil?
|
92
284
|
while not str = stream.read(size)
|
93
285
|
IO.select([stream],nil,nil,1)
|
286
|
+
Thread.pass
|
94
287
|
raise ClosedStream if stream.eof?
|
95
288
|
end
|
96
289
|
|
@@ -1114,7 +1307,6 @@ end
|
|
1114
1307
|
res = yield file, *args
|
1115
1308
|
end
|
1116
1309
|
rescue Interrupt
|
1117
|
-
Log.error "Process #{Process.pid} interrupted while in lock: #{ lock_path }"
|
1118
1310
|
raise $!
|
1119
1311
|
end
|
1120
1312
|
|
@@ -1219,9 +1411,12 @@ end
|
|
1219
1411
|
File.open(tmp_path, 'w', &block)
|
1220
1412
|
when String === content
|
1221
1413
|
File.open(tmp_path, 'w') do |f| f.write content end
|
1222
|
-
when (IO === content or StringIO === content)
|
1414
|
+
when (IO === content or StringIO === content or File === content)
|
1415
|
+
#Thread.pass while IO.select([content], nil, nil, 1) if IO === content
|
1223
1416
|
File.open(tmp_path, 'w') do |f|
|
1224
|
-
while
|
1417
|
+
while block = content.read(2048);
|
1418
|
+
f.write block
|
1419
|
+
end
|
1225
1420
|
end
|
1226
1421
|
else
|
1227
1422
|
File.open(tmp_path, 'w') do |f| end
|
@@ -1512,21 +1707,6 @@ end
|
|
1512
1707
|
chunks
|
1513
1708
|
end
|
1514
1709
|
|
1515
|
-
def self.open_pipe
|
1516
|
-
sout, sin = IO.pipe
|
1517
|
-
raise "No block given" unless block_given?
|
1518
|
-
Thread.new{
|
1519
|
-
begin
|
1520
|
-
yield sin
|
1521
|
-
rescue
|
1522
|
-
Log.exception $!
|
1523
|
-
raise $!
|
1524
|
-
ensure
|
1525
|
-
sin.close
|
1526
|
-
end
|
1527
|
-
}
|
1528
|
-
sout
|
1529
|
-
end
|
1530
1710
|
|
1531
1711
|
def self.append_zipped(current, new)
|
1532
1712
|
current.each do |v|
|