rbbt-util 5.10.1 → 5.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,6 +16,7 @@ module TSV
16
16
 
17
17
  # Get line
18
18
 
19
+ Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
19
20
  line = stream.gets
20
21
  raise "Empty content" if line.nil?
21
22
  line = Misc.fixutf8 line
@@ -40,6 +41,7 @@ module TSV
40
41
  @key_field = @fields.shift
41
42
  @key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
42
43
 
44
+ Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
43
45
  line = @header_hash != "" ? Misc.fixutf8(stream.gets) : nil
44
46
  end
45
47
 
@@ -348,7 +350,8 @@ module TSV
348
350
 
349
351
  options = header_options.merge options
350
352
 
351
- @type = Misc.process_options(options, :type) || :double
353
+ @type ||= Misc.process_options(options, :type) || :double
354
+ @type ||= :double
352
355
 
353
356
  @filename = Misc.process_options(options, :filename)
354
357
  @filename ||= stream.filename if stream.respond_to? :filename
@@ -497,7 +500,10 @@ module TSV
497
500
 
498
501
  yield key, values
499
502
 
503
+ Thread.pass while IO.select([stream], nil, nil, 1).nil? if IO === stream
504
+
500
505
  line = stream.gets
506
+
501
507
  line_num += 1
502
508
  raise END_PARSING if head and line_num > head.to_i
503
509
  rescue SKIP_LINE
@@ -514,8 +520,6 @@ module TSV
514
520
  break
515
521
  end
516
522
  end
517
- #ensure
518
- # stream.close unless stream.closed?
519
523
  end
520
524
 
521
525
  self
data/lib/rbbt/tsv/util.rb CHANGED
@@ -98,7 +98,10 @@ module TSV
98
98
  when Path
99
99
  file.open(open_options)
100
100
  when IO, StringIO
101
- file.rewind if file.respond_to?(:rewind) and file.eof?
101
+ begin
102
+ file.rewind if file.respond_to?(:rewind) and file.eof?
103
+ rescue
104
+ end
102
105
  file
103
106
  when String
104
107
  raise "Could not open file given by String: #{Misc.fingerprint file}" unless Open.remote?(file) or File.exists? file
data/lib/rbbt/util/cmd.rb CHANGED
@@ -124,7 +124,7 @@ module CMD
124
124
 
125
125
  in_content = StringIO.new in_content if String === in_content
126
126
 
127
- sout, serr, sin = IO.pipe, IO.pipe, IO.pipe
127
+ sout, serr, sin = Misc.pipe, Misc.pipe, Misc.pipe
128
128
 
129
129
  pid = fork {
130
130
  begin
@@ -1,14 +1,14 @@
1
1
  require 'rbbt/util/concurrency/processes/worker'
2
2
  require 'rbbt/util/concurrency/processes/socket'
3
3
 
4
-
5
4
  class RbbtProcessQueue
6
5
  #{{{ RbbtProcessQueue
7
6
 
8
- attr_accessor :num_processes, :processes, :queue, :process_monitor
9
- def initialize(num_processes)
7
+ attr_accessor :num_processes, :processes, :queue, :process_monitor, :cleanup
8
+ def initialize(num_processes, cleanup = nil)
10
9
  @num_processes = num_processes
11
10
  @processes = []
11
+ @cleanup = cleanup
12
12
  @queue = RbbtProcessSocket.new
13
13
  end
14
14
 
@@ -24,13 +24,16 @@ class RbbtProcessQueue
24
24
  loop do
25
25
  p = @callback_queue.pop
26
26
  raise p if Exception === p
27
+ raise p.first if Array === p and Exception === p.first
27
28
  @callback.call p
28
29
  end
29
30
  rescue ClosedStream
30
31
  rescue Exception
31
- Log.debug $!
32
+ Log.exception $!
33
+ sleep 1
32
34
  parent.raise $!
33
- Thread.exit
35
+ ensure
36
+ @callback_queue.sread.close unless @callback_queue.sread.closed?
34
37
  end
35
38
  end
36
39
  else
@@ -40,47 +43,51 @@ class RbbtProcessQueue
40
43
 
41
44
  def init(&block)
42
45
  num_processes.times do |i|
43
- @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, &block)
46
+ @processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, &block)
44
47
  end
45
- @queue.sread.close
46
- @callback_queue.swrite.close if @callback_queue
48
+ @queue.close_read
47
49
 
48
50
  @process_monitor = Thread.new(Thread.current) do |parent|
49
51
  begin
50
- while @processes.any? do
51
- pid = Process.wait -1, Process::WNOHANG
52
- if pid
53
- next unless @processes.collect{|p| p.pid }.include? pid
54
- @processes.delete_if{|p| p.pid == pid}
55
- raise "Process #{pid} failed" unless $?.success?
56
- else
57
- sleep 1
52
+ while @processes.any?
53
+ @processes[0].join
54
+ @processes.shift
55
+ end
56
+ rescue Exception
57
+ @processes.each do |p|
58
+ begin
59
+ Process.kill :INT, p
60
+ rescue
58
61
  end
59
62
  end
60
- rescue
63
+ Log.exception $!
61
64
  parent.raise $!
62
- ensure
63
- Thread.exit
64
65
  end
65
66
  end
66
67
  end
67
68
 
68
69
  def close_callback
69
- @callback_thread.join if @callback_thread and @callback_thread.alive?
70
+ @callback_queue.push ClosedStream.new if @callback_thread.alive?
71
+ @callback_queue.swrite.close
72
+ @callback_thread.join
70
73
  end
71
74
 
72
75
  def join
73
- @queue.push ClosedStream.new
74
- @queue.swrite.close
76
+ @processes.length.times do
77
+ @queue.push ClosedStream.new
78
+ end
75
79
  begin
76
80
  @process_monitor.join
77
- ensure
78
81
  close_callback if @callback
82
+ rescue Exception
83
+ Log.exception $!
84
+ ensure
85
+ @queue.swrite.close
79
86
  end
80
87
  end
81
88
 
82
89
  def clean
83
- @processes.each{|p| p.abort }.clear
90
+ @processes.each{|p| p.abort }
84
91
  @callback_thread.raise Aborted if @callback_thread and @callback_thread.alive?
85
92
  end
86
93
 
@@ -7,7 +7,7 @@ class RbbtProcessQueue
7
7
 
8
8
  attr_accessor :sread, :swrite, :write_sem, :read_sem
9
9
  def initialize
10
- @sread, @swrite = IO.pipe
10
+ @sread, @swrite = Misc.pipe
11
11
 
12
12
  key = rand(100000).to_s;
13
13
  @write_sem = key + '.in'
@@ -62,25 +62,33 @@ class RbbtProcessQueue
62
62
  end
63
63
  end
64
64
 
65
+ def closed_read?
66
+ @sread.closed?
67
+ end
68
+
69
+ def closed_write?
70
+ @swrite.closed?
71
+ end
72
+
73
+ def close_write
74
+ @swrite.close
75
+ end
76
+
77
+ def close_read
78
+ @sread.close
79
+ end
65
80
  #{{{ ACCESSOR
81
+
66
82
 
67
83
  def push(obj)
68
- begin
69
- RbbtSemaphore.synchronize(@write_sem) do
70
- self.dump(obj, @swrite)
71
- end
72
- rescue
73
- return ClosedStream.new
84
+ RbbtSemaphore.synchronize(@write_sem) do
85
+ self.dump(obj, @swrite)
74
86
  end
75
87
  end
76
88
 
77
89
  def pop
78
- begin
79
- RbbtSemaphore.synchronize(@read_sem) do
80
- self.load(@sread)
81
- end
82
- rescue IOError, ClosedStream
83
- return ClosedStream.new
90
+ RbbtSemaphore.synchronize(@read_sem) do
91
+ self.load(@sread)
84
92
  end
85
93
  end
86
94
  end
@@ -1,14 +1,21 @@
1
1
  require 'rbbt/util/concurrency/processes/socket'
2
2
  class RbbtProcessQueue
3
3
  class RbbtProcessQueueWorker
4
- attr_accessor :pid, :queue, :callback_queue, :block
5
- def initialize(queue, callback_queue = nil, &block)
6
- @queue, @callback_queue, @block = queue, callback_queue, block
4
+ attr_reader :pid, :queue, :callback_queue, :cleanup, :block
5
+ def initialize(queue, callback_queue = nil, cleanup = nil, &block)
6
+ @queue, @callback_queue, @cleanup, @block = queue, callback_queue, cleanup, block
7
7
 
8
8
  @pid = Process.fork do
9
9
  begin
10
- @queue.swrite.close
11
- @callback_queue.sread.close if @callback_queue
10
+ @cleanup.call if @cleanup
11
+ @queue.close_write
12
+
13
+ if @callback_queue
14
+ Misc.purge_pipes(@callback_queue.swrite)
15
+ @callback_queue.close_read
16
+ else
17
+ Misc.purge_pipes
18
+ end
12
19
 
13
20
  Signal.trap(:INT){ raise Aborted; }
14
21
  loop do
@@ -19,26 +26,29 @@ class RbbtProcessQueue
19
26
  @callback_queue.push res if @callback_queue
20
27
  end
21
28
 
22
- exit 0
23
29
  rescue ClosedStream
24
- exit 0
25
30
  rescue Aborted
26
- exit -1
27
- rescue Exception
28
31
  Log.exception $!
32
+ rescue Exception
29
33
  @callback_queue.push($!) if @callback_queue
30
- exit -1
34
+ ensure
35
+ @callback_queue.close_write if @callback_queue
31
36
  end
32
-
33
37
  end
34
38
  end
35
39
 
36
40
  def join
37
- Process.waitpid @pid
41
+ begin
42
+ joined_pid = Process.waitpid @pid
43
+ rescue
44
+ end
38
45
  end
39
46
 
40
47
  def abort
41
- Process.kill :INT, @pid
48
+ begin
49
+ Process.kill :INT, @pid
50
+ rescue
51
+ end
42
52
  end
43
53
 
44
54
  def done?
data/lib/rbbt/util/log.rb CHANGED
@@ -17,8 +17,7 @@ module Log
17
17
  end
18
18
  self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
19
19
  require "highline/system_extensions.rb"
20
- self.tty_size = HighLine::SystemExtensions.terminal_size.first
21
-
20
+ self.tty_size = HighLine::SystemExtensions.terminal_size.first
22
21
 
23
22
  def self.with_severity(level)
24
23
  orig = Log.severity
@@ -60,7 +59,7 @@ module Log
60
59
  end
61
60
 
62
61
  def self.clear_line(out = STDOUT)
63
- out.puts Log.return_line << " " * Log.tty_size << Log.return_line unless nocolor
62
+ out.puts Log.return_line << " " * (Log.tty_size || 80) << Log.return_line unless nocolor
64
63
  end
65
64
 
66
65
  def self.highlight(str = nil)
@@ -74,7 +73,7 @@ module Log
74
73
  end
75
74
 
76
75
  def self.log(message = nil, severity = MEDIUM, &block)
77
- return if severity < self.severity
76
+ return if severity < self.severity
78
77
  message ||= block.call if block_given?
79
78
  return if message.nil?
80
79
 
@@ -195,6 +194,21 @@ def iii(message, file = $stdout)
195
194
  Log.info{""}
196
195
  end
197
196
 
197
+ def www(message, file = $stdout)
198
+ stack = caller
199
+ Log.warn{"#{Log.color :cyan, "INFO:"} " << stack.first}
200
+ Log.warn{""}
201
+ Log.warn{"=> " << message.inspect}
202
+ Log.warn{""}
203
+ end
204
+
205
+ def eee(message, file = $stdout)
206
+ stack = caller
207
+ Log.error{"#{Log.color :cyan, "INFO:"} " << stack.first}
208
+ Log.error{""}
209
+ Log.error{"=> " << message.inspect}
210
+ Log.error{""}
211
+ end
198
212
 
199
213
  if __FILE__ == $0
200
214
  Log.severity = 0
@@ -28,10 +28,201 @@ module LaterString
28
28
  end
29
29
  end
30
30
 
31
+ module ConcurrentStream
32
+ attr_accessor :threads, :pids, :callback, :filename
33
+
34
+ def consume
35
+ Thread.pass while IO.select([self], nil, nil).nil? #if IO === self
36
+ while block = self.read(2048)
37
+ Thread.pass while IO.select([self], nil, nil).nil? # if IO === self
38
+ end
39
+ end
40
+
41
+ def join
42
+ filename = self.respond_to?(:filename)? self.filename : :none
43
+ if @callback
44
+ @callback.call
45
+ end
46
+ @threads.each{|t| t.join } if @threads
47
+
48
+ @pids.each do |pid|
49
+ begin
50
+ Process.waitpid(pid, Process::WUNTRACED)
51
+ raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
52
+ rescue Errno::ECHILD
53
+ end
54
+ end if @pids
55
+ end
56
+
57
+ def self.setup(stream, options = {}, &block)
58
+ threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
59
+ stream.extend ConcurrentStream unless ConcurrentStream === stream
60
+
61
+ stream.threads ||= []
62
+ stream.pids ||= []
63
+ stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
64
+ stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
65
+
66
+ callback = block if block_given?
67
+ if stream.callback and callback
68
+ old_callback = stream.callback
69
+ stream.callback = Proc.new do
70
+ old_callback.call
71
+ callback.call
72
+ end
73
+ else
74
+ stream.callback = callback
75
+ end
76
+
77
+ stream.filename = filename unless filename.nil?
78
+
79
+ stream
80
+ end
81
+ end
82
+
83
+
31
84
  Lockfile.refresh = false if ENV["RBBT_NO_LOCKFILE_REFRESH"] == "true"
32
85
  module Misc
33
86
 
34
87
 
88
+ PIPE_MUTEX = Mutex.new
89
+
90
+ OPEN_PIPE_IN = []
91
+ def self.pipe
92
+ OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
93
+ PIPE_MUTEX.synchronize do
94
+ sout, sin = IO.pipe
95
+ OPEN_PIPE_IN << sin
96
+
97
+ [sout, sin]
98
+ end
99
+ end
100
+
101
+ def self.release_pipes(*pipes)
102
+ PIPE_MUTEX.synchronize do
103
+ pipes.flatten.each do |pipe|
104
+ pipe.close unless pipe.closed?
105
+ end
106
+ end
107
+ end
108
+
109
+
110
+ def self.purge_pipes(*save)
111
+ PIPE_MUTEX.synchronize do
112
+ OPEN_PIPE_IN.each do |pipe|
113
+ next if save.include? pipe
114
+ pipe.close unless pipe.closed?
115
+ end
116
+ end
117
+ end
118
+
119
+ def self.open_pipe(do_fork = false, other_stream = nil)
120
+ raise "No block given" unless block_given?
121
+ sout, sin = Misc.pipe
122
+ if do_fork
123
+ parent_pid = Process.pid
124
+ pid = Process.fork {
125
+ purge_pipes(sin)
126
+ sout.close
127
+ begin
128
+ yield sin
129
+ rescue
130
+ Log.exception $!
131
+ Process.kill :INT, parent_pid
132
+ Kernel.exit! -1
133
+ ensure
134
+ Misc.release_pipes(sin)
135
+ end
136
+ Kernel.exit! 0
137
+ }
138
+ Misc.release_pipes(sin)
139
+ else
140
+ thread = Thread.new(Thread.current) do |parent|
141
+ begin
142
+ yield sin
143
+ rescue
144
+ Log.exception $!
145
+ parent.raise $!
146
+ ensure
147
+ Misc.release_pipes(sin)
148
+ end
149
+ end
150
+ end
151
+ sout
152
+ end
153
+
154
+ def self.tee_stream_fork(stream)
155
+ stream_out1, stream_in1 = Misc.pipe
156
+ stream_out2, stream_in2 = Misc.pipe
157
+
158
+ splitter_pid = Process.fork do
159
+ Misc.purge_pipes(stream_in1, stream_in2)
160
+ stream_out1.close
161
+ stream_out2.close
162
+ begin
163
+ filename = stream.respond_to?(:filename)? stream.filename : nil
164
+ while block = stream.read(2048)
165
+ begin stream_in1.write block; rescue Exception; Log.exception $! end
166
+ begin stream_in2.write block; rescue Exception; Log.exception $! end
167
+ end
168
+ rescue IOError
169
+ Log.exception $!
170
+ rescue Exception
171
+ Log.exception $!
172
+ ensure
173
+ if stream.respond_to? :join
174
+ stream.join
175
+ end
176
+ Misc.release_pipes(stream_in1)
177
+ Misc.release_pipes(stream_in2)
178
+ end
179
+ end
180
+ stream.close
181
+ stream_in1.close
182
+ stream_in2.close
183
+ #stream.join if stream.respond_to? :join
184
+
185
+ ConcurrentStream.setup stream_out1, :pids => [splitter_pid]
186
+ ConcurrentStream.setup stream_out2, :pids => [splitter_pid]
187
+
188
+ [stream_out1, stream_out2]
189
+ end
190
+
191
+ def self.tee_stream_thread(stream)
192
+ stream_out1, stream_in1 = Misc.pipe
193
+ stream_out2, stream_in2 = Misc.pipe
194
+
195
+ splitter_thread = Thread.new(Thread.current, stream_in1, stream_in2) do |parent,stream_in1,stream_in2|
196
+ begin
197
+ filename = stream.respond_to?(:filename)? stream.filename : nil
198
+ while block = stream.read(2048)
199
+ begin stream_in1.write block; rescue Exception; Log.exception $! end
200
+ begin stream_in2.write block; rescue Exception; Log.exception $! end
201
+ end
202
+ rescue IOError
203
+ Log.exception $!
204
+ rescue Exception
205
+ Log.exception $!
206
+ parent.raise $!
207
+ ensure
208
+ if stream.respond_to? :join
209
+ stream.join
210
+ end
211
+ Misc.release_pipes(stream_in1)
212
+ Misc.release_pipes(stream_in2)
213
+ end
214
+ end
215
+
216
+ ConcurrentStream.setup stream_out1, :threads => splitter_thread
217
+ ConcurrentStream.setup stream_out2, :threads => splitter_thread
218
+
219
+ [stream_out1, stream_out2]
220
+ end
221
+
222
+ class << self
223
+ alias tee_stream tee_stream_fork
224
+ end
225
+
35
226
  def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
36
227
  i = 0
37
228
  re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
@@ -89,8 +280,10 @@ module Misc
89
280
 
90
281
  def self.read_stream(stream, size)
91
282
  str = nil
283
+ Thread.pass while IO.select([stream],nil,nil,1).nil?
92
284
  while not str = stream.read(size)
93
285
  IO.select([stream],nil,nil,1)
286
+ Thread.pass
94
287
  raise ClosedStream if stream.eof?
95
288
  end
96
289
 
@@ -1114,7 +1307,6 @@ end
1114
1307
  res = yield file, *args
1115
1308
  end
1116
1309
  rescue Interrupt
1117
- Log.error "Process #{Process.pid} interrupted while in lock: #{ lock_path }"
1118
1310
  raise $!
1119
1311
  end
1120
1312
 
@@ -1219,9 +1411,12 @@ end
1219
1411
  File.open(tmp_path, 'w', &block)
1220
1412
  when String === content
1221
1413
  File.open(tmp_path, 'w') do |f| f.write content end
1222
- when (IO === content or StringIO === content)
1414
+ when (IO === content or StringIO === content or File === content)
1415
+ #Thread.pass while IO.select([content], nil, nil, 1) if IO === content
1223
1416
  File.open(tmp_path, 'w') do |f|
1224
- while l = content.gets; f.write l; end
1417
+ while block = content.read(2048);
1418
+ f.write block
1419
+ end
1225
1420
  end
1226
1421
  else
1227
1422
  File.open(tmp_path, 'w') do |f| end
@@ -1512,21 +1707,6 @@ end
1512
1707
  chunks
1513
1708
  end
1514
1709
 
1515
- def self.open_pipe
1516
- sout, sin = IO.pipe
1517
- raise "No block given" unless block_given?
1518
- Thread.new{
1519
- begin
1520
- yield sin
1521
- rescue
1522
- Log.exception $!
1523
- raise $!
1524
- ensure
1525
- sin.close
1526
- end
1527
- }
1528
- sout
1529
- end
1530
1710
 
1531
1711
  def self.append_zipped(current, new)
1532
1712
  current.each do |v|