rbbt-util 5.11.2 → 5.11.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 06c3a98f22b69a5738d6c9b478a2cee9a4e52e66
4
- data.tar.gz: d60e07e6371c2c6c88359d6b028e7c7fdcbd0c64
3
+ metadata.gz: add772f9d207913045a9042098c44b2c5549215c
4
+ data.tar.gz: 5932efc5c957bc4cf93de0228d4e654a47a788ce
5
5
  SHA512:
6
- metadata.gz: dab5cbee06c6169ab384555b53e6ea3407cb2c3f18b0b8d99f9aac6d6209d4c8656a6baeba4d46ede3c87c8eb141a870723d7a8336f7f58c85535d6d2f3f54a6
7
- data.tar.gz: e3268aebe76ca88cfbc948b34367992188e38289fe966fc90ffc07609e2f6b355d751327008b691fbe4e94a1d7b9085e32fd0f73734ee538b041b9bd982486b0
6
+ metadata.gz: 864893d125d7ca6e03736829a5c77d11e936229dfd3569139c22b1acd10d8430156603b8a23f3ecf5c9b0602040278b6fff1482ccddde3cc9ca3686f40c16791
7
+ data.tar.gz: 71da4b77c4fad1c83fcaa068118150bd6cfbc0e6e029a2bd3ffc2867293fa585cba4a4638eff98df9d4120fe2a95847c290ae7f6b07bfad2699ba15566f1a99c
@@ -83,48 +83,53 @@ module Persist
83
83
 
84
84
  TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
85
85
  def self.load_file(path, type)
86
- case (type || :marshal).to_sym
87
- when :nil
88
- nil
89
- when :boolean
90
- TRUE_STRINGS.include? Open.read(path).chomp.strip
91
- when :annotations
92
- Annotated.load_tsv TSV.open(path)
93
- when :tsv
94
- TSV.open(path)
95
- when :marshal_tsv
96
- TSV.setup(Marshal.load(Open.open(path)))
97
- when :fwt
98
- FixWidthTable.get(path)
99
- when :string, :text
100
- Open.read(path)
101
- when :binary
102
- f = File.open(path, 'rb')
103
- res = f.read
104
- f.close
105
- res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
106
- res
107
- when :array
108
- res = Open.read(path).split("\n", -1)
109
- res.pop
110
- res
111
- when :marshal
112
- Marshal.load(Open.open(path))
113
- when :yaml
114
- YAML.load(Open.open(path))
115
- when :float
116
- Open.read(path).to_f
117
- when :integer
118
- Open.read(path).to_i
119
- else
120
- raise "Unknown persistence: #{ type }"
86
+ begin
87
+ case (type || :marshal).to_sym
88
+ when :nil
89
+ nil
90
+ when :boolean
91
+ TRUE_STRINGS.include? Open.read(path).chomp.strip
92
+ when :annotations
93
+ Annotated.load_tsv TSV.open(path)
94
+ when :tsv
95
+ TSV.open(path)
96
+ when :marshal_tsv
97
+ TSV.setup(Marshal.load(Open.open(path)))
98
+ when :fwt
99
+ FixWidthTable.get(path)
100
+ when :string, :text
101
+ Open.read(path)
102
+ when :binary
103
+ f = File.open(path, 'rb')
104
+ res = f.read
105
+ f.close
106
+ res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
107
+ res
108
+ when :array
109
+ res = Open.read(path).split("\n", -1)
110
+ res.pop
111
+ res
112
+ when :marshal
113
+ Marshal.load(Open.open(path))
114
+ when :yaml
115
+ YAML.load(Open.open(path))
116
+ when :float
117
+ Open.read(path).to_f
118
+ when :integer
119
+ Open.read(path).to_i
120
+ else
121
+ raise "Unknown persistence: #{ type }"
122
+ end
123
+ rescue
124
+ Log.error "Exception loading #{ type } #{ path }: #{$!.message}"
125
+ raise $!
121
126
  end
122
127
  end
123
128
 
124
129
  def self.save_file(path, type, content)
125
130
 
126
131
  return if content.nil?
127
-
132
+
128
133
  case (type || :marshal).to_sym
129
134
  when :nil
130
135
  nil
@@ -248,9 +253,11 @@ module Persist
248
253
  end
249
254
  rescue Aborted
250
255
  Log.error "Tee stream thread aborted"
251
- stream.abort if stream.respond_to? :abort
256
+ sout.abort if sout.respond_to? :abort
257
+ sin.abort if sin.respond_to? :abort
252
258
  rescue Exception
253
- stream.abort if stream.respond_to? :abort
259
+ sin.abort if sin.respond_to? :abort
260
+ sout.abort if sout.respond_to? :abort
254
261
  Log.exception $!
255
262
  parent.raise $!
256
263
  ensure
@@ -413,6 +420,7 @@ module Persist
413
420
  res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
414
421
  ConcurrentStream.setup res do
415
422
  begin
423
+ iii [:unlock, lockfile.path]
416
424
  lockfile.unlock
417
425
  rescue
418
426
  Log.warn "Lockfile exception: " << $!.message
@@ -199,12 +199,9 @@ module TSV
199
199
  end
200
200
  end
201
201
 
202
- #stream = obj_stream(obj)
203
- #stream.close if stream
204
-
205
202
  Process.waitpid pid
206
203
  rescue Exception
207
- Log.exception $!
204
+ Log.error "Exception traversin in cpus: #{$!.message}"
208
205
  q.abort
209
206
  raise $!
210
207
  ensure
@@ -9,7 +9,7 @@ class RbbtProcessQueue
9
9
  def initialize
10
10
  @sread, @swrite = Misc.pipe
11
11
 
12
- key = rand(100000).to_s;
12
+ key = rand(100000000).to_s;
13
13
  @write_sem = key + '.in'
14
14
  @read_sem = key + '.out'
15
15
  RbbtSemaphore.create_semaphore(@write_sem,1)
@@ -35,7 +35,7 @@ module LaterString
35
35
  end
36
36
 
37
37
  module ConcurrentStream
38
- attr_accessor :threads, :pids, :callback, :filename, :joined
38
+ attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
39
39
 
40
40
  def joined?
41
41
  @joined
@@ -72,6 +72,7 @@ module ConcurrentStream
72
72
  def abort
73
73
  @threads.each{|t| t.raise Aborted.new } if @threads
74
74
  @pids.each{|pid| Process.kill :INT, pid } if @pids
75
+ @abort_callback.call if @abort_callback
75
76
  end
76
77
 
77
78
  def self.setup(stream, options = {}, &block)
@@ -234,7 +234,8 @@ module Workflow
234
234
  def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
235
235
  step_path = step_path.call if Proc === step_path
236
236
  persist = input_values.nil? ? false : true
237
- step = Persist.memory("Step", :key => step_path, :repo => step_cache, :persist => persist ) do
237
+ key = Path === step_path ? step_path.find : step_path
238
+ step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist ) do
238
239
  step = Step.new step_path, task, input_values, dependencies
239
240
 
240
241
  helpers.each do |name, block|
@@ -105,10 +105,21 @@ class Step
105
105
  def self.log(status, message, path, &block)
106
106
  if block_given?
107
107
  start = Time.now
108
+ status = status.to_s
109
+ status_color = case status
110
+ when "starting"
111
+ :yellow
112
+ when "error"
113
+ :red
114
+ when "done"
115
+ :green
116
+ else
117
+ :cyan
118
+ end
108
119
  Log.info do
109
120
  now = Time.now
110
121
  str = Log.color :reset
111
- str << "#{ Log.color :cyan, status.to_s }"
122
+ str << "#{ Log.color status_color, status}"
112
123
  str << ": #{ message }" if message
113
124
  str << " -- #{Log.color :blue, path.to_s}" if path
114
125
  str
@@ -123,10 +134,21 @@ class Step
123
134
  end
124
135
  res
125
136
  else
137
+ status = status.to_s
138
+ status_color = case status
139
+ when "starting"
140
+ :yellow
141
+ when "error"
142
+ :red
143
+ when "done"
144
+ :green
145
+ else
146
+ :cyan
147
+ end
126
148
  Log.info do
127
149
  now = Time.now
128
150
  str = Log.color :reset
129
- str << "#{ Log.color :cyan, status.to_s }"
151
+ str << "#{ Log.color status_color, status}"
130
152
  str << ": #{ message }" if message
131
153
  str << " -- #{Log.color :blue, path.to_s}" if path
132
154
  str
@@ -52,7 +52,7 @@ class Step
52
52
  def log(status, message = nil)
53
53
  self.status = status
54
54
  message message
55
- relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message )
55
+ relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless relay_step.done? or relay_step.error? or relay_step.aborted?
56
56
  end
57
57
  end
58
58
  end
@@ -80,8 +80,8 @@ class Step
80
80
  rescue Exception
81
81
  value.abort if value.respond_to? :abort
82
82
  ensure
83
- value.close unless value.closed?
84
83
  value.join if value.respond_to? :join
84
+ value.close unless value.closed?
85
85
  end
86
86
  when (not defined? Entity or description.nil? or not Entity.formats.include? description)
87
87
  value
@@ -152,10 +152,26 @@ class Step
152
152
  rec_dependencies.collect{|dependency| dependency.path }.uniq
153
153
  end
154
154
 
155
+ def kill_children
156
+ children_pids = info[:children_pids]
157
+ if children_pids and children_pids.any?
158
+ Log.medium("Killing children: #{ children_pids * ", " }")
159
+ children_pids.each do |pid|
160
+ Log.medium("Killing child #{ pid }")
161
+ begin
162
+ Process.kill "INT", pid
163
+ rescue Exception
164
+ Log.medium("Exception killing child #{ pid }: #{$!.message}")
165
+ end
166
+ end
167
+ end
168
+ end
169
+
155
170
  def run(no_load = false)
156
171
 
157
172
  @mutex.synchronize do
158
- result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load ? :stream : false do
173
+ no_load = no_load ? :stream : false
174
+ result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
159
175
  if Step === Step.log_relay_step and not self == Step.log_relay_step
160
176
  relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
161
177
  end
@@ -176,7 +192,8 @@ class Step
176
192
  dependency.relay_log self
177
193
  dependency.clean if not dependency.done? and dependency.error?
178
194
  dependency.clean if dependency.streaming? and not dependency.running?
179
- dependency.run true unless dependency.result
195
+ dependency.run true unless dependency.result or dependency.done?
196
+ seen_deps << dependency.path
180
197
  seen_deps.concat dependency.rec_dependencies.collect{|d| d.path}
181
198
  rescue Exception
182
199
  backtrace = $!.backtrace
@@ -196,29 +213,30 @@ class Step
196
213
  rescue Aborted
197
214
  log(:error, "Aborted")
198
215
 
199
- children_pids = info[:children_pids]
200
- if children_pids and children_pids.any?
201
- Log.medium("Killing children: #{ children_pids * ", " }")
202
- children_pids.each do |pid|
203
- Log.medium("Killing child #{ pid }")
204
- begin
205
- Process.kill "INT", pid
206
- rescue Exception
207
- Log.medium("Exception killing child #{ pid }: #{$!.message}")
208
- end
209
- end
210
- end
211
-
216
+ # children_pids = info[:children_pids]
217
+ # if children_pids and children_pids.any?
218
+ # Log.medium("Killing children: #{ children_pids * ", " }")
219
+ # children_pids.each do |pid|
220
+ # Log.medium("Killing child #{ pid }")
221
+ # begin
222
+ # Process.kill "INT", pid
223
+ # rescue Exception
224
+ # Log.medium("Exception killing child #{ pid }: #{$!.message}")
225
+ # end
226
+ # end
227
+ # end
228
+
229
+ kill_children
212
230
  raise $!
213
231
  rescue Exception
214
232
  backtrace = $!.backtrace
215
233
 
216
234
  # HACK: This fixes an strange behaviour in 1.9.3 where some
217
235
  # backtrace strings are coded in ASCII-8BIT
218
- backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
219
-
236
+ kill_children
220
237
  set_info :backtrace, backtrace
221
238
  log(:error, "#{$!.class}: #{$!.message}")
239
+ backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
222
240
  raise $!
223
241
  end
224
242
 
@@ -234,6 +252,13 @@ class Step
234
252
  Log.exception $!
235
253
  end
236
254
  end
255
+ result.abort_callback = Proc.new do
256
+ begin
257
+ log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
258
+ rescue
259
+ Log.exception $!
260
+ end
261
+ end
237
262
  when TSV::Dumper
238
263
  log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
239
264
  ConcurrentStream.setup result.stream do
@@ -246,6 +271,13 @@ class Step
246
271
  Log.exception $!
247
272
  end
248
273
  end
274
+ result.stream.abort_callback = Proc.new do
275
+ begin
276
+ log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
277
+ rescue
278
+ Log.exception $!
279
+ end
280
+ end
249
281
  else
250
282
  set_info :done, (done_time = Time.now)
251
283
  set_info :time_elapsed, (time_elapsed = done_time - start_time)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.11.2
4
+ version: 5.11.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez