rbbt-util 5.20.26 → 5.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/rbbt/tsv/util.rb CHANGED
@@ -108,8 +108,13 @@ module TSV
108
108
  StringIO.new file
109
109
  end
110
110
  when (defined? Step and Step)
111
- if file.respond_to?(:base_url) and file.result
112
- file.result
111
+ if file.respond_to?(:base_url)
112
+ if file.result and IO === file.result
113
+ file.result
114
+ else
115
+ file.join
116
+ get_stream(file.path, open_options.merge(:nocache => true))
117
+ end
113
118
  else
114
119
  file.grace
115
120
  stream = file.get_stream
@@ -105,6 +105,7 @@ module Log
105
105
 
106
106
  def save
107
107
  info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
108
+ info.delete_if{|k,v| v.nil?}
108
109
  Open.write(@file, info.to_yaml)
109
110
  end
110
111
 
@@ -104,7 +104,11 @@ module ConcurrentStream
104
104
 
105
105
  def join_callback
106
106
  if @callback and not joined?
107
- @callback.call
107
+ begin
108
+ @callback.call
109
+ rescue Exception
110
+ Log.exception $!
111
+ end
108
112
  @callback = nil
109
113
  end
110
114
  end
@@ -162,7 +166,7 @@ module ConcurrentStream
162
166
 
163
167
  def abort(exception = nil)
164
168
  if @aborted
165
- Log.medium "YET aborted stream #{Misc.fingerprint self} [#{@aborted}]"
169
+ Log.medium "Already aborted stream #{Misc.fingerprint self} [#{@aborted}]"
166
170
  return
167
171
  else
168
172
  Log.medium "Aborting stream #{Misc.fingerprint self} [#{@aborted}]"
@@ -1,20 +1,20 @@
1
1
  module Misc
2
2
 
3
- def self.add_libdir(dir=nil)
3
+ def self.add_libdir(dir=nil)
4
4
  dir ||= File.join(Path.caller_lib_dir(caller.first), 'lib')
5
5
  $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
6
6
  end
7
7
 
8
8
  def self.pre_fork
9
- Persist::CONNECTIONS.values.each do |db|
10
- db.close if db.write?
9
+ Persist::CONNECTIONS.values.each do |db|
10
+ db.close if db.write?
11
11
  end
12
12
  Log::ProgressBar::BARS.clear
13
- ObjectSpace.each_object(Mutex) do |m|
14
- begin
15
- m.unlock
13
+ ObjectSpace.each_object(Mutex) do |m|
14
+ begin
15
+ m.unlock
16
16
  rescue ThreadError
17
- end if m.locked?
17
+ end if m.locked?
18
18
  end
19
19
  end
20
20
 
@@ -146,6 +146,8 @@ module Misc
146
146
  rescue TryAgain
147
147
  sleep sleep
148
148
  retry
149
+ rescue StopInsist
150
+ raise $!.exception
149
151
  rescue Aborted, Interrupt
150
152
  if msg
151
153
  Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
@@ -158,7 +160,7 @@ module Misc
158
160
  Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
159
161
  else
160
162
  Log.warn("Insisting after exception: #{$!.class} #{$!.message}")
161
- end
163
+ end
162
164
 
163
165
  if sleep and try > 0
164
166
  sleep sleep
@@ -216,14 +218,14 @@ module Misc
216
218
  else
217
219
  v, n = template[pos], template[-1]
218
220
  template.pop
219
- template[pos] = n
221
+ template[pos] = n
220
222
  end
221
223
  p << v
222
224
  end
223
225
  else
224
- size.times do
226
+ size.times do
225
227
  pos = nil
226
- while pos.nil?
228
+ while pos.nil?
227
229
  pos = (rand * total).floor
228
230
  if p.include? pos
229
231
  pos = nil
@@ -245,9 +247,11 @@ module Misc
245
247
  end
246
248
  end
247
249
 
250
+ MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
251
+
248
252
  def self.object_delta(*args)
249
253
  res, delta = nil, nil
250
- Thread.exclusive do
254
+ MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
251
255
  pre = Set.new
252
256
  delta = Set.new
253
257
 
@@ -264,7 +268,7 @@ module Misc
264
268
  end
265
269
 
266
270
  end
267
- Log.info "Delta: #{delta.inspect}"
271
+ Log.info "Delta: #{delta.inspect}"
268
272
  res
269
273
  end
270
274
 
@@ -362,7 +366,7 @@ module Misc
362
366
 
363
367
  PUSHBULLET_KEY=begin
364
368
  if ENV["PUSHBULLET_KEY"]
365
- ENV["PUSHBULLET_KEY"]
369
+ ENV["PUSHBULLET_KEY"]
366
370
  else
367
371
  config_api = File.join(ENV['HOME'], 'config/apps/pushbullet/apikey')
368
372
  if File.exist? config_api
@@ -1,12 +1,13 @@
1
- class ParameterException < Exception; end
2
- class FieldNotFoundError < Exception;end
3
- class TryAgain < Exception; end
4
- class ClosedStream < Exception; end
1
+ class RbbtException < StandardError; end
2
+ class ParameterException < RbbtException; end
3
+ class FieldNotFoundError < RbbtException;end
4
+ class TryAgain < RbbtException; end
5
+ class ClosedStream < RbbtException; end
5
6
 
6
- class ProcessFailed < Exception; end
7
- class Aborted < Exception; end
7
+ class ProcessFailed < RbbtException; end
8
+ class Aborted < RbbtException; end
8
9
 
9
- class RemoteServerError < Exception; end
10
+ class RemoteServerError < RbbtException; end
10
11
 
11
12
  class DependencyError < Aborted
12
13
  def initialize(msg)
@@ -35,3 +36,11 @@ class KeepBar < Exception
35
36
  @payload = payload
36
37
  end
37
38
  end
39
+
40
+ class StopInsist < Exception
41
+ attr_accessor :exception
42
+ def initialize(exception)
43
+ @exception = exception
44
+ end
45
+ end
46
+
@@ -182,8 +182,9 @@ module Misc
182
182
  end
183
183
  end
184
184
 
185
- if defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
186
- info = Annotated.purge(_v.info)
185
+ if _v and defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
186
+ info = _v.info
187
+ info = Annotated.purge(info)
187
188
  str << "_" << hash2md5(info)
188
189
  end
189
190
  end
@@ -23,12 +23,14 @@ module Misc
23
23
  OPEN_PIPE_IN = []
24
24
  def self.pipe
25
25
  OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
26
- PIPE_MUTEX.synchronize do
26
+ res = PIPE_MUTEX.synchronize do
27
27
  sout, sin = IO.pipe
28
28
  OPEN_PIPE_IN << sin
29
29
 
30
30
  [sout, sin]
31
31
  end
32
+ Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
33
+ res
32
34
  end
33
35
 
34
36
  def self.release_pipes(*pipes)
@@ -188,6 +190,7 @@ module Misc
188
190
  end
189
191
 
190
192
  stream.join if stream.respond_to? :join
193
+
191
194
  rescue Aborted, Interrupt
192
195
  stream.abort if stream.respond_to? :abort
193
196
  out_pipes.each do |sout|
@@ -206,14 +209,14 @@ module Misc
206
209
  end
207
210
 
208
211
  out_pipes.each do |sout|
209
- ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
212
+ ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :_pair => stream
210
213
  end
211
214
 
212
- abort_callback = Proc.new do
213
- out_pipes.each do |s|
214
- s.abort if s.respond_to? :abort
215
- end
216
- end
215
+ #abort_callback = Proc.new do
216
+ # out_pipes.each do |s|
217
+ # s.abort if s.respond_to? :abort
218
+ # end
219
+ #end
217
220
 
218
221
  out_pipes
219
222
  end
@@ -77,7 +77,7 @@ class Task
77
77
  end
78
78
 
79
79
  def info
80
- return {} if not File.exists?(info_file)
80
+ return {} if not File.exist?(info_file)
81
81
  info = YAML.load(File.open(info_file)) || {}
82
82
  info.extend IndiferentHash
83
83
  end
@@ -146,7 +146,7 @@ class Task
146
146
  end
147
147
 
148
148
  def run_dependencies
149
- required_files.each do |file| file.produce unless File.exists? file end unless required_files.nil?
149
+ required_files.each do |file| file.produce unless File.exist? file end unless required_files.nil?
150
150
  previous_jobs.each do |job|
151
151
  if not job.recursive_done?
152
152
  job.clean if job.error?
@@ -222,7 +222,7 @@ class Task
222
222
  def run
223
223
  return self if recursive_done?
224
224
  begin
225
- FileUtils.rm info_file if File.exists? info_file
225
+ FileUtils.rm info_file if File.exist? info_file
226
226
  step(:started)
227
227
  start
228
228
  step(:done)
@@ -293,9 +293,9 @@ class Task
293
293
  end
294
294
 
295
295
  def clean
296
- FileUtils.rm path if File.exists? path
297
- FileUtils.rm info_file if File.exists? info_file
298
- FileUtils.rm_rf path + '.files' if File.exists? path + '.files'
296
+ FileUtils.rm path if File.exist? path
297
+ FileUtils.rm info_file if File.exist? info_file
298
+ FileUtils.rm_rf path + '.files' if File.exist? path + '.files'
299
299
  self
300
300
  end
301
301
 
data/lib/rbbt/workflow.rb CHANGED
@@ -181,7 +181,7 @@ module Workflow
181
181
  attr_accessor :libdir, :workdir
182
182
  attr_accessor :helpers, :tasks
183
183
  attr_accessor :task_dependencies, :task_description, :last_task
184
- attr_accessor :asynchronous_exports, :synchronous_exports, :exec_exports
184
+ attr_accessor :stream_exports, :asynchronous_exports, :synchronous_exports, :exec_exports
185
185
  attr_accessor :step_cache
186
186
  attr_accessor :remote_tasks
187
187
 
@@ -237,6 +237,10 @@ module Workflow
237
237
  @task_description ||= {}
238
238
  end
239
239
 
240
+ def stream_exports
241
+ @stream_exports ||= []
242
+ end
243
+
240
244
  def asynchronous_exports
241
245
  @asynchronous_exports ||= []
242
246
  end
@@ -250,7 +254,7 @@ module Workflow
250
254
  end
251
255
 
252
256
  def all_exports
253
- @all_exports ||= asynchronous_exports + synchronous_exports + exec_exports
257
+ @all_exports ||= asynchronous_exports + synchronous_exports + exec_exports + stream_exports
254
258
  end
255
259
 
256
260
  # {{{ JOB MANAGEMENT
@@ -404,4 +408,27 @@ module Workflow
404
408
  self.workdir = saved
405
409
  end
406
410
  end
411
+
412
+ def add_remote_tasks(remote_tasks)
413
+ remote_tasks.each do |remote, tasks|
414
+ tasks.each do |task|
415
+ self.remote_tasks[task.to_f] = remote
416
+ end
417
+ end
418
+ end
419
+
420
+ def self.process_remote_tasks(remote_tasks)
421
+ require 'rbbt/rest/client'
422
+ remote_tasks.each do |workflow, info|
423
+ wf = Workflow.require_workflow workflow
424
+ wf.remote_tasks ||= {}
425
+ info.each do |remote, tasks|
426
+ remote_wf = WorkflowRESTClient.new remote, workflow
427
+ tasks.each do |task|
428
+ wf.remote_tasks[task.to_sym] = remote_wf
429
+ end
430
+ end
431
+ end
432
+ end
433
+
407
434
  end
@@ -315,6 +315,17 @@ class Step
315
315
  self._abort
316
316
  end
317
317
 
318
+ def recoverable_error?
319
+ return true if aborted?
320
+ return false unless error?
321
+ begin
322
+ klass = Kernel.const_get(info[:exception][:class])
323
+ not RbbtException === klass
324
+ rescue Exception
325
+ false
326
+ end
327
+ end
328
+
318
329
  def started?
319
330
  Open.exists?(path) or Open.exists?(pid_file)
320
331
  end
@@ -469,6 +480,8 @@ module Workflow
469
480
  :asynchronous
470
481
  when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
471
482
  :exec
483
+ when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
484
+ :stream
472
485
  else
473
486
  :none
474
487
  end
@@ -772,7 +785,7 @@ module Workflow
772
785
  end
773
786
 
774
787
  def task_exports
775
- [exec_exports, synchronous_exports, asynchronous_exports].compact.flatten.uniq
788
+ [exec_exports, synchronous_exports, asynchronous_exports, stream_exports].compact.flatten.uniq
776
789
  end
777
790
 
778
791
  end
@@ -92,23 +92,42 @@ module Workflow
92
92
  tasks[name] = task
93
93
  task_dependencies[name] = consume_dependencies
94
94
  end
95
+
96
+ def unexport(*names)
97
+ names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
98
+ names.uniq!
99
+ exec_exports.replace exec_exports - names if exec_exports
100
+ synchronous_exports.replace synchronous_exports - names if synchronous_exports
101
+ asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
102
+ stream_exports.replace stream_exports - names if stream_exports
103
+ end
95
104
 
96
105
  def export_exec(*names)
106
+ unexport *names
97
107
  exec_exports.concat names
98
108
  exec_exports.uniq!
99
109
  exec_exports
100
110
  end
101
111
 
112
+ def export_synchronous(*names)
113
+ unexport *names
114
+ synchronous_exports.concat names
115
+ synchronous_exports.uniq!
116
+ synchronous_exports
117
+ end
118
+
102
119
  def export_asynchronous(*names)
120
+ unexport *names
103
121
  asynchronous_exports.concat names
104
122
  asynchronous_exports.uniq!
105
123
  asynchronous_exports
106
124
  end
107
125
 
108
- def export_synchronous(*names)
109
- synchronous_exports.concat names
110
- synchronous_exports.uniq!
111
- synchronous_exports
126
+ def export_stream(*names)
127
+ unexport *names
128
+ stream_exports.concat names
129
+ stream_exports.uniq!
130
+ stream_exports
112
131
  end
113
132
 
114
133
  alias export export_asynchronous
@@ -19,6 +19,7 @@ class WorkflowSOAP < SimpleWS
19
19
  @workflow = workflow
20
20
  @workflow.synchronous_exports.each do |name| synchronous name end
21
21
  @workflow.asynchronous_exports.each do |name| asynchronous name end
22
+ @workflow.stream_exports.each do |name| asynchronous name end
22
23
 
23
24
  desc "Job management: Check the status of a job"
24
25
  param_desc :jobid => "Job identifier", :return => "Status code. Special status codes are: 'done' and 'error'"
@@ -114,7 +114,7 @@ class Step
114
114
 
115
115
  if not dependency.started?
116
116
  log_dependency_exec(dependency, :starting)
117
- dependency.run(:stream)
117
+ dependency.run(true)
118
118
  raise TryAgain
119
119
  end
120
120
 
@@ -154,6 +154,7 @@ class Step
154
154
  raise $!
155
155
  rescue Exception
156
156
  Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
157
+ Log.exception $!
157
158
  raise $!
158
159
  end
159
160
  end
@@ -187,7 +188,7 @@ class Step
187
188
  Misc.insist do
188
189
  begin
189
190
  step.produce
190
- rescue
191
+ rescue Exception
191
192
  step.abort
192
193
  raise $!
193
194
  end
@@ -203,12 +204,13 @@ class Step
203
204
  Misc.insist do
204
205
  begin
205
206
  dep.produce
207
+ Log.warn "Error producing #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
206
208
  rescue Aborted
207
209
  dep.abort
208
210
  raise $!
209
211
  rescue Exception
210
212
  dep.exception $!
211
- raise $!
213
+ raise StopInsist.new($!)
212
214
  end
213
215
  end
214
216
  nil