rbbt-util 4.4.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -109,22 +109,22 @@ module Open
109
109
 
110
110
  # Grep
111
111
 
112
- def self.grep(stream, grep)
112
+ def self.grep(stream, grep, invert = false)
113
113
  case
114
114
  when Array === grep
115
115
  TmpFile.with_file(grep * "\n", false) do |f|
116
- CMD.cmd("grep", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
116
+ CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
117
117
  end
118
118
  else
119
- CMD.cmd("grep '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
119
+ CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
120
120
  end
121
121
  end
122
122
 
123
- def self.file_open(file, grep)
123
+ def self.file_open(file, grep, mode = 'r', invert_grep = false)
124
124
  if grep
125
- grep(File.open(file), grep)
125
+ grep(File.open(file, mode), grep, invert_grep)
126
126
  else
127
- File.open(file)
127
+ File.open(file, mode)
128
128
  end
129
129
  end
130
130
 
@@ -161,8 +161,28 @@ module Open
161
161
 
162
162
  # Open Read Write
163
163
 
164
+ def self.clean_cache(url, options = {})
165
+ options = Misc.add_defaults options, :noz => false, :mode => 'r'
166
+
167
+ wget_options = options[:wget_options] || {}
168
+ wget_options[:nice] = options.delete(:nice)
169
+ wget_options[:nice_key] = options.delete(:nice_key)
170
+ wget_options[:quiet] = options.delete(:quiet)
171
+ wget_options["--post-data="] = options.delete(:post) if options.include? :post
172
+ wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
173
+ wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
174
+ wget_options[:cookies] = options.delete(:cookies)
175
+
176
+ cache_file = in_cache(url, wget_options)
177
+ Misc.lock(cache_file) do
178
+ FileUtils.rm(cache_file)
179
+ end if cache_file
180
+ end
181
+
164
182
  def self.open(url, options = {})
165
- options = Misc.add_defaults options, :noz => false
183
+ options = Misc.add_defaults options, :noz => false, :mode => 'r'
184
+
185
+ mode = Misc.process_options options, :mode
166
186
 
167
187
  wget_options = options[:wget_options] || {}
168
188
  wget_options[:nice] = options.delete(:nice)
@@ -177,19 +197,19 @@ module Open
177
197
  when (IO === url or StringIO === url)
178
198
  url
179
199
  when (not remote?(url))
180
- file_open(url, options[:grep])
200
+ file_open(url, options[:grep], mode, options[:invert_grep])
181
201
  when (options[:nocache] and options[:nocache] != :update)
182
202
  # What about grep?
183
203
  wget(url, wget_options)
184
204
  when (options[:nocache] != :update and in_cache(url, wget_options))
185
205
  Misc.lock(in_cache(url, wget_options)) do
186
- file_open(in_cache(url, wget_options), options[:grep])
206
+ file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
187
207
  end
188
208
  else
189
209
  io = wget(url, wget_options)
190
210
  add_cache(url, io, wget_options)
191
211
  io.close
192
- file_open(in_cache(url, wget_options), options[:grep])
212
+ file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
193
213
  end
194
214
  io = unzip(io) if ((String === url and zip?(url)) and not options[:noz]) or options[:zip]
195
215
  io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
@@ -231,12 +251,16 @@ module Open
231
251
  end
232
252
  end
233
253
 
234
- def self.write(file, content = nil)
254
+ def self.write(file, content = nil, options = {})
255
+ options = Misc.add_defaults options, :mode => 'w'
256
+
257
+ mode = Misc.process_options options, :mode
258
+
235
259
  FileUtils.mkdir_p File.dirname(file)
236
260
  case
237
261
  when content.nil?
238
262
  begin
239
- File.open(file, 'w') do |f|
263
+ File.open(file, mode) do |f|
240
264
  yield f
241
265
  end
242
266
  rescue Exception
@@ -244,14 +268,14 @@ module Open
244
268
  raise $!
245
269
  end
246
270
  when String === content
247
- File.open(file, 'w') do |f|
271
+ File.open(file, mode) do |f|
248
272
  f.flock(File::LOCK_EX)
249
273
  f.write content
250
274
  f.flock(File::LOCK_UN)
251
275
  end
252
276
  else
253
277
  begin
254
- File.open(file, 'w') do |f|
278
+ File.open(file, mode) do |f|
255
279
  f.flock(File::LOCK_EX)
256
280
  while not content.eof?
257
281
  f.write content.gets
@@ -53,7 +53,7 @@ module SimpleDSL
53
53
 
54
54
  @config[@@method_name] = File.open(actions).read
55
55
 
56
- eval File.open(actions).read
56
+ instance_eval Open.read(actions), actions
57
57
  end
58
58
 
59
59
  unhook_method
data/lib/rbbt/workflow.rb CHANGED
@@ -94,10 +94,11 @@ module Workflow
94
94
  require_local_workflow(wf_name)
95
95
  rescue Exception
96
96
  Log.debug $!.message
97
- raise "Workflow not found: #{ wf_name }" if wf_name == wf_name.downcase
98
- Log.debug "Trying with downcase: '#{wf_name.downcase}'"
97
+ Log.debug $!.backtrace.first
98
+ raise "Workflow not found: #{ wf_name }" if wf_name == Misc.humanize(wf_name)
99
+ Log.debug "Trying with humanized: '#{Misc.humanize wf_name}'"
99
100
  begin
100
- require_local_workflow(wf_name.downcase)
101
+ require_local_workflow(Misc.humanize(wf_name))
101
102
  rescue Exception
102
103
  Log.debug $!.message
103
104
  raise "Workflow not found: #{ wf_name }"
@@ -213,7 +214,7 @@ module Workflow
213
214
  end
214
215
 
215
216
  def job(taskname, jobname = nil, inputs = {})
216
- jobname ||= "Default"
217
+ jobname = "Default" if jobname.nil? or jobname.empty?
217
218
  task = tasks[taskname]
218
219
  raise "Task not found: #{ taskname }" if task.nil?
219
220
 
@@ -23,8 +23,8 @@ class Step
23
23
  File.open(info_file) do |file|
24
24
  YAML.load(file) || {}
25
25
  end
26
- rescue
27
- Log.debug "Error loading yaml: " + Open.read(info_file)
26
+ rescue Exception
27
+ Log.debug "Error loading yaml: " + info_file
28
28
  raise $!
29
29
  end
30
30
  end
@@ -33,7 +33,7 @@ class Step
33
33
  Misc.lock(info_file) do
34
34
  i = info
35
35
  i[key] = value
36
- Open.write(info_file, YAML.dump(i))
36
+ Open.write(info_file, i.to_yaml)
37
37
  value
38
38
  end
39
39
  end
@@ -54,12 +54,14 @@ class Step
54
54
  set_info(:messages, (messages || []) << message)
55
55
  end
56
56
 
57
- def log(status, message = nil)
57
+ def log(status, message = nil, do_log = true)
58
+
58
59
  if message
59
60
  Log.low "[#{ status }] #{ message }: #{path}"
60
61
  else
61
62
  Log.low "[#{ status }]: #{path}"
62
- end
63
+ end if do_log
64
+
63
65
  self.status = status
64
66
  message(message) unless message.nil?
65
67
  end
@@ -24,6 +24,26 @@ class Step
24
24
  @inputs = inputs || []
25
25
  end
26
26
 
27
+ class << self
28
+ attr_accessor :log_relay_step
29
+ end
30
+
31
+ def relay_log(step)
32
+ return self unless Task === self.task and not self.task.name.nil?
33
+ if not self.respond_to? :original_log
34
+ class << self
35
+ attr_accessor :relay_step
36
+ alias original_log log
37
+ def log(status, message = nil, do_log = true)
38
+ original_log(status, message, do_log)
39
+ relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : [task.name.to_s, message] * ">", false)
40
+ end
41
+ end
42
+ end
43
+ @relay_step = step
44
+ self
45
+ end
46
+
27
47
  def prepare_result(value, description = nil, info = {})
28
48
  return value if description.nil?
29
49
  Entity.formats[description].setup(value, info.merge(:format => description)) if defined?(Entity) and Entity.respond_to?(:formats) and Entity.formats.include? description
@@ -54,16 +74,28 @@ class Step
54
74
 
55
75
  def run(no_load = false)
56
76
  result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
77
+ if Step === Step.log_relay_step and not self == Step.log_relay_step
78
+ relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
79
+ end
80
+
57
81
  FileUtils.rm info_file if File.exists? info_file
58
- log(:starting, "Starting task: #{task.name || "unnamed task"}")
59
82
 
60
83
  set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
61
84
  @dependencies.each{|dependency|
62
- log dependency.task.name || "dependency", "Processing dependency: #{ dependency.path }"
63
- dependency.run true
85
+ begin
86
+ dependency.relay_log self
87
+ dependency.run true
88
+ rescue Exception
89
+ backtrace = $!.backtrace
90
+ set_info :backtrace, backtrace
91
+ log(:error, "Exception processing dependency #{dependency.path}")
92
+ log(:error, "#{$!.class}: #{$!.message}")
93
+ log(:error, "backtrace: #{$!.backtrace.first}")
94
+ raise "Exception processing dependency #{dependency.path}"
95
+ end
64
96
  }
65
-
66
- set_info :status, :started
97
+
98
+ log(:started, "Starting task #{task.name || ""}")
67
99
 
68
100
  set_info :started, Time.now
69
101
 
@@ -72,9 +104,16 @@ class Step
72
104
  res = begin
73
105
  exec
74
106
  rescue Step::Aborted
107
+ log(:error, "Aborted")
75
108
  raise $!
76
109
  rescue Exception
77
- set_info :backtrace, $!.backtrace
110
+ backtrace = $!.backtrace
111
+
112
+ # HACK: This fixes an strange behaviour in 1.9.3 where some
113
+ # bactrace strings are coded in ASCII-8BIT
114
+ backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
115
+
116
+ set_info :backtrace, backtrace
78
117
  log(:error, "#{$!.class}: #{$!.message}")
79
118
  log(:error, "backtrace: #{$!.backtrace.first}")
80
119
  raise $!
@@ -84,7 +123,11 @@ class Step
84
123
  res
85
124
  end
86
125
 
87
- prepare_result result, @task.result_description, info
126
+ if no_load
127
+ self
128
+ else
129
+ prepare_result result, @task.result_description, info
130
+ end
88
131
  end
89
132
 
90
133
  def fork
@@ -42,10 +42,25 @@ class TestAnnotations < Test::Unit::TestCase
42
42
  assert_equal annotation_str, ary[0].annotation_str
43
43
  end
44
44
 
45
+ def test_double_array
46
+ ary = ["string"]
47
+ annotation_str = "Annotation String"
48
+ ary.extend AnnotatedArray
49
+ ary_ary = [ary]
50
+ ary_ary.extend AnnotatedArray
51
+ AnnotatedString.setup(ary, annotation_str)
52
+ AnnotatedString.setup(ary_ary, annotation_str)
53
+ assert_equal [AnnotatedString], ary.annotation_types
54
+ assert_equal annotation_str, ary.annotation_str
55
+ assert_equal annotation_str, ary[0].annotation_str
56
+ end
57
+
58
+
45
59
  def test_info
46
60
  ary = ["string"]
47
61
  annotation_str = "Annotation String"
48
62
  AnnotatedString.setup(ary, annotation_str)
63
+
49
64
  assert_equal({:annotation_str => annotation_str, :annotation_types => [AnnotatedString]}, ary.info)
50
65
  end
51
66
 
@@ -65,7 +80,6 @@ class TestAnnotations < Test::Unit::TestCase
65
80
  annotation_str2 = "Annotation String 2"
66
81
  AnnotatedString.setup(str1, annotation_str1)
67
82
  AnnotatedString.setup(str2, annotation_str2)
68
- ddd Annotated.json(str1, true)
69
83
  end
70
84
 
71
85
  def test_tsv
@@ -75,8 +89,8 @@ class TestAnnotations < Test::Unit::TestCase
75
89
  annotation_str2 = "Annotation String 2"
76
90
  AnnotatedString.setup(str1, annotation_str1)
77
91
  AnnotatedString.setup(str2, annotation_str2)
78
- assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id]["literal"]
79
- assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id]["annotation_str"]
92
+ assert_equal str1, Annotated.tsv([str1, str2], :all)[str1.id + ":0"]["literal"]
93
+ assert_equal annotation_str1, Annotated.tsv([str1, str2], :annotation_str, :JSON)[str1.id + ":0"]["annotation_str"]
80
94
  end
81
95
 
82
96
  def test_literal
@@ -128,10 +142,22 @@ class TestAnnotations < Test::Unit::TestCase
128
142
  assert_equal str + annotation_str, str.add_annot
129
143
  end
130
144
 
145
+ def test_double_array
146
+ a = ["a"]
147
+ b = AnnotatedString.setup([AnnotatedString.setup(["a"])])
148
+ AnnotatedString.setup(a)
149
+ a.extend AnnotatedArray
150
+ b.extend AnnotatedArray
151
+ assert AnnotatedString === b[0]
152
+ assert(!a.double_array)
153
+ assert(b.double_array)
154
+ end
155
+
131
156
  def test_annotation_positional2hash
132
157
  str = "string"
133
158
  annotation_str = "Annotation String"
134
159
  AnnotatedString.setup(str, :annotation_str => annotation_str)
135
160
  assert_equal str + annotation_str, str.add_annot
136
161
  end
162
+
137
163
  end
@@ -41,6 +41,43 @@ class TestPersist < Test::Unit::TestCase
41
41
  end
42
42
  end
43
43
 
44
+ def test_annotation_persist_with_repeitions
45
+ TmpFile.with_file do |tmp|
46
+ entity1 = "Entity 1"
47
+ entity2 = "Entity 2"
48
+ entity2bis = "Entity 2"
49
+
50
+ TestAnnotation.setup(entity1, :test_annotation => "1")
51
+ TestAnnotation.setup(entity2, :test_annotation => "2")
52
+ TestAnnotation.setup(entity2bis, :test_annotation => "2")
53
+
54
+ annotations = [entity1, entity2, entity2bis]
55
+
56
+ persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
57
+ annotations
58
+ end
59
+
60
+ assert_equal 3, persisted_annotations.length
61
+
62
+ assert_equal "Entity 1", persisted_annotations.first
63
+ assert_equal "Entity 2", persisted_annotations.last
64
+ assert_equal "1", persisted_annotations.first.test_annotation
65
+ assert_equal "2", persisted_annotations.last.test_annotation
66
+
67
+ persisted_annotations = Persist.persist("Test", :annotations, :file => tmp) do
68
+ annotations
69
+ end
70
+
71
+ assert_equal 3, persisted_annotations.length
72
+
73
+ assert_equal "Entity 1", persisted_annotations.sort.first
74
+ assert_equal "Entity 2", persisted_annotations.sort.last
75
+ assert_equal "1", persisted_annotations.sort.first.test_annotation
76
+ assert_equal "2", persisted_annotations.sort.last.test_annotation
77
+ end
78
+ end
79
+
80
+
44
81
  def test_bdb
45
82
  TmpFile.with_file do |tmp|
46
83
  repo = Persist.open_tokyocabinet(tmp, true, :double, TokyoCabinet::BDB)
@@ -85,6 +122,119 @@ class TestPersist < Test::Unit::TestCase
85
122
  end
86
123
  end
87
124
 
125
+ def test_annotation_persist_repo_annotated_array
126
+ TmpFile.with_file do |tmp|
127
+ repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
128
+
129
+ entity1 = "Entity 1"
130
+ entity2 = "Entity 2"
131
+
132
+ annotations = [entity1, entity2]
133
+ TestAnnotation.setup(annotations, :test_annotation => "1")
134
+ annotations.extend AnnotatedArray
135
+
136
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
137
+ annotations
138
+ end
139
+
140
+ assert_equal "Entity 1", persisted_annotations.first
141
+ assert_equal "Entity 2", persisted_annotations.last
142
+ assert_equal "1", persisted_annotations.first.test_annotation
143
+ assert_equal "1", persisted_annotations.last.test_annotation
144
+
145
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
146
+ annotations
147
+ end
148
+
149
+ persisted_annotations.extend AnnotatedArray
150
+
151
+ assert_equal "Entity 1", persisted_annotations.sort.first
152
+ assert_equal "Entity 2", persisted_annotations.sort.last
153
+ assert_equal "1", persisted_annotations.sort.first.test_annotation
154
+ assert_equal "1", persisted_annotations.sort.last.test_annotation
155
+ end
156
+ end
157
+
158
+
159
+ def test_annotation_persist_repo_triple_array
160
+ TmpFile.with_file do |tmp|
161
+ repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
162
+
163
+ entity1 = "Entity 1"
164
+ entity2 = "Entity 2"
165
+
166
+ annotations = [entity1, entity2]
167
+ TestAnnotation.setup(annotations, :test_annotation => "1")
168
+ annotations.extend AnnotatedArray
169
+
170
+ annotations_ary = [annotations]
171
+ TestAnnotation.setup(annotations_ary, :test_annotation => "1")
172
+ annotations_ary.extend AnnotatedArray
173
+
174
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
175
+ annotations_ary
176
+ end
177
+ assert AnnotatedArray === persisted_annotations
178
+ ddd persisted_annotations.info
179
+ ddd persisted_annotations
180
+
181
+ assert_equal "Entity 1", persisted_annotations.first.first
182
+ assert_equal "Entity 2", persisted_annotations.first.last
183
+ assert_equal "1", persisted_annotations.first.first.test_annotation
184
+ assert_equal "1", persisted_annotations.first.last.test_annotation
185
+
186
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
187
+ annotations_ary
188
+ end
189
+
190
+ assert AnnotatedArray === persisted_annotations
191
+
192
+ assert_equal "Entity 1", persisted_annotations.sort.first.first
193
+ assert_equal "Entity 2", persisted_annotations.sort.first.last
194
+ assert_equal "1", persisted_annotations.sort.first.first.test_annotation
195
+ assert_equal "1", persisted_annotations.sort.first.last.test_annotation
196
+
197
+ end
198
+ end
199
+
200
+ def test_annotation_persist_repo_with_repetitions
201
+ TmpFile.with_file do |tmp|
202
+ repo = Persist.open_tokyocabinet(tmp, true, :list, TokyoCabinet::BDB)
203
+
204
+ entity1 = "Entity 1"
205
+ entity2 = "Entity 2"
206
+ entity2bis = "Entity 2"
207
+
208
+ TestAnnotation.setup(entity1, :test_annotation => "1")
209
+ TestAnnotation.setup(entity2, :test_annotation => "2")
210
+ TestAnnotation.setup(entity2bis, :test_annotation => "2")
211
+
212
+ annotations = [entity1, entity2, entity2bis]
213
+
214
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
215
+ annotations
216
+ end
217
+
218
+ assert_equal 3, persisted_annotations.length
219
+
220
+ assert_equal "Entity 1", persisted_annotations.first
221
+ assert_equal "Entity 2", persisted_annotations.last
222
+ assert_equal "1", persisted_annotations.first.test_annotation
223
+ assert_equal "2", persisted_annotations.last.test_annotation
224
+
225
+ persisted_annotations = Persist.persist("Test", :annotations, :annotation_repo => repo) do
226
+ annotations
227
+ end
228
+
229
+ assert_equal 3, persisted_annotations.length
230
+
231
+ assert_equal "Entity 1", persisted_annotations.sort.first
232
+ assert_equal "Entity 2", persisted_annotations.sort.last
233
+ assert_equal "1", persisted_annotations.sort.first.test_annotation
234
+ assert_equal "2", persisted_annotations.sort.last.test_annotation
235
+ end
236
+ end
237
+
88
238
 
89
239
  def test_array_persist
90
240
  TmpFile.with_file do |tmp|