rbbt-util 4.3.0 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/rbbt/workflow.rb CHANGED
@@ -9,43 +9,6 @@ module Workflow
9
9
  end
10
10
  self.workflows = []
11
11
 
12
- def self.require_local_workflow2(wf_name, wf_dir = nil)
13
- require 'rbbt/resource/path'
14
-
15
- if File.exists?(wf_name) or File.exists?(wf_name + '.rb')
16
- $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
17
- require wf_name
18
- return
19
- end
20
-
21
- wf_dir ||= case
22
- when File.exists?(File.join(File.dirname(Path.caller_lib_dir), wf_name))
23
- dir = File.join(File.dirname(Path.caller_lib_dir), wf_name)
24
- Log.debug "Loading workflow from lib dir: #{dir}"
25
- dir
26
- File.join(File.dirname(Path.caller_lib_dir), wf_name)
27
- when defined? Rbbt
28
- if Rbbt.etc.workflow_dir.exists?
29
- dir = File.join(Rbbt.etc.workflow_dir.read.strip, wf_name)
30
- Log.debug "Loading workflow from etc dir: #{dir}"
31
- dir
32
- else
33
- dir = Rbbt.workflows[wf_name]
34
- Log.debug "Loading workflow from main dir: #{dir}"
35
- dir
36
- end
37
- else
38
- dir = File.join(ENV["HOME"], '.workflows')
39
- Log.debug "Loading workflow from home dir: #{dir}"
40
- dir
41
- end
42
-
43
- wf_dir = Path.setup(wf_dir)
44
-
45
- $LOAD_PATH.unshift(File.join(File.dirname(wf_dir["workflow.rb"].find), 'lib'))
46
- require wf_dir["workflow.rb"].find
47
- end
48
-
49
12
  def self.require_remote_workflow(wf_name, url)
50
13
  require 'rbbt/workflow/rest/client'
51
14
  eval "Object::#{wf_name} = RbbtRestClient.new '#{ url }', '#{wf_name}'"
@@ -75,7 +38,6 @@ module Workflow
75
38
 
76
39
  else
77
40
  case
78
-
79
41
  # Points to workflow file
80
42
  when ((File.exists?(wf_name) and not File.directory?(wf_name)) or File.exists?(wf_name + '.rb'))
81
43
  $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(wf_name)), 'lib'))
@@ -204,7 +166,7 @@ module Workflow
204
166
  block = self.method(name) unless block_given?
205
167
 
206
168
  result_type = result_type
207
- task = Task.setup({
169
+ annotations = {
208
170
  :name => name,
209
171
  :inputs => inputs,
210
172
  :description => description,
@@ -212,8 +174,11 @@ module Workflow
212
174
  :result_type => Array == result_type ? result_type.to_sym : result_type,
213
175
  :input_defaults => input_defaults,
214
176
  :input_descriptions => input_descriptions,
177
+ :input_options => input_options,
215
178
  :result_description => result_description
216
- }, &block)
179
+ }
180
+
181
+ task = Task.setup(annotations, &block)
217
182
 
218
183
  @last_task = task
219
184
  @tasks[name] = task
@@ -77,6 +77,10 @@ class Step
77
77
  info[:status] == :error
78
78
  end
79
79
 
80
+ def aborted?
81
+ info[:status] == :aborted
82
+ end
83
+
80
84
  # {{{ INFO
81
85
 
82
86
  def files_dir
@@ -168,6 +172,7 @@ module Workflow
168
172
  input_types = rec_input_types(name)
169
173
  input_descriptions = rec_input_descriptions(name)
170
174
  input_defaults = rec_input_defaults(name)
175
+ input_options = rec_input_options(name)
171
176
  export = case
172
177
  when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
173
178
  :synchronous
@@ -188,6 +193,7 @@ module Workflow
188
193
  :input_types => input_types,
189
194
  :input_descriptions => input_descriptions,
190
195
  :input_defaults => input_defaults,
196
+ :input_options => input_options,
191
197
  :result_type => result_type,
192
198
  :result_description => result_description,
193
199
  :dependencies => dependencies
@@ -220,6 +226,11 @@ module Workflow
220
226
  [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_descriptions}
221
227
  end
222
228
 
229
+ def rec_input_options(taskname)
230
+ [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn| acc.merge tasks[tn].input_options}
231
+ end
232
+
233
+
223
234
  def real_dependencies(task, jobname, inputs, dependencies)
224
235
  real_dependencies = []
225
236
  dependencies.each do |dependency|
@@ -2,7 +2,7 @@ module AnnotatedModule
2
2
  def self.extended(base)
3
3
  if not base.respond_to? :inputs
4
4
  class << base
5
- attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :result_description, :helpers
5
+ attr_accessor :description, :inputs, :input_types, :input_descriptions, :input_defaults, :input_options, :result_description, :helpers
6
6
 
7
7
  def description
8
8
  i = @description; @description = ""; i
@@ -24,6 +24,10 @@ module AnnotatedModule
24
24
  i = @input_defaults; @input_defaults = {}; i
25
25
  end
26
26
 
27
+ def input_options
28
+ i = @input_options; @input_options = {}; i
29
+ end
30
+
27
31
  def description
28
32
  i = @description; @description = ""; i
29
33
  end
@@ -38,6 +42,7 @@ module AnnotatedModule
38
42
  base.input_types = {}
39
43
  base.input_descriptions = {}
40
44
  base.input_defaults = {}
45
+ base.input_options = {}
41
46
  base.helpers = {}
42
47
 
43
48
  end
@@ -60,13 +65,14 @@ module AnnotatedModule
60
65
  @dependencies.concat dependencies
61
66
  end
62
67
 
63
- def input(name, type = nil, desc = nil, default = nil)
68
+ def input(name, type = nil, desc = nil, default = nil, options = nil)
64
69
  name = name.to_sym
65
70
  type = type.to_sym
66
71
  @inputs << name
67
72
  @input_types[name] = type unless type.nil?
68
73
  @input_descriptions[name] = desc unless desc.nil?
69
74
  @input_defaults[name] = default unless default.nil?
75
+ @input_options[name] = options unless options.nil?
70
76
  end
71
77
  end
72
78
 
@@ -42,7 +42,11 @@ class Step
42
42
  end
43
43
  else
44
44
  Log.debug "Waiting for pid: #{@pid}"
45
- Process.waitpid @pid
45
+ begin
46
+ Process.waitpid @pid
47
+ rescue Errno::ECHILD
48
+ Log.debug "Process #{ @pid } already finished: #{ path }"
49
+ end
46
50
  @pid = nil
47
51
  end
48
52
  self
@@ -51,21 +55,31 @@ class Step
51
55
  def run(no_load = false)
52
56
  result = Persist.persist "Job", @task.result_type, :file => @path, :check => rec_dependencies.collect{|dependency| dependency.path}.uniq, :no_load => no_load do
53
57
  FileUtils.rm info_file if File.exists? info_file
54
- log((task.name || "unnamed task"), "Starting task")
58
+ log(:starting, "Starting task: #{task.name || "unnamed task"}")
59
+
55
60
  set_info :dependencies, @dependencies.collect{|dep| [dep.task.name, dep.name]}
56
61
  @dependencies.each{|dependency|
57
62
  log dependency.task.name || "dependency", "Processing dependency: #{ dependency.path }"
58
63
  dependency.run true
59
64
  }
65
+
60
66
  set_info :status, :started
67
+
68
+ set_info :started, Time.now
69
+
61
70
  set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
71
+
62
72
  res = begin
63
73
  exec
74
+ rescue Step::Aborted
75
+ raise $!
64
76
  rescue Exception
65
77
  set_info :backtrace, $!.backtrace
66
78
  log(:error, "#{$!.class}: #{$!.message}")
79
+ log(:error, "backtrace: #{$!.backtrace.first}")
67
80
  raise $!
68
81
  end
82
+
69
83
  set_info :status, :done
70
84
  res
71
85
  end
@@ -85,14 +99,19 @@ class Step
85
99
  end
86
100
  end
87
101
  set_info :pid, @pid
102
+ Process.detach(@pid)
88
103
  self
89
104
  end
90
105
 
91
106
  def abort
107
+ @pid ||= info[:pid]
92
108
  if @pid.nil?
109
+ Log.medium "Could not abort #{path}: no pid"
93
110
  false
94
111
  else
112
+ Log.medium "Aborting #{path}: #{ @pid }"
95
113
  Process.kill("INT", @pid)
114
+ log(:aborted, "Job aborted by user")
96
115
  true
97
116
  end
98
117
  end
@@ -2,15 +2,15 @@ require 'rbbt/util/misc'
2
2
  require 'rbbt/persist'
3
3
 
4
4
  module Task
5
- attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :description, :name, :result_description
5
+ attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :description, :name, :result_description
6
6
 
7
7
  def self.setup(options = {}, &block)
8
8
  block.extend Task
9
9
  options = IndiferentHash.setup options
10
10
  block.singleton_methods.
11
11
  select{|method| method.to_s[-1] != "="[0]}.each{|method|
12
- if block.respond_to?(method + "=") and options.include? method.to_sym
13
- block.send(method + '=', options[method.to_sym])
12
+ if block.respond_to?(method.to_s + "=") and options.include? method.to_sym
13
+ block.send(method.to_s + '=', options[method.to_sym])
14
14
  end
15
15
  }
16
16
  block
data/share/lib/R/util.R CHANGED
@@ -1,28 +1,50 @@
1
- rbbt.ruby <- function(code, load = TRUE, flat = FALSE){
1
+ rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
2
2
  file = system('rbbt_exec.rb - file', input = code, intern=TRUE);
3
+
4
+ error_str = "^#:rbbt_exec Error"
5
+ if (regexpr(error_str, file)[1] != -1 ){
6
+ print(file);
7
+ return(NULL);
8
+ }
9
+
3
10
  if (load){
4
- if(flat){
5
- data = rbbt.flat.tsv(file);
6
- }else{
7
- data = rbbt.tsv(file);
8
- }
9
- rm(file);
10
- return(data);
11
+ if(type == 'tsv'){
12
+ if(flat){
13
+ data = rbbt.flat.tsv(file, ...);
14
+ }else{
15
+ data = rbbt.tsv(file, ...);
16
+ }
17
+ rm(file);
18
+ return(data)
19
+ }
20
+
21
+ if(type == 'list'){
22
+ data = scan(file, ...)
23
+ return(data);
24
+ }
25
+
26
+ return(NULL);
11
27
  }else{
12
28
  return(file);
13
29
  }
14
30
  }
15
31
 
32
+ rbbt.ruby.substitutions <- function(script, substitutions = list(), ...){
33
+
34
+ for (keyword in names(substitutions)){
35
+ script = sub(keyword, substitutions[[keyword]], script);
36
+ }
37
+
38
+ result = rbbt.ruby(script, ...);
39
+
40
+ return(result);
41
+ }
42
+
16
43
  rbbt.glob <- function(d, pattern){
17
44
  d=sub("/$", '', d);
18
45
  sapply(dir(d, pattern), function(file){paste(d,file,sep="/")});
19
46
  }
20
47
 
21
- rbbt.png_plot <- function(filename, width, height, p){
22
- png(filename="temp.png", width=width, height=height);
23
- eval(p);
24
- dev.off();
25
- }
26
48
 
27
49
  rbbt.load.data <- function(filename, sep = "\t", ...){
28
50
  data=read.table(file=filename, sep=sep, fill=TRUE, as.is=TRUE, ...);
@@ -47,8 +69,8 @@ rbbt.flat.tsv <- function(filename, sep = "\t", comment.char ="#", ...){
47
69
  return(result);
48
70
  }
49
71
 
50
- rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1, ...){
51
- data=read.table(file=filename, sep=sep, fill=TRUE, as.is=TRUE, quote='', row.names= row.names, comment.char = comment.char, ...);
72
+ rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1, check.names=FALSE, fill=TRUE, as.is=TRUE, quote='', ...){
73
+ data=read.table(file=filename, sep=sep, fill=fill, as.is=as.is, quote=quote, row.names= row.names, comment.char = comment.char, ...);
52
74
  f = file(filename, 'r');
53
75
  headers = readLines(f, 1);
54
76
  if (length(grep("^#: ", headers)) > 0){
@@ -100,6 +122,10 @@ rbbt.percent <- function(values){
100
122
  values=values/sum(values);
101
123
  }
102
124
 
125
+ rbbt.a.to.string <- function(a){
126
+ paste("'",paste(a, collapse="', '"), "'", sep="");
127
+ }
128
+
103
129
  rbbt.split <- function(string){
104
130
  return(unlist(strsplit(string, "\\|")));
105
131
  }
@@ -134,6 +160,12 @@ rbbt.acc <- function(data, new){
134
160
  }
135
161
  }
136
162
 
163
+ rbbt.png_plot <- function(filename, width, height, p, ...){
164
+ png(filename=filename, width=width, height=height, ...);
165
+ eval(parse(text=p));
166
+ dev.off();
167
+ }
168
+
137
169
  rbbt.init <- function(data, new){
138
170
  if (is.null(data)){
139
171
  return(new);
@@ -166,6 +198,7 @@ rbbt.run <- function(filename){
166
198
  }
167
199
 
168
200
 
201
+
169
202
  # UTILITIES
170
203
 
171
204
  # Addapted from http://www.phaget4.org/R/image_matrix.html
@@ -21,4 +21,19 @@ class TestTSV < Test::Unit::TestCase
21
21
  path = Path.setup "/tmp"
22
22
  assert_equal "/tmp/bar/foo", path.foo("bar")
23
23
  end
24
+
25
+ def test_doc_file
26
+ path = Path.setup "lib/rbbt/resource.rb"
27
+ assert_equal File.join('doc', path), path.doc_file
28
+ assert_equal Path.setup(File.join('doc', path)).find(:lib), path.find(:lib).doc_file
29
+
30
+ assert_equal "lib/rbbt/resource.rb", path.doc_file.source_for_doc_file
31
+ assert_equal path.find, path.doc_file.find(:lib).source_for_doc_file
32
+ assert_equal path.find, path.doc_file.source_for_doc_file.find
33
+
34
+ assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
35
+ assert_equal "lib/rbbt/resource.rb", path.doc_file.set_extension('doc').source_for_doc_file
36
+
37
+ assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
38
+ end
24
39
  end
@@ -58,6 +58,16 @@ class TestAnnotations < Test::Unit::TestCase
58
58
  assert_equal annotation_str, str.annotation_str
59
59
  end
60
60
 
61
+ def test_json
62
+ str1 = "string1"
63
+ annotation_str1 = "Annotation String 1"
64
+ str2 = "string2"
65
+ annotation_str2 = "Annotation String 2"
66
+ AnnotatedString.setup(str1, annotation_str1)
67
+ AnnotatedString.setup(str2, annotation_str2)
68
+ ddd Annotated.json(str1, true)
69
+ end
70
+
61
71
  def test_tsv
62
72
  str1 = "string1"
63
73
  annotation_str1 = "Annotation String 1"
@@ -98,7 +108,7 @@ class TestAnnotations < Test::Unit::TestCase
98
108
 
99
109
  assert_equal annotation_str, Annotated.load_tsv(Annotated.tsv(a, :all)).annotation_str
100
110
 
101
- assert_equal str1, Annotated.load_tsv(Annotated.tsv(a, :literal, :JSON)).sort.first
111
+ assert_equal str2, Annotated.load_tsv(Annotated.tsv(a, :literal, :JSON)).sort.last
102
112
  end
103
113
 
104
114
  def test_inheritance
@@ -13,8 +13,6 @@ class TestTSV < Test::Unit::TestCase
13
13
 
14
14
  a.extend TSV
15
15
 
16
- assert a.methods.include? "key_field="
17
-
18
16
  a.key_field = "Number"
19
17
 
20
18
  assert_equal "1", a["one"]
@@ -314,6 +312,21 @@ b 2
314
312
  end
315
313
  end
316
314
 
315
+ def test_grep_header
316
+ content =<<-EOF
317
+ #: :sep=/\\s+/#:type=:single#:namespace=Test
318
+ #Id Value
319
+ a 1
320
+ b 2
321
+ EOF
322
+
323
+ TmpFile.with_file(content) do |filename|
324
+ tsv = TSV.open(filename, :key_field => "Value", :grep => "2")
325
+ assert(! tsv.include?("1"))
326
+ assert(tsv.include?("2"))
327
+ end
328
+ end
329
+
317
330
  def test_json
318
331
  content =<<-EOF
319
332
  #: :sep=/\\s+/#:type=:single
@@ -13,6 +13,7 @@ class TestTSVManipulate < Test::Unit::TestCase
13
13
  values["Range"].last
14
14
  end
15
15
 
16
+ puts tsv.type
16
17
  tsv = tsv.slice ["Start", "End"]
17
18
 
18
19
  tsv
@@ -45,12 +46,12 @@ row3 A a|B Id4
45
46
  TmpFile.with_file(content) do |filename|
46
47
  tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID")
47
48
  index = tsv.index(:order => true)
48
- assert_equal "Id1", index['a'].first
49
- assert_equal "Id3", index['A'].first
49
+ assert_equal "Id1", index['a']
50
+ assert_equal "Id3", index['A']
50
51
  assert_equal "OtherID", index.fields.first
51
52
 
52
53
  index = tsv.index(:order => false)
53
- assert_equal "Id1", index['a'].first
54
+ assert_equal "Id1", index['a']
54
55
  end
55
56
 
56
57
  TmpFile.with_file(content) do |filename|
@@ -73,8 +74,8 @@ row3 A a|B Id4
73
74
  TmpFile.with_file(content) do |filename|
74
75
  tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
75
76
  index = tsv.index(:order => true)
76
- assert_equal "Id1", index['a'].first
77
- assert_equal "Id3", index['A'].first
77
+ assert_equal "Id1", index['a']
78
+ assert_equal "Id3", index['A']
78
79
  assert_equal "OtherID", index.fields.first
79
80
  end
80
81
  end
@@ -91,19 +92,19 @@ row3 A a|B Id4
91
92
  tsv = TSV.open(Open.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => true)
92
93
 
93
94
  index = tsv.index(:order => true, :persist => true)
94
- assert_equal "Id1", index['a'].first
95
- assert_equal "Id3", index['A'].first
95
+ assert_equal "Id1", index['a']
96
+ assert_equal "Id3", index['A']
96
97
  assert_equal "OtherID", index.fields.first
97
98
 
98
99
  tsv.delete "Id1"
99
100
 
100
101
  index = tsv.index(:order => true, :persist => true)
101
- assert_equal "Id1", index['a'].first
102
- assert_equal "Id3", index['A'].first
102
+ assert_equal "Id1", index['a']
103
+ assert_equal "Id3", index['A']
103
104
  assert_equal "OtherID", index.fields.first
104
105
 
105
106
  index = tsv.index(:order => true, :persist => false)
106
- assert_equal "Id1", index['a'].first
107
+ assert_equal "Id1", index['a']
107
108
  end
108
109
  end
109
110
 
@@ -117,8 +118,9 @@ row3 A a|B Id4
117
118
 
118
119
  TmpFile.with_file(content) do |filename|
119
120
  index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
120
- assert_equal "Id1", index['a'].first
121
- assert_equal "Id3", index['A'].first
121
+ ddd index
122
+ assert_equal "Id1", index['a']
123
+ assert_equal "Id3", index['A']
122
124
  assert_equal "OtherID", index.fields.first
123
125
  end
124
126
  end
@@ -133,20 +135,20 @@ row3 A a|B Id4
133
135
 
134
136
  TmpFile.with_file(content) do |filename|
135
137
  index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => false)
136
- assert_equal "Id1", index['a'].first
137
- assert_equal "Id3", index['A'].first
138
+ assert_equal "Id1", index['a']
139
+ assert_equal "Id3", index['A']
138
140
  assert_equal "OtherID", index.fields.first
139
141
 
140
142
  index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
141
- assert_equal "Id1", index['a'].first
142
- assert_equal "Id3", index['A'].first
143
+ assert_equal "Id1", index['a']
144
+ assert_equal "Id3", index['A']
143
145
  assert_equal "OtherID", index.fields.first
144
146
 
145
147
  Open.write(filename, Open.read(filename).sub(/row1.*Id1\n/,''))
146
148
 
147
149
  index = TSV.index(filename, :target => "OtherID", :data_sep => /\s+/, :order => true, :persist => true)
148
- assert_equal "Id1", index['a'].first
149
- assert_equal "Id3", index['A'].first
150
+ assert_equal "Id1", index['a']
151
+ assert_equal "Id3", index['A']
150
152
  assert_equal "OtherID", index.fields.first
151
153
  assert index.include?('aaa')
152
154
 
@@ -224,15 +226,16 @@ f: ___
224
226
  g: ____
225
227
  EOF
226
228
  TmpFile.with_file(data) do |datafile|
229
+ load_segment_data(datafile)
227
230
  TmpFile.with_file(load_segment_data(datafile)) do |tsvfile|
228
231
  f = TSV.range_index(tsvfile, "Start", "End", :persistence => true)
229
232
 
230
- assert_equal %w(), f[0].sort
231
- assert_equal %w(b), f[1].sort
232
- assert_equal %w(), f[20].sort
233
- assert_equal %w(), f[(20..100)].sort
234
- assert_equal %w(a b d), f[3].sort
235
- assert_equal %w(a b c d e), f[(3..4)].sort
233
+ #assert_equal %w(), f[0].sort
234
+ #assert_equal %w(b), f[1].sort
235
+ #assert_equal %w(), f[20].sort
236
+ #assert_equal %w(), f[(20..100)].sort
237
+ #assert_equal %w(a b d), f[3].sort
238
+ #assert_equal %w(a b c d e), f[(3..4)].sort
236
239
  end
237
240
  end
238
241
  end