rbbt-util 3.2.1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. data/README.rdoc +65 -0
  2. data/bin/run_workflow.rb +142 -69
  3. data/lib/rbbt-util.rb +3 -3
  4. data/lib/rbbt.rb +12 -3
  5. data/lib/rbbt/annotations.rb +215 -0
  6. data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb} +17 -13
  7. data/lib/rbbt/persist.rb +164 -0
  8. data/lib/rbbt/persist/tsv.rb +135 -0
  9. data/lib/rbbt/resource.rb +100 -0
  10. data/lib/rbbt/resource/path.rb +180 -0
  11. data/lib/rbbt/resource/rake.rb +48 -0
  12. data/lib/rbbt/resource/util.rb +111 -0
  13. data/lib/rbbt/resource/with_key.rb +28 -0
  14. data/lib/rbbt/tsv.rb +134 -0
  15. data/lib/rbbt/tsv/accessor.rb +345 -0
  16. data/lib/rbbt/tsv/attach.rb +183 -0
  17. data/lib/rbbt/tsv/attach/util.rb +277 -0
  18. data/lib/rbbt/{util/tsv/filters.rb → tsv/filter.rb} +76 -37
  19. data/lib/rbbt/tsv/index.rb +453 -0
  20. data/lib/rbbt/tsv/manipulate.rb +361 -0
  21. data/lib/rbbt/tsv/parser.rb +231 -0
  22. data/lib/rbbt/tsv/serializers.rb +79 -0
  23. data/lib/rbbt/tsv/util.rb +67 -0
  24. data/lib/rbbt/util/R.rb +3 -3
  25. data/lib/rbbt/util/chain_methods.rb +64 -0
  26. data/lib/rbbt/util/cmd.rb +17 -13
  27. data/lib/rbbt/util/excel2tsv.rb +4 -3
  28. data/lib/rbbt/util/log.rb +1 -0
  29. data/lib/rbbt/util/misc.rb +296 -285
  30. data/lib/rbbt/util/open.rb +9 -2
  31. data/lib/rbbt/util/persistence.rb +1 -1
  32. data/lib/rbbt/util/task/job.rb +3 -1
  33. data/lib/rbbt/workflow.rb +193 -0
  34. data/lib/rbbt/workflow/accessor.rb +249 -0
  35. data/lib/rbbt/workflow/annotate.rb +60 -0
  36. data/lib/rbbt/workflow/soap.rb +100 -0
  37. data/lib/rbbt/workflow/step.rb +102 -0
  38. data/lib/rbbt/workflow/task.rb +76 -0
  39. data/test/rbbt/resource/test_path.rb +12 -0
  40. data/test/rbbt/test_annotations.rb +106 -0
  41. data/test/rbbt/{util/test_fix_width_table.rb → test_fix_width_table.rb} +8 -9
  42. data/test/rbbt/test_resource.rb +66 -0
  43. data/test/rbbt/test_tsv.rb +332 -0
  44. data/test/rbbt/test_workflow.rb +102 -0
  45. data/test/rbbt/tsv/test_accessor.rb +163 -0
  46. data/test/rbbt/{util/tsv → tsv}/test_attach.rb +86 -43
  47. data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb} +31 -13
  48. data/test/rbbt/tsv/test_index.rb +284 -0
  49. data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb +35 -105
  50. data/test/rbbt/util/test_R.rb +1 -1
  51. data/test/rbbt/util/test_chain_methods.rb +22 -0
  52. data/test/rbbt/util/test_filecache.rb +0 -1
  53. data/test/rbbt/util/test_misc.rb +97 -79
  54. data/test/rbbt/util/test_open.rb +1 -0
  55. data/test/rbbt/util/test_tmpfile.rb +1 -1
  56. data/test/rbbt/workflow/test_soap.rb +103 -0
  57. data/test/rbbt/workflow/test_step.rb +142 -0
  58. data/test/rbbt/workflow/test_task.rb +84 -0
  59. data/test/test_helper.rb +7 -7
  60. metadata +80 -54
  61. data/lib/rbbt/util/rake.rb +0 -176
  62. data/lib/rbbt/util/resource.rb +0 -355
  63. data/lib/rbbt/util/task.rb +0 -183
  64. data/lib/rbbt/util/tc_hash.rb +0 -324
  65. data/lib/rbbt/util/tsv.rb +0 -236
  66. data/lib/rbbt/util/tsv/accessor.rb +0 -312
  67. data/lib/rbbt/util/tsv/attach.rb +0 -416
  68. data/lib/rbbt/util/tsv/index.rb +0 -419
  69. data/lib/rbbt/util/tsv/manipulate.rb +0 -300
  70. data/lib/rbbt/util/tsv/misc.rb +0 -41
  71. data/lib/rbbt/util/tsv/parse.rb +0 -324
  72. data/lib/rbbt/util/tsv/resource.rb +0 -88
  73. data/lib/rbbt/util/workflow.rb +0 -135
  74. data/lib/rbbt/util/workflow/soap.rb +0 -116
  75. data/test/rbbt/util/test_persistence.rb +0 -201
  76. data/test/rbbt/util/test_rake.rb +0 -54
  77. data/test/rbbt/util/test_resource.rb +0 -77
  78. data/test/rbbt/util/test_task.rb +0 -133
  79. data/test/rbbt/util/test_tc_hash.rb +0 -144
  80. data/test/rbbt/util/test_tsv.rb +0 -221
  81. data/test/rbbt/util/test_workflow.rb +0 -135
  82. data/test/rbbt/util/tsv/test_accessor.rb +0 -150
  83. data/test/rbbt/util/tsv/test_index.rb +0 -241
  84. data/test/rbbt/util/tsv/test_parse.rb +0 -87
  85. data/test/rbbt/util/tsv/test_resource.rb +0 -9
@@ -1,355 +0,0 @@
1
- require 'rbbt/util/open'
2
- require 'rbbt/util/rake'
3
-
4
- module Resource
5
- class << self
6
- attr_accessor :resources, :rake_dirs
7
- end
8
-
9
- def self.extended(base)
10
- class << base
11
- attr_accessor :pkgdir, :lib_dir, :base, :offsetdir, :namespace
12
- end
13
- base.base = base
14
- base.lib_dir = caller_lib_dir
15
- base.pkgdir = base.to_s.downcase unless base.to_s == "Rbbt"
16
- end
17
-
18
- def self.caller_base_dir(file = nil)
19
- file = caller.reject{|l| l =~ /\/util\/(?:resource\.rb|progress-monitor\.rb|workflow\.rb)/ }.first.sub(/\.rb.*/,'.rb') if file.nil?
20
- File.dirname(File.expand_path(file))
21
- end
22
-
23
- def self.caller_lib_dir(file = nil)
24
- file = caller.reject{|l| l =~ /\/util\/(?:resource\.rb|progress-monitor\.rb|workflow\.rb)/ }.first.sub(/\.rb.*/,'.rb') if file.nil?
25
-
26
- file = File.expand_path file
27
- while file != '/'
28
- dir = File.dirname file
29
- return dir if File.exists? File.join(dir, 'lib')
30
- file = File.dirname file
31
- end
32
-
33
- return nil
34
- end
35
-
36
- def self.resolve(path, pkgdir, type = :find, lib_dir = nil)
37
- if path.match(/(.*?)\/(.*)/)
38
- location, subpath = path.match(/(.*?)\/(.*)/).values_at 1, 2
39
- else
40
- location, subpath = path, ""
41
- end
42
-
43
- case type.to_sym
44
- when :user
45
- pkgdir = 'rbbt' if pkgdir.nil? or pkgdir.empty?
46
- File.join(ENV['HOME'], '.' + pkgdir, location, subpath)
47
- when :local
48
- File.join('/usr/local', location, pkgdir, subpath)
49
- when :global
50
- File.join('/', location, pkgdir, subpath)
51
- when :lib
52
- if not caller_lib_dir.nil? and not caller_lib_dir == "/"
53
- path = File.join(caller_lib_dir, location, subpath)
54
- return path if File.exists?(path) or lib_dir.nil?
55
- end
56
- raise "Root of library not found" if lib_dir.nil?
57
- File.join(lib_dir, location, subpath)
58
- when :find
59
- %w(user local global lib).each do |_type|
60
- file = resolve(path, pkgdir, _type.to_sym, lib_dir)
61
- return file if File.exists? file
62
- end
63
-
64
- resolve(path, pkgdir, :user)
65
- end
66
- end
67
-
68
- def self.define_rake(path, rakefile)
69
- @rake_dirs ||= {}
70
- @rake_dirs[path.find] = rakefile
71
- end
72
-
73
- def self.define_resource(path, type, content)
74
- @resources ||= {}
75
- @resources[path.find] = [type, content]
76
- end
77
-
78
- def self.set_software_env(software_dir)
79
- bin_dir = File.join(software_dir, 'bin')
80
- opt_dir = File.join(software_dir, 'opt')
81
-
82
- Misc.env_add 'PATH', bin_dir
83
-
84
- FileUtils.mkdir_p opt_dir unless File.exists? opt_dir
85
- %w(.ld-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
86
- filename = File.join(opt_dir, file)
87
- FileUtils.touch filename unless File.exists? filename
88
- end
89
-
90
- if not File.exists? File.join(opt_dir,'.post_install')
91
- Open.write(File.join(opt_dir,'.post_install'),"#!/bin/bash\n")
92
- end
93
-
94
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
95
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
96
- Misc.env_add('LD_RUN_PATH',line.chomp)
97
- end
98
-
99
- Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
100
- Misc.env_add('PKG_CONFIG_PATH',line.chomp)
101
- end
102
-
103
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
104
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
105
- end
106
-
107
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
108
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
109
- end
110
-
111
- Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
112
- Misc.env_add('ACLOCAL_FLAGS', "-I#{File.join(opt_dir, line.chomp)}", ' ')
113
- end
114
-
115
- Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
116
- Misc.env_add('CLASSPATH', "#{File.join(opt_dir,'java', 'lib', line.chomp)}")
117
- end
118
-
119
- Dir.glob(File.join opt_dir, 'jars', '*').each do |file|
120
- Misc.env_add('CLASSPATH', "#{File.expand_path(file)}")
121
- end
122
-
123
- File.chmod 0774, File.join(opt_dir, '.post_install')
124
-
125
- CMD.cmd(File.join(opt_dir, '.post_install'))
126
- end
127
-
128
- def self.produce(resource)
129
- resource = resource.find if Path === resource
130
- return resource if File.exists? resource
131
-
132
- @resources ||= {}
133
- @rake_dirs ||= {}
134
- case
135
- when @resources.include?(resource)
136
- type, content = @resources[resource]
137
-
138
- case type
139
- when :string
140
- Open.write(resource, content)
141
- when :url
142
- Open.write(resource, Open.read(content))
143
- when :proc
144
- Open.write(resource, content.call)
145
- when :install
146
- software_dir = File.dirname(File.dirname(resource.to_s))
147
- preamble = <<-EOF
148
- #!/bin/bash
149
-
150
- RBBT_SOFTWARE_DIR="#{software_dir}"
151
-
152
- INSTALL_HELPER_FILE="#{Rbbt.share.install.software.lib.install_helpers.find :lib, caller_lib_dir(__FILE__)}"
153
- source "$INSTALL_HELPER_FILE"
154
- EOF
155
-
156
- CMD.cmd('bash', :in => preamble + "\n" + content.read)
157
- set_software_env(software_dir)
158
- else
159
- raise "Could not produce #{ resource }. (#{ type }, #{ content })"
160
- end
161
- resource
162
-
163
- when @rake_dirs.select{|dir,rakefile| resource.in_dir?(dir)}.any?
164
- dir, rakefile = @rake_dirs.select{|dir,rakefile| resource.in_dir?(dir)}.first
165
- file = resource.sub(dir, '').sub(/^\//,'')
166
- rakefile = rakefile.find if Resource::Path === rakefile
167
- dir = dir.find if Resource::Path === dir
168
- FileUtils.mkdir_p dir unless File.exists? dir
169
- RakeHelper.run(rakefile, file, dir)
170
- resource
171
- end
172
- resource
173
- end
174
-
175
- def relative_to(klass, path)
176
- self.offsetdir = path
177
- if Rbbt == klass
178
- self.pkgdir = ""
179
- else
180
- self.pkgdir = klass.to_s.downcase
181
- end
182
- end
183
-
184
- def data_module(klass)
185
- relative_to klass, "share/#{self.to_s.downcase}" unless klass == base
186
- rakefile = klass.share.install[self.to_s].Rakefile
187
- rakefile.lib_dir = Resource.caller_lib_dir
188
-
189
-
190
- self[''].define_as_rake rakefile
191
- self.namespace = base.to_s
192
- self.lib_dir = Resource.caller_lib_dir
193
- end
194
-
195
- module Path
196
- attr_accessor :pkgdir, :namespace, :lib_dir
197
-
198
- def self.path(name = nil, pkgdir = nil, namespace = nil, lib_dir = nil)
199
- name = name.nil? ? "" : name.to_s
200
- name.extend Path
201
- name.pkgdir = pkgdir
202
- name.namespace = namespace
203
- name.lib_dir = lib_dir
204
- name
205
- end
206
-
207
- def find(type = :find, lib_dir = nil)
208
- lib_dir ||= @lib_dir
209
- return self if pkgdir.nil?
210
- path = Path.path(Resource.resolve(self, pkgdir, type, lib_dir), nil)
211
- path.namespace = namespace
212
- path
213
- end
214
-
215
- def produce
216
- Resource.produce self.find
217
- end
218
-
219
- def dirname
220
- Path.path(File.dirname(self), pkgdir, namespace, lib_dir)
221
- end
222
-
223
- def join(name)
224
- Path.path(File.join(self, name.to_s), pkgdir, namespace, lib_dir)
225
- end
226
-
227
- def [](name)
228
- join name
229
- end
230
-
231
- alias :old_method_missing :method_missing
232
- def method_missing(name, prev = nil, *args)
233
- old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
234
- join prev unless prev.nil?
235
- join name
236
- end
237
-
238
- def open(*args)
239
- Resource.produce self.find
240
- Open.open self.find, *args
241
- end
242
-
243
- def read(*args)
244
- Resource.produce self.find
245
- Open.read self.find, *args
246
- end
247
-
248
- def yaml(*args)
249
- YAML.load(open)
250
- end
251
-
252
- def marshal(*args)
253
- Marshal.load(open)
254
- end
255
-
256
- def write(content, *args)
257
- FileUtils.mkdir_p File.dirname(self.find) unless File.exists? self.find
258
- Open.write(self.find, content, *args)
259
- end
260
-
261
- def define_as_string(content)
262
- Resource.define_resource(self, :string, content)
263
- end
264
-
265
- def define_as_url(url)
266
- Resource.define_resource(self, :url, url)
267
- end
268
-
269
- def define_as_rake(rakefile)
270
- Resource.define_rake(self, rakefile)
271
- end
272
-
273
- def define_as_proc(&block)
274
- Resource.define_resource(self, :proc, block)
275
- end
276
-
277
-
278
- def define_as_install(install_file)
279
- Resource.define_resource(self, :install, install_file.find)
280
- self.produce
281
- software_dir = File.dirname(File.dirname(self.to_s))
282
- Resource.set_software_env(software_dir)
283
- end
284
-
285
- def in_dir?(dir)
286
- ! ! File.expand_path(self).match(/^#{Regexp.quote dir}/)
287
- end
288
-
289
- def to_s
290
- self.find
291
- end
292
-
293
- def filename
294
- self.find
295
- end
296
-
297
- def exists?
298
- begin
299
- self.produce
300
- File.exists? self.find
301
- rescue
302
- false
303
- end
304
- end
305
- end
306
-
307
- module WithKey
308
- def self.extended(base)
309
- class << base
310
- attr_accessor :klass, :key
311
- end
312
- end
313
-
314
- alias :old_method_missing :method_missing
315
- def method_missing(name, *args)
316
- return old_method_missing(name, *args) if name.to_s =~ /^to_/
317
- if key
318
- klass.send(name, key, *args)
319
- else
320
- klass.send(name, *args)
321
- end
322
- end
323
- end
324
-
325
- def with_key(key)
326
- klass = self
327
- o = Object.new
328
- o.extend WithKey
329
- o.klass = self
330
- o.key = key
331
- o
332
- end
333
-
334
- def [](name)
335
- if pkgdir.nil?
336
- @pkgdir = (base == Rbbt ? '' : base.to_s.downcase)
337
- end
338
- name = File.join(offsetdir.to_s, name.to_s) unless offsetdir.nil? or offsetdir.empty?
339
- Path.path(name, pkgdir, namespace, lib_dir)
340
- end
341
-
342
- alias :old_method_missing :method_missing
343
- def method_missing(name, prev = nil, *args)
344
- return old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
345
- if prev
346
- self[prev][name]
347
- else
348
- self[name]
349
- end
350
- end
351
- end
352
-
353
- def resource_path(path)
354
- Resource::Path.path(path)
355
- end
@@ -1,183 +0,0 @@
1
- require 'rbbt/util/open'
2
- require 'rbbt/util/task/job'
3
-
4
- class Task
5
- class << self
6
- attr_accessor :basedir
7
- end
8
-
9
- @basedir = "."
10
-
11
- def load(job_id)
12
- Job.load(self, job_id)
13
- end
14
-
15
- def job_id(name, job_options, previous_jobs)
16
- job_options = job_options.merge :previous_jobs => previous_jobs.collect{|job| job.id} if previous_jobs.any?
17
- if job_options.any?
18
- name.to_s + "_" + Misc.hash2md5(job_options)
19
- else
20
- name.to_s
21
- end
22
- end
23
-
24
- attr_accessor :name, :persistence, :options, :option_descriptions, :option_types, :option_defaults, :workflow, :dependencies, :scope, :description, :block
25
- def initialize(name, persistence = nil, options = nil, option_descriptions = nil, option_types = nil, option_defaults = nil, workflow = nil, dependencies = nil, scope = nil, description = nil, &block)
26
- dependencies = [dependencies] unless dependencies.nil? or Array === dependencies
27
- @name = name.to_s
28
-
29
- @persistence = persistence || :marshal
30
-
31
- @options = Array === options ? options : [options] unless options.nil?
32
-
33
- @option_defaults = option_defaults
34
- @option_descriptions = option_descriptions
35
- @option_types = option_types
36
- @workflow = workflow
37
- @dependencies = dependencies || []
38
- @scope = scope
39
- @description = description
40
-
41
- @block = block unless not block_given?
42
- end
43
-
44
- def process_options(args, optional_args)
45
- run_options = {}
46
-
47
- options.each do |option|
48
- if option_defaults and option_defaults.include? option
49
- run_options[option] = Misc.process_options(optional_args, option) || option_defaults[option]
50
- else
51
- run_options[option] = args.shift
52
- end
53
- end unless options.nil?
54
-
55
- [run_options, args, optional_args]
56
- end
57
-
58
- def pull_from_hash(args, optional_args)
59
- option_summary.first.each do |info|
60
- name = info[:name]
61
- if optional_args.include? name
62
- args.push optional_args.delete name
63
- end
64
- end
65
- end
66
-
67
- def setup(jobname, args, optional_args, dependencies)
68
- previous_jobs = []
69
- required_files = []
70
-
71
- pull_from_hash(args, optional_args)
72
- run_options, args, optional_args = process_options args, optional_args
73
-
74
- dependencies.each do |dependency|
75
- case
76
- when Proc === dependency
77
- deps = dependency.call(jobname, run_options)
78
- if Array === deps
79
- previous_jobs.concat deps
80
- else
81
- previous_jobs << deps
82
- end
83
- when Task::Job === dependency
84
- previous_jobs << dependency
85
- when Task === dependency
86
- previous_jobs << dependency.job(jobname, *(args + [optional_args]))
87
- else
88
- required_files << dependency
89
- end
90
- end
91
-
92
- [previous_jobs, required_files, run_options]
93
- end
94
-
95
- def job(jobname, *args)
96
- if Hash === args.last
97
- optional_args = args.pop
98
- else
99
- optional_args = {}
100
- end
101
-
102
-
103
- previous_jobs, required_files, run_options = setup(jobname, args, optional_args, dependencies)
104
-
105
- job_id = self.job_id jobname, run_options, previous_jobs
106
-
107
- Job.new(self, job_id, jobname, run_options, previous_jobs, required_files, previous_jobs.first)
108
- end
109
-
110
- def run(*args)
111
- job(*args).start
112
- end
113
-
114
- def option_info(option)
115
- info = {}
116
- info[:name] = option
117
- info[:source] = name
118
- info[:description] = option_descriptions[option] if option_descriptions and option_descriptions.include? option
119
- info[:type] = option_types[option] if option_types and option_types.include? option
120
- info[:default] = option_defaults[option] if option_defaults and option_defaults.include? option
121
-
122
- info
123
- end
124
-
125
- def option_summary
126
- options = []
127
- optional_options = []
128
-
129
- if self.options
130
- self.options.collect{|option|
131
- info = option_info(option)
132
- if info[:default].nil?
133
- options << info
134
- else
135
- optional_options << info
136
- end
137
- }
138
- end
139
-
140
- dependencies.select{|dep| Task === dep}.each do |task|
141
- more_options, more_optional_options = task.option_summary
142
- options.concat more_options
143
- optional_options.concat more_optional_options
144
- end
145
-
146
- [options, optional_options]
147
- end
148
-
149
- def usage
150
- usage = ""
151
- usage << "Task: #{name}\n"
152
- usage << "\nDescription: #{description.chomp}\n" if description
153
- options, optional_options = option_summary
154
-
155
- if options.any?
156
- usage << "\nMandatory options:\n"
157
- usage << "\tTask\tName\tType \tDescription\n"
158
- usage << "\t----\t----\t---- \t-----------\n"
159
-
160
- options.each do |option|
161
- option_line = "\t[#{option[:source]}]\t#{option[:name]}"
162
- option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
163
- option_line << "\t#{option[:description]}" if option[:description]
164
- usage << option_line << "\n"
165
- end
166
- end
167
-
168
- if optional_options.any?
169
- usage << "\nOptional options:\n"
170
- usage << "\tTask\tName\tDefault \tType \tDescription\n"
171
- usage << "\t----\t----\t------- \t---- \t-----------\n"
172
- optional_options.each do |option|
173
- option_line = "\t[#{option[:source]}]\t#{option[:name]}\t#{option[:default]}"
174
- option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
175
- option_line << "\t#{option[:description]}" if option[:description]
176
- usage << option_line << "\n"
177
- end
178
- end
179
-
180
- usage
181
- end
182
-
183
- end