rbbt-util 3.2.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/README.rdoc +65 -0
  2. data/bin/run_workflow.rb +142 -69
  3. data/lib/rbbt-util.rb +3 -3
  4. data/lib/rbbt.rb +12 -3
  5. data/lib/rbbt/annotations.rb +215 -0
  6. data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb} +17 -13
  7. data/lib/rbbt/persist.rb +164 -0
  8. data/lib/rbbt/persist/tsv.rb +135 -0
  9. data/lib/rbbt/resource.rb +100 -0
  10. data/lib/rbbt/resource/path.rb +180 -0
  11. data/lib/rbbt/resource/rake.rb +48 -0
  12. data/lib/rbbt/resource/util.rb +111 -0
  13. data/lib/rbbt/resource/with_key.rb +28 -0
  14. data/lib/rbbt/tsv.rb +134 -0
  15. data/lib/rbbt/tsv/accessor.rb +345 -0
  16. data/lib/rbbt/tsv/attach.rb +183 -0
  17. data/lib/rbbt/tsv/attach/util.rb +277 -0
  18. data/lib/rbbt/{util/tsv/filters.rb → tsv/filter.rb} +76 -37
  19. data/lib/rbbt/tsv/index.rb +453 -0
  20. data/lib/rbbt/tsv/manipulate.rb +361 -0
  21. data/lib/rbbt/tsv/parser.rb +231 -0
  22. data/lib/rbbt/tsv/serializers.rb +79 -0
  23. data/lib/rbbt/tsv/util.rb +67 -0
  24. data/lib/rbbt/util/R.rb +3 -3
  25. data/lib/rbbt/util/chain_methods.rb +64 -0
  26. data/lib/rbbt/util/cmd.rb +17 -13
  27. data/lib/rbbt/util/excel2tsv.rb +4 -3
  28. data/lib/rbbt/util/log.rb +1 -0
  29. data/lib/rbbt/util/misc.rb +296 -285
  30. data/lib/rbbt/util/open.rb +9 -2
  31. data/lib/rbbt/util/persistence.rb +1 -1
  32. data/lib/rbbt/util/task/job.rb +3 -1
  33. data/lib/rbbt/workflow.rb +193 -0
  34. data/lib/rbbt/workflow/accessor.rb +249 -0
  35. data/lib/rbbt/workflow/annotate.rb +60 -0
  36. data/lib/rbbt/workflow/soap.rb +100 -0
  37. data/lib/rbbt/workflow/step.rb +102 -0
  38. data/lib/rbbt/workflow/task.rb +76 -0
  39. data/test/rbbt/resource/test_path.rb +12 -0
  40. data/test/rbbt/test_annotations.rb +106 -0
  41. data/test/rbbt/{util/test_fix_width_table.rb → test_fix_width_table.rb} +8 -9
  42. data/test/rbbt/test_resource.rb +66 -0
  43. data/test/rbbt/test_tsv.rb +332 -0
  44. data/test/rbbt/test_workflow.rb +102 -0
  45. data/test/rbbt/tsv/test_accessor.rb +163 -0
  46. data/test/rbbt/{util/tsv → tsv}/test_attach.rb +86 -43
  47. data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb} +31 -13
  48. data/test/rbbt/tsv/test_index.rb +284 -0
  49. data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb +35 -105
  50. data/test/rbbt/util/test_R.rb +1 -1
  51. data/test/rbbt/util/test_chain_methods.rb +22 -0
  52. data/test/rbbt/util/test_filecache.rb +0 -1
  53. data/test/rbbt/util/test_misc.rb +97 -79
  54. data/test/rbbt/util/test_open.rb +1 -0
  55. data/test/rbbt/util/test_tmpfile.rb +1 -1
  56. data/test/rbbt/workflow/test_soap.rb +103 -0
  57. data/test/rbbt/workflow/test_step.rb +142 -0
  58. data/test/rbbt/workflow/test_task.rb +84 -0
  59. data/test/test_helper.rb +7 -7
  60. metadata +80 -54
  61. data/lib/rbbt/util/rake.rb +0 -176
  62. data/lib/rbbt/util/resource.rb +0 -355
  63. data/lib/rbbt/util/task.rb +0 -183
  64. data/lib/rbbt/util/tc_hash.rb +0 -324
  65. data/lib/rbbt/util/tsv.rb +0 -236
  66. data/lib/rbbt/util/tsv/accessor.rb +0 -312
  67. data/lib/rbbt/util/tsv/attach.rb +0 -416
  68. data/lib/rbbt/util/tsv/index.rb +0 -419
  69. data/lib/rbbt/util/tsv/manipulate.rb +0 -300
  70. data/lib/rbbt/util/tsv/misc.rb +0 -41
  71. data/lib/rbbt/util/tsv/parse.rb +0 -324
  72. data/lib/rbbt/util/tsv/resource.rb +0 -88
  73. data/lib/rbbt/util/workflow.rb +0 -135
  74. data/lib/rbbt/util/workflow/soap.rb +0 -116
  75. data/test/rbbt/util/test_persistence.rb +0 -201
  76. data/test/rbbt/util/test_rake.rb +0 -54
  77. data/test/rbbt/util/test_resource.rb +0 -77
  78. data/test/rbbt/util/test_task.rb +0 -133
  79. data/test/rbbt/util/test_tc_hash.rb +0 -144
  80. data/test/rbbt/util/test_tsv.rb +0 -221
  81. data/test/rbbt/util/test_workflow.rb +0 -135
  82. data/test/rbbt/util/tsv/test_accessor.rb +0 -150
  83. data/test/rbbt/util/tsv/test_index.rb +0 -241
  84. data/test/rbbt/util/tsv/test_parse.rb +0 -87
  85. data/test/rbbt/util/tsv/test_resource.rb +0 -9
@@ -1,355 +0,0 @@
1
- require 'rbbt/util/open'
2
- require 'rbbt/util/rake'
3
-
4
- module Resource
5
- class << self
6
- attr_accessor :resources, :rake_dirs
7
- end
8
-
9
- def self.extended(base)
10
- class << base
11
- attr_accessor :pkgdir, :lib_dir, :base, :offsetdir, :namespace
12
- end
13
- base.base = base
14
- base.lib_dir = caller_lib_dir
15
- base.pkgdir = base.to_s.downcase unless base.to_s == "Rbbt"
16
- end
17
-
18
- def self.caller_base_dir(file = nil)
19
- file = caller.reject{|l| l =~ /\/util\/(?:resource\.rb|progress-monitor\.rb|workflow\.rb)/ }.first.sub(/\.rb.*/,'.rb') if file.nil?
20
- File.dirname(File.expand_path(file))
21
- end
22
-
23
- def self.caller_lib_dir(file = nil)
24
- file = caller.reject{|l| l =~ /\/util\/(?:resource\.rb|progress-monitor\.rb|workflow\.rb)/ }.first.sub(/\.rb.*/,'.rb') if file.nil?
25
-
26
- file = File.expand_path file
27
- while file != '/'
28
- dir = File.dirname file
29
- return dir if File.exists? File.join(dir, 'lib')
30
- file = File.dirname file
31
- end
32
-
33
- return nil
34
- end
35
-
36
- def self.resolve(path, pkgdir, type = :find, lib_dir = nil)
37
- if path.match(/(.*?)\/(.*)/)
38
- location, subpath = path.match(/(.*?)\/(.*)/).values_at 1, 2
39
- else
40
- location, subpath = path, ""
41
- end
42
-
43
- case type.to_sym
44
- when :user
45
- pkgdir = 'rbbt' if pkgdir.nil? or pkgdir.empty?
46
- File.join(ENV['HOME'], '.' + pkgdir, location, subpath)
47
- when :local
48
- File.join('/usr/local', location, pkgdir, subpath)
49
- when :global
50
- File.join('/', location, pkgdir, subpath)
51
- when :lib
52
- if not caller_lib_dir.nil? and not caller_lib_dir == "/"
53
- path = File.join(caller_lib_dir, location, subpath)
54
- return path if File.exists?(path) or lib_dir.nil?
55
- end
56
- raise "Root of library not found" if lib_dir.nil?
57
- File.join(lib_dir, location, subpath)
58
- when :find
59
- %w(user local global lib).each do |_type|
60
- file = resolve(path, pkgdir, _type.to_sym, lib_dir)
61
- return file if File.exists? file
62
- end
63
-
64
- resolve(path, pkgdir, :user)
65
- end
66
- end
67
-
68
- def self.define_rake(path, rakefile)
69
- @rake_dirs ||= {}
70
- @rake_dirs[path.find] = rakefile
71
- end
72
-
73
- def self.define_resource(path, type, content)
74
- @resources ||= {}
75
- @resources[path.find] = [type, content]
76
- end
77
-
78
- def self.set_software_env(software_dir)
79
- bin_dir = File.join(software_dir, 'bin')
80
- opt_dir = File.join(software_dir, 'opt')
81
-
82
- Misc.env_add 'PATH', bin_dir
83
-
84
- FileUtils.mkdir_p opt_dir unless File.exists? opt_dir
85
- %w(.ld-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
86
- filename = File.join(opt_dir, file)
87
- FileUtils.touch filename unless File.exists? filename
88
- end
89
-
90
- if not File.exists? File.join(opt_dir,'.post_install')
91
- Open.write(File.join(opt_dir,'.post_install'),"#!/bin/bash\n")
92
- end
93
-
94
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
95
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
96
- Misc.env_add('LD_RUN_PATH',line.chomp)
97
- end
98
-
99
- Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
100
- Misc.env_add('PKG_CONFIG_PATH',line.chomp)
101
- end
102
-
103
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
104
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
105
- end
106
-
107
- Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
108
- Misc.env_add('LD_LIBRARY_PATH',line.chomp)
109
- end
110
-
111
- Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
112
- Misc.env_add('ACLOCAL_FLAGS', "-I#{File.join(opt_dir, line.chomp)}", ' ')
113
- end
114
-
115
- Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
116
- Misc.env_add('CLASSPATH', "#{File.join(opt_dir,'java', 'lib', line.chomp)}")
117
- end
118
-
119
- Dir.glob(File.join opt_dir, 'jars', '*').each do |file|
120
- Misc.env_add('CLASSPATH', "#{File.expand_path(file)}")
121
- end
122
-
123
- File.chmod 0774, File.join(opt_dir, '.post_install')
124
-
125
- CMD.cmd(File.join(opt_dir, '.post_install'))
126
- end
127
-
128
- def self.produce(resource)
129
- resource = resource.find if Path === resource
130
- return resource if File.exists? resource
131
-
132
- @resources ||= {}
133
- @rake_dirs ||= {}
134
- case
135
- when @resources.include?(resource)
136
- type, content = @resources[resource]
137
-
138
- case type
139
- when :string
140
- Open.write(resource, content)
141
- when :url
142
- Open.write(resource, Open.read(content))
143
- when :proc
144
- Open.write(resource, content.call)
145
- when :install
146
- software_dir = File.dirname(File.dirname(resource.to_s))
147
- preamble = <<-EOF
148
- #!/bin/bash
149
-
150
- RBBT_SOFTWARE_DIR="#{software_dir}"
151
-
152
- INSTALL_HELPER_FILE="#{Rbbt.share.install.software.lib.install_helpers.find :lib, caller_lib_dir(__FILE__)}"
153
- source "$INSTALL_HELPER_FILE"
154
- EOF
155
-
156
- CMD.cmd('bash', :in => preamble + "\n" + content.read)
157
- set_software_env(software_dir)
158
- else
159
- raise "Could not produce #{ resource }. (#{ type }, #{ content })"
160
- end
161
- resource
162
-
163
- when @rake_dirs.select{|dir,rakefile| resource.in_dir?(dir)}.any?
164
- dir, rakefile = @rake_dirs.select{|dir,rakefile| resource.in_dir?(dir)}.first
165
- file = resource.sub(dir, '').sub(/^\//,'')
166
- rakefile = rakefile.find if Resource::Path === rakefile
167
- dir = dir.find if Resource::Path === dir
168
- FileUtils.mkdir_p dir unless File.exists? dir
169
- RakeHelper.run(rakefile, file, dir)
170
- resource
171
- end
172
- resource
173
- end
174
-
175
- def relative_to(klass, path)
176
- self.offsetdir = path
177
- if Rbbt == klass
178
- self.pkgdir = ""
179
- else
180
- self.pkgdir = klass.to_s.downcase
181
- end
182
- end
183
-
184
- def data_module(klass)
185
- relative_to klass, "share/#{self.to_s.downcase}" unless klass == base
186
- rakefile = klass.share.install[self.to_s].Rakefile
187
- rakefile.lib_dir = Resource.caller_lib_dir
188
-
189
-
190
- self[''].define_as_rake rakefile
191
- self.namespace = base.to_s
192
- self.lib_dir = Resource.caller_lib_dir
193
- end
194
-
195
- module Path
196
- attr_accessor :pkgdir, :namespace, :lib_dir
197
-
198
- def self.path(name = nil, pkgdir = nil, namespace = nil, lib_dir = nil)
199
- name = name.nil? ? "" : name.to_s
200
- name.extend Path
201
- name.pkgdir = pkgdir
202
- name.namespace = namespace
203
- name.lib_dir = lib_dir
204
- name
205
- end
206
-
207
- def find(type = :find, lib_dir = nil)
208
- lib_dir ||= @lib_dir
209
- return self if pkgdir.nil?
210
- path = Path.path(Resource.resolve(self, pkgdir, type, lib_dir), nil)
211
- path.namespace = namespace
212
- path
213
- end
214
-
215
- def produce
216
- Resource.produce self.find
217
- end
218
-
219
- def dirname
220
- Path.path(File.dirname(self), pkgdir, namespace, lib_dir)
221
- end
222
-
223
- def join(name)
224
- Path.path(File.join(self, name.to_s), pkgdir, namespace, lib_dir)
225
- end
226
-
227
- def [](name)
228
- join name
229
- end
230
-
231
- alias :old_method_missing :method_missing
232
- def method_missing(name, prev = nil, *args)
233
- old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
234
- join prev unless prev.nil?
235
- join name
236
- end
237
-
238
- def open(*args)
239
- Resource.produce self.find
240
- Open.open self.find, *args
241
- end
242
-
243
- def read(*args)
244
- Resource.produce self.find
245
- Open.read self.find, *args
246
- end
247
-
248
- def yaml(*args)
249
- YAML.load(open)
250
- end
251
-
252
- def marshal(*args)
253
- Marshal.load(open)
254
- end
255
-
256
- def write(content, *args)
257
- FileUtils.mkdir_p File.dirname(self.find) unless File.exists? self.find
258
- Open.write(self.find, content, *args)
259
- end
260
-
261
- def define_as_string(content)
262
- Resource.define_resource(self, :string, content)
263
- end
264
-
265
- def define_as_url(url)
266
- Resource.define_resource(self, :url, url)
267
- end
268
-
269
- def define_as_rake(rakefile)
270
- Resource.define_rake(self, rakefile)
271
- end
272
-
273
- def define_as_proc(&block)
274
- Resource.define_resource(self, :proc, block)
275
- end
276
-
277
-
278
- def define_as_install(install_file)
279
- Resource.define_resource(self, :install, install_file.find)
280
- self.produce
281
- software_dir = File.dirname(File.dirname(self.to_s))
282
- Resource.set_software_env(software_dir)
283
- end
284
-
285
- def in_dir?(dir)
286
- ! ! File.expand_path(self).match(/^#{Regexp.quote dir}/)
287
- end
288
-
289
- def to_s
290
- self.find
291
- end
292
-
293
- def filename
294
- self.find
295
- end
296
-
297
- def exists?
298
- begin
299
- self.produce
300
- File.exists? self.find
301
- rescue
302
- false
303
- end
304
- end
305
- end
306
-
307
- module WithKey
308
- def self.extended(base)
309
- class << base
310
- attr_accessor :klass, :key
311
- end
312
- end
313
-
314
- alias :old_method_missing :method_missing
315
- def method_missing(name, *args)
316
- return old_method_missing(name, *args) if name.to_s =~ /^to_/
317
- if key
318
- klass.send(name, key, *args)
319
- else
320
- klass.send(name, *args)
321
- end
322
- end
323
- end
324
-
325
- def with_key(key)
326
- klass = self
327
- o = Object.new
328
- o.extend WithKey
329
- o.klass = self
330
- o.key = key
331
- o
332
- end
333
-
334
- def [](name)
335
- if pkgdir.nil?
336
- @pkgdir = (base == Rbbt ? '' : base.to_s.downcase)
337
- end
338
- name = File.join(offsetdir.to_s, name.to_s) unless offsetdir.nil? or offsetdir.empty?
339
- Path.path(name, pkgdir, namespace, lib_dir)
340
- end
341
-
342
- alias :old_method_missing :method_missing
343
- def method_missing(name, prev = nil, *args)
344
- return old_method_missing(name, prev, *args) if name.to_s =~ /^to_/
345
- if prev
346
- self[prev][name]
347
- else
348
- self[name]
349
- end
350
- end
351
- end
352
-
353
- def resource_path(path)
354
- Resource::Path.path(path)
355
- end
@@ -1,183 +0,0 @@
1
- require 'rbbt/util/open'
2
- require 'rbbt/util/task/job'
3
-
4
- class Task
5
- class << self
6
- attr_accessor :basedir
7
- end
8
-
9
- @basedir = "."
10
-
11
- def load(job_id)
12
- Job.load(self, job_id)
13
- end
14
-
15
- def job_id(name, job_options, previous_jobs)
16
- job_options = job_options.merge :previous_jobs => previous_jobs.collect{|job| job.id} if previous_jobs.any?
17
- if job_options.any?
18
- name.to_s + "_" + Misc.hash2md5(job_options)
19
- else
20
- name.to_s
21
- end
22
- end
23
-
24
- attr_accessor :name, :persistence, :options, :option_descriptions, :option_types, :option_defaults, :workflow, :dependencies, :scope, :description, :block
25
- def initialize(name, persistence = nil, options = nil, option_descriptions = nil, option_types = nil, option_defaults = nil, workflow = nil, dependencies = nil, scope = nil, description = nil, &block)
26
- dependencies = [dependencies] unless dependencies.nil? or Array === dependencies
27
- @name = name.to_s
28
-
29
- @persistence = persistence || :marshal
30
-
31
- @options = Array === options ? options : [options] unless options.nil?
32
-
33
- @option_defaults = option_defaults
34
- @option_descriptions = option_descriptions
35
- @option_types = option_types
36
- @workflow = workflow
37
- @dependencies = dependencies || []
38
- @scope = scope
39
- @description = description
40
-
41
- @block = block unless not block_given?
42
- end
43
-
44
- def process_options(args, optional_args)
45
- run_options = {}
46
-
47
- options.each do |option|
48
- if option_defaults and option_defaults.include? option
49
- run_options[option] = Misc.process_options(optional_args, option) || option_defaults[option]
50
- else
51
- run_options[option] = args.shift
52
- end
53
- end unless options.nil?
54
-
55
- [run_options, args, optional_args]
56
- end
57
-
58
- def pull_from_hash(args, optional_args)
59
- option_summary.first.each do |info|
60
- name = info[:name]
61
- if optional_args.include? name
62
- args.push optional_args.delete name
63
- end
64
- end
65
- end
66
-
67
- def setup(jobname, args, optional_args, dependencies)
68
- previous_jobs = []
69
- required_files = []
70
-
71
- pull_from_hash(args, optional_args)
72
- run_options, args, optional_args = process_options args, optional_args
73
-
74
- dependencies.each do |dependency|
75
- case
76
- when Proc === dependency
77
- deps = dependency.call(jobname, run_options)
78
- if Array === deps
79
- previous_jobs.concat deps
80
- else
81
- previous_jobs << deps
82
- end
83
- when Task::Job === dependency
84
- previous_jobs << dependency
85
- when Task === dependency
86
- previous_jobs << dependency.job(jobname, *(args + [optional_args]))
87
- else
88
- required_files << dependency
89
- end
90
- end
91
-
92
- [previous_jobs, required_files, run_options]
93
- end
94
-
95
- def job(jobname, *args)
96
- if Hash === args.last
97
- optional_args = args.pop
98
- else
99
- optional_args = {}
100
- end
101
-
102
-
103
- previous_jobs, required_files, run_options = setup(jobname, args, optional_args, dependencies)
104
-
105
- job_id = self.job_id jobname, run_options, previous_jobs
106
-
107
- Job.new(self, job_id, jobname, run_options, previous_jobs, required_files, previous_jobs.first)
108
- end
109
-
110
- def run(*args)
111
- job(*args).start
112
- end
113
-
114
- def option_info(option)
115
- info = {}
116
- info[:name] = option
117
- info[:source] = name
118
- info[:description] = option_descriptions[option] if option_descriptions and option_descriptions.include? option
119
- info[:type] = option_types[option] if option_types and option_types.include? option
120
- info[:default] = option_defaults[option] if option_defaults and option_defaults.include? option
121
-
122
- info
123
- end
124
-
125
- def option_summary
126
- options = []
127
- optional_options = []
128
-
129
- if self.options
130
- self.options.collect{|option|
131
- info = option_info(option)
132
- if info[:default].nil?
133
- options << info
134
- else
135
- optional_options << info
136
- end
137
- }
138
- end
139
-
140
- dependencies.select{|dep| Task === dep}.each do |task|
141
- more_options, more_optional_options = task.option_summary
142
- options.concat more_options
143
- optional_options.concat more_optional_options
144
- end
145
-
146
- [options, optional_options]
147
- end
148
-
149
- def usage
150
- usage = ""
151
- usage << "Task: #{name}\n"
152
- usage << "\nDescription: #{description.chomp}\n" if description
153
- options, optional_options = option_summary
154
-
155
- if options.any?
156
- usage << "\nMandatory options:\n"
157
- usage << "\tTask\tName\tType \tDescription\n"
158
- usage << "\t----\t----\t---- \t-----------\n"
159
-
160
- options.each do |option|
161
- option_line = "\t[#{option[:source]}]\t#{option[:name]}"
162
- option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
163
- option_line << "\t#{option[:description]}" if option[:description]
164
- usage << option_line << "\n"
165
- end
166
- end
167
-
168
- if optional_options.any?
169
- usage << "\nOptional options:\n"
170
- usage << "\tTask\tName\tDefault \tType \tDescription\n"
171
- usage << "\t----\t----\t------- \t---- \t-----------\n"
172
- optional_options.each do |option|
173
- option_line = "\t[#{option[:source]}]\t#{option[:name]}\t#{option[:default]}"
174
- option_line << "\t#{option[:type] ? option[:type] : "Unspec."}"
175
- option_line << "\t#{option[:description]}" if option[:description]
176
- usage << option_line << "\n"
177
- end
178
- end
179
-
180
- usage
181
- end
182
-
183
- end