rbbt-util 3.2.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/README.rdoc +65 -0
  2. data/bin/run_workflow.rb +142 -69
  3. data/lib/rbbt-util.rb +3 -3
  4. data/lib/rbbt.rb +12 -3
  5. data/lib/rbbt/annotations.rb +215 -0
  6. data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb} +17 -13
  7. data/lib/rbbt/persist.rb +164 -0
  8. data/lib/rbbt/persist/tsv.rb +135 -0
  9. data/lib/rbbt/resource.rb +100 -0
  10. data/lib/rbbt/resource/path.rb +180 -0
  11. data/lib/rbbt/resource/rake.rb +48 -0
  12. data/lib/rbbt/resource/util.rb +111 -0
  13. data/lib/rbbt/resource/with_key.rb +28 -0
  14. data/lib/rbbt/tsv.rb +134 -0
  15. data/lib/rbbt/tsv/accessor.rb +345 -0
  16. data/lib/rbbt/tsv/attach.rb +183 -0
  17. data/lib/rbbt/tsv/attach/util.rb +277 -0
  18. data/lib/rbbt/{util/tsv/filters.rb → tsv/filter.rb} +76 -37
  19. data/lib/rbbt/tsv/index.rb +453 -0
  20. data/lib/rbbt/tsv/manipulate.rb +361 -0
  21. data/lib/rbbt/tsv/parser.rb +231 -0
  22. data/lib/rbbt/tsv/serializers.rb +79 -0
  23. data/lib/rbbt/tsv/util.rb +67 -0
  24. data/lib/rbbt/util/R.rb +3 -3
  25. data/lib/rbbt/util/chain_methods.rb +64 -0
  26. data/lib/rbbt/util/cmd.rb +17 -13
  27. data/lib/rbbt/util/excel2tsv.rb +4 -3
  28. data/lib/rbbt/util/log.rb +1 -0
  29. data/lib/rbbt/util/misc.rb +296 -285
  30. data/lib/rbbt/util/open.rb +9 -2
  31. data/lib/rbbt/util/persistence.rb +1 -1
  32. data/lib/rbbt/util/task/job.rb +3 -1
  33. data/lib/rbbt/workflow.rb +193 -0
  34. data/lib/rbbt/workflow/accessor.rb +249 -0
  35. data/lib/rbbt/workflow/annotate.rb +60 -0
  36. data/lib/rbbt/workflow/soap.rb +100 -0
  37. data/lib/rbbt/workflow/step.rb +102 -0
  38. data/lib/rbbt/workflow/task.rb +76 -0
  39. data/test/rbbt/resource/test_path.rb +12 -0
  40. data/test/rbbt/test_annotations.rb +106 -0
  41. data/test/rbbt/{util/test_fix_width_table.rb → test_fix_width_table.rb} +8 -9
  42. data/test/rbbt/test_resource.rb +66 -0
  43. data/test/rbbt/test_tsv.rb +332 -0
  44. data/test/rbbt/test_workflow.rb +102 -0
  45. data/test/rbbt/tsv/test_accessor.rb +163 -0
  46. data/test/rbbt/{util/tsv → tsv}/test_attach.rb +86 -43
  47. data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb} +31 -13
  48. data/test/rbbt/tsv/test_index.rb +284 -0
  49. data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb +35 -105
  50. data/test/rbbt/util/test_R.rb +1 -1
  51. data/test/rbbt/util/test_chain_methods.rb +22 -0
  52. data/test/rbbt/util/test_filecache.rb +0 -1
  53. data/test/rbbt/util/test_misc.rb +97 -79
  54. data/test/rbbt/util/test_open.rb +1 -0
  55. data/test/rbbt/util/test_tmpfile.rb +1 -1
  56. data/test/rbbt/workflow/test_soap.rb +103 -0
  57. data/test/rbbt/workflow/test_step.rb +142 -0
  58. data/test/rbbt/workflow/test_task.rb +84 -0
  59. data/test/test_helper.rb +7 -7
  60. metadata +80 -54
  61. data/lib/rbbt/util/rake.rb +0 -176
  62. data/lib/rbbt/util/resource.rb +0 -355
  63. data/lib/rbbt/util/task.rb +0 -183
  64. data/lib/rbbt/util/tc_hash.rb +0 -324
  65. data/lib/rbbt/util/tsv.rb +0 -236
  66. data/lib/rbbt/util/tsv/accessor.rb +0 -312
  67. data/lib/rbbt/util/tsv/attach.rb +0 -416
  68. data/lib/rbbt/util/tsv/index.rb +0 -419
  69. data/lib/rbbt/util/tsv/manipulate.rb +0 -300
  70. data/lib/rbbt/util/tsv/misc.rb +0 -41
  71. data/lib/rbbt/util/tsv/parse.rb +0 -324
  72. data/lib/rbbt/util/tsv/resource.rb +0 -88
  73. data/lib/rbbt/util/workflow.rb +0 -135
  74. data/lib/rbbt/util/workflow/soap.rb +0 -116
  75. data/test/rbbt/util/test_persistence.rb +0 -201
  76. data/test/rbbt/util/test_rake.rb +0 -54
  77. data/test/rbbt/util/test_resource.rb +0 -77
  78. data/test/rbbt/util/test_task.rb +0 -133
  79. data/test/rbbt/util/test_tc_hash.rb +0 -144
  80. data/test/rbbt/util/test_tsv.rb +0 -221
  81. data/test/rbbt/util/test_workflow.rb +0 -135
  82. data/test/rbbt/util/tsv/test_accessor.rb +0 -150
  83. data/test/rbbt/util/tsv/test_index.rb +0 -241
  84. data/test/rbbt/util/tsv/test_parse.rb +0 -87
  85. data/test/rbbt/util/tsv/test_resource.rb +0 -9
@@ -0,0 +1,180 @@
1
+ require 'rbbt/util/chain_methods'
2
+ require 'rbbt/resource/util'
3
+ require 'rbbt/tsv'
4
+
5
+ module Path
6
+ attr_accessor :resource, :pkgdir
7
+
8
+ extend ChainMethods
9
+ self.chain_prefix = :path
10
+
11
+ def self.setup(string, pkgdir = nil, resource = nil)
12
+ string.extend Path
13
+ string.pkgdir = pkgdir || 'rbbt'
14
+ string.resource = resource
15
+ string
16
+ end
17
+
18
+ def self.extended(string)
19
+ setup_chains(string)
20
+ if not string.respond_to? :byte
21
+ class << string
22
+ alias byte path_clean_get_brackets
23
+ end
24
+ end
25
+ end
26
+
27
+ def join(name)
28
+ if self.empty?
29
+ Path.setup name.to_s, @pkgdir, @resource
30
+ else
31
+ Path.setup File.join(self, name.to_s), @pkgdir, @resource
32
+ end
33
+ end
34
+
35
+ def dirname
36
+ Path.setup File.dirname(self), @pkgdir, @resource
37
+ end
38
+
39
+ def path_get_brackets(name)
40
+ join name
41
+ end
42
+
43
+ def path_method_missing(name, prev = nil, *args, &block)
44
+ if block_given?
45
+ path_clean_method_missing name, prev, *args, &block
46
+ else
47
+ # Fix problem with ruby 1.9 calling methods by its own initiative. ARG
48
+ path_clean_method_missing(name, prev, *args) if name.to_s =~ /^to_/
49
+ if prev.nil?
50
+ join name
51
+ else
52
+ join(prev).join(name)
53
+ end
54
+ end
55
+ end
56
+
57
+ SEARCH_PATHS = {
58
+ :user => File.join(ENV['HOME'], ".{PKGDIR}", "{TOPLEVEL}", "{SUBPATH}"),
59
+ :global => File.join('/', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
60
+ :local => File.join('/usr/local', "{TOPLEVEL}", "{PKGDIR}", "{SUBPATH}"),
61
+ :lib => File.join('{LIBDIR}', "{TOPLEVEL}", "{SUBPATH}"),
62
+ }
63
+
64
+ def find(where = nil, caller_lib = nil)
65
+ return self if located?
66
+ if self.match(/(.*?)\/(.*)/)
67
+ toplevel, subpath = self.match(/(.*?)\/(.*)/).values_at 1, 2
68
+ else
69
+ toplevel, subpath = self, ""
70
+ end
71
+
72
+ path = nil
73
+ if where.nil?
74
+ SEARCH_PATHS.keys.each do |w|
75
+ path = find(w, caller_lib)
76
+ return path if File.exists? path
77
+ end
78
+ find :user
79
+ else
80
+ libdir = where == :lib ? Path.caller_lib_dir(caller_lib) : ""
81
+ libdir ||= ""
82
+ Path.setup SEARCH_PATHS[where].sub('{PKGDIR}', pkgdir).sub('{TOPLEVEL}', toplevel).sub('{SUBPATH}', subpath).sub('{LIBDIR}', libdir), @pkgdir, @resource
83
+ end
84
+ end
85
+
86
+ #{{{ Methods
87
+
88
+ def in_dir?(dir)
89
+ ! ! File.expand_path(self).match(/^#{Regexp.quote dir}/)
90
+ end
91
+
92
+ def to_s
93
+ self.find
94
+ end
95
+
96
+ def filename
97
+ self.find
98
+ end
99
+
100
+ def exists?
101
+ begin
102
+ self.produce
103
+ File.exists? self.find
104
+ rescue
105
+ false
106
+ end
107
+ end
108
+
109
+ def produce
110
+ path = self.find
111
+ return self if File.exists? path
112
+
113
+ raise "No resource defined to produce file: #{ self }" if resource.nil?
114
+
115
+ resource.produce path
116
+
117
+ path
118
+ end
119
+
120
+ def read
121
+ Open.read(self.produce.find)
122
+ end
123
+
124
+ def open
125
+ Open.open(self.produce.find)
126
+ end
127
+
128
+ def to_s
129
+ "" + self
130
+ end
131
+
132
+ def tsv(*args)
133
+ TSV.open(self.produce, *args)
134
+ end
135
+
136
+ def yaml
137
+ YAML.load self.open
138
+ end
139
+
140
+ def index(options = {})
141
+ TSV.index(self.produce.find, options)
142
+ end
143
+
144
+ def range_index(start, eend, options = {})
145
+ TSV.range_index(self.produce.find, start, eend, options)
146
+ end
147
+
148
+ def pos_index(pos, options = {})
149
+ TSV.pos_index(self.produce.find, pos, options)
150
+ end
151
+
152
+
153
+ def to_yaml(*args)
154
+ self.to_s.to_yaml(*args)
155
+ end
156
+
157
+ def fields
158
+ TSV.parse_header(self.open).fields
159
+ end
160
+
161
+ def all_fields
162
+ TSV.parse_header(self.open).all_fields
163
+ end
164
+
165
+ def identifier_file_path
166
+ if self.dirname.identifiers.exists?
167
+ self.dirname.identifiers
168
+ else
169
+ nil
170
+ end
171
+ end
172
+
173
+ def identifier_files
174
+ if identifier_file_path.nil?
175
+ []
176
+ else
177
+ [identifier_file_path.nil?]
178
+ end
179
+ end
180
+ end
@@ -0,0 +1,48 @@
1
+ require 'rake'
2
+ module Rake
3
+ class TaskNotFound < StandardError; end
4
+ def self.run(rakefile, dir, task)
5
+ old_pwd = FileUtils.pwd
6
+
7
+ Rake::FileTask.module_eval do
8
+ if not self.respond_to? :old_define_task
9
+ class << self
10
+ alias_method :old_define_task, :define_task
11
+ end
12
+
13
+ def self.define_task(file, *args, &block)
14
+ @@files ||= []
15
+ @@files << file
16
+ old_define_task(file, *args, &block)
17
+ end
18
+ end
19
+
20
+ def self.files
21
+ @@files
22
+ end
23
+
24
+ def self.clear_files
25
+ @@files = []
26
+ end
27
+ end
28
+
29
+ Rake::Task.clear
30
+ Rake::FileTask.clear_files
31
+
32
+ if block_given?
33
+ yield
34
+ else
35
+ load rakefile
36
+ end
37
+
38
+ raise TaskNotFound if Rake::Task[task].nil?
39
+
40
+ Misc.in_dir(dir) do
41
+ Rake::Task[task].invoke
42
+
43
+ Rake::Task.clear
44
+ Rake::FileTask.clear_files
45
+ end
46
+
47
+ end
48
+ end
@@ -0,0 +1,111 @@
1
+ require 'rbbt/resource/rake'
2
+
3
+ module Path
4
+
5
+ def self.caller_lib_dir(file = nil)
6
+ file = caller.reject{|l|
7
+ l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
8
+ l =~ /rbbt\/resource\/path\.rb/ or
9
+ l =~ /rbbt\/util\/misc\.rb/ or
10
+ l =~ /progress-monitor\.rb/
11
+ }.first.sub(/\.rb.*/,'.rb') if file.nil?
12
+
13
+ file = File.expand_path file
14
+ return Path.setup(file) if File.exists? File.join(file, 'lib')
15
+
16
+ while file != '/'
17
+ dir = File.dirname file
18
+ return Path.setup(dir) if File.exists? File.join(dir, 'lib')
19
+ file = File.dirname file
20
+ end
21
+
22
+ return nil
23
+ end
24
+
25
+ SLASH = "/"[0]
26
+ DOT = "."[0]
27
+ def located?
28
+ self.byte(0) == SLASH or (self.byte(0) == DOT and self.byte(1) == SLASH)
29
+ end
30
+ end
31
+
32
+ module Resource
33
+ def set_software_env(software_dir)
34
+ bin_dir = File.join(software_dir, 'bin')
35
+ opt_dir = File.join(software_dir, 'opt')
36
+
37
+ Misc.env_add 'PATH', bin_dir
38
+
39
+ FileUtils.mkdir_p opt_dir unless File.exists? opt_dir
40
+ %w(.ld-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
41
+ filename = File.join(opt_dir, file)
42
+ FileUtils.touch filename unless File.exists? filename
43
+ end
44
+
45
+ if not File.exists? File.join(opt_dir,'.post_install')
46
+ Open.write(File.join(opt_dir,'.post_install'),"#!/bin/bash\n")
47
+ end
48
+
49
+ Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
50
+ Misc.env_add('LD_LIBRARY_PATH',line.chomp)
51
+ Misc.env_add('LD_RUN_PATH',line.chomp)
52
+ end
53
+
54
+ Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
55
+ Misc.env_add('PKG_CONFIG_PATH',line.chomp)
56
+ end
57
+
58
+ Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
59
+ Misc.env_add('LD_LIBRARY_PATH',line.chomp)
60
+ end
61
+
62
+ Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
63
+ Misc.env_add('LD_LIBRARY_PATH',line.chomp)
64
+ end
65
+
66
+ Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
67
+ Misc.env_add('ACLOCAL_FLAGS', "-I#{File.join(opt_dir, line.chomp)}", ' ')
68
+ end
69
+
70
+ Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
71
+ Misc.env_add('CLASSPATH', "#{File.join(opt_dir,'java', 'lib', line.chomp)}")
72
+ end
73
+
74
+ Dir.glob(File.join opt_dir, 'jars', '*').each do |file|
75
+ Misc.env_add('CLASSPATH', "#{File.expand_path(file)}")
76
+ end
77
+
78
+ File.chmod 0774, File.join(opt_dir, '.post_install')
79
+
80
+ CMD.cmd(File.join(opt_dir, '.post_install'))
81
+ end
82
+
83
+
84
+ def rake_for(path)
85
+ entry = @rake_dirs.reject{|dir, content|
86
+ !Misc.common_path(dir, path)
87
+ }.sort_by{|dir, content|
88
+ dir.length
89
+ }.last
90
+ end
91
+
92
+ def has_rake(path)
93
+ !! rake_for(path)
94
+ end
95
+
96
+ def run_rake(path, rakefile, rake_dir)
97
+ task = Misc.path_relative_to rake_dir, path
98
+ rakefile = rakefile.produce if rakefile.respond_to? :produce
99
+
100
+ begin
101
+ Rake.run(rakefile, rake_dir, task)
102
+ rescue Rake::TaskNotFound
103
+ raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
104
+ task = File.join(File.basename(rake_dir), task)
105
+ rake_dir = File.dirname(rake_dir)
106
+ retry
107
+ end
108
+ end
109
+ end
110
+
111
+
@@ -0,0 +1,28 @@
1
+ module Resource
2
+ module WithKey
3
+ def self.extended(base)
4
+ class << base
5
+ attr_accessor :klass, :key
6
+ end
7
+ end
8
+
9
+ alias :old_method_missing :method_missing
10
+ def method_missing(name, *args)
11
+ return old_method_missing(name, *args) if name.to_s =~ /^to_/
12
+ if key
13
+ klass.send(name, key, *args)
14
+ else
15
+ klass.send(name, *args)
16
+ end
17
+ end
18
+ end
19
+
20
+ def with_key(key)
21
+ klass = self
22
+ o = Object.new
23
+ o.extend WithKey
24
+ o.klass = self
25
+ o.key = key
26
+ o
27
+ end
28
+ end
data/lib/rbbt/tsv.rb ADDED
@@ -0,0 +1,134 @@
1
+ require 'yaml'
2
+
3
+ require 'rbbt/util/misc'
4
+ require 'rbbt/util/log'
5
+
6
+ require 'rbbt/persist'
7
+
8
+ require 'rbbt/tsv/util'
9
+ require 'rbbt/tsv/serializers'
10
+ require 'rbbt/tsv/parser'
11
+ require 'rbbt/tsv/accessor'
12
+ require 'rbbt/tsv/manipulate'
13
+ require 'rbbt/tsv/index'
14
+ require 'rbbt/tsv/attach'
15
+ require 'rbbt/tsv/filter'
16
+
17
+ module TSV
18
+ def self.setup(hash, options = {})
19
+ hash = Misc.array2hash hash if Array === hash
20
+ hash.extend TSV
21
+
22
+ IndiferentHash.setup(options)
23
+ ENTRIES.each do |entry|
24
+ hash.send("#{ entry }=", options[entry]) if options.include? entry
25
+ hash.send("#{ entry }=", options[entry.to_sym]) if options.include? entry.to_sym
26
+ end
27
+
28
+ hash
29
+ end
30
+
31
+ # options shifts if type.nil?
32
+ def self.open(source, type = nil, options = nil)
33
+ type, options = nil, type if options.nil? and Hash === type
34
+ options ||= {}
35
+ options[:type] ||= type unless type.nil?
36
+
37
+ persist_options = Misc.pull_keys options, :persist
38
+
39
+ filename = get_filename source
40
+ serializer = Misc.process_options options, :serializer
41
+
42
+ Log.debug "TSV open: #{ filename } - #{options.inspect}"
43
+
44
+ Persist.persist_tsv source, filename, options, persist_options do |data|
45
+ if serializer
46
+ data.extend TSV unless TSV === data
47
+ data.serializer = serializer
48
+ end
49
+
50
+ stream = get_stream source
51
+ parse stream, data, options
52
+
53
+ data.filename = filename.to_s unless filename.nil?
54
+ if data.identifiers.nil? and Path === filename and filename.identifier_file_path
55
+ data.identifiers = filename.identifier_file_path.dup
56
+ end
57
+
58
+ data
59
+ end
60
+ end
61
+
62
+ def self.parse_header(stream, options = {})
63
+ Parser.new stream, options
64
+ end
65
+
66
+ def self.parse(stream, data, options = {})
67
+ monitor, grep = Misc.process_options options, :monitor, :grep
68
+ if grep
69
+ stream = Open.grep(stream, grep)
70
+ end
71
+
72
+ parser = Parser.new stream, options
73
+ line = parser.rescue_first_line
74
+
75
+ if TokyoCabinet::HDB === data and parser.straight
76
+ data.close
77
+ begin
78
+ CMD.cmd("tchmgr importtsv '#{data.persistence_path}'", :in => stream, :log => false)
79
+ rescue
80
+ Log.debug("tchmgr importtsv failed for: #{data.persistence_path}")
81
+ Log.debug($!.message)
82
+ end
83
+ data.write
84
+ end
85
+
86
+ data.extend TSV unless TSV === data
87
+ data.unnamed = true
88
+
89
+ data.serializer = parser.type if data.serializer == :type
90
+
91
+ if monitor and (stream.respond_to?(:size) or (stream.respond_to?(:stat) and stream.stat.respond_to? :size)) and stream.respond_to?(:pos)
92
+ size = case
93
+ when stream.respond_to?(:size)
94
+ stream.size
95
+ else
96
+ stream.stat.size
97
+ end
98
+ desc = "Parsing Stream"
99
+ step = 100
100
+ if Hash === monitor
101
+ desc = monitor[:desc] if monitor.include? :desc
102
+ step = monitor[:step] if monitor.include? :step
103
+ end
104
+ progress_monitor = Progress::Bar.new(size, 0, step, desc)
105
+ else
106
+ progress_monitor = nil
107
+ end
108
+
109
+ while not line.nil?
110
+ begin
111
+
112
+ progress_monitor.tick(stream.pos) if progress_monitor
113
+
114
+ line = parser.process line
115
+ parts = parser.chop_line line
116
+ key, values = parser.get_values parts
117
+ values = parser.cast_values values if parser.cast?
118
+ parser.add_to_data data, key, values
119
+ line = stream.gets
120
+ rescue Parser::SKIP_LINE
121
+ line = stream.gets
122
+ next
123
+ rescue IOError
124
+ break
125
+ end
126
+ end
127
+
128
+ parser.setup data
129
+
130
+ data.unnamed = false
131
+
132
+ data
133
+ end
134
+ end