rbbt-util 3.2.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/README.rdoc +65 -0
  2. data/bin/run_workflow.rb +142 -69
  3. data/lib/rbbt-util.rb +3 -3
  4. data/lib/rbbt.rb +12 -3
  5. data/lib/rbbt/annotations.rb +215 -0
  6. data/lib/rbbt/{util/fix_width_table.rb → fix_width_table.rb} +17 -13
  7. data/lib/rbbt/persist.rb +164 -0
  8. data/lib/rbbt/persist/tsv.rb +135 -0
  9. data/lib/rbbt/resource.rb +100 -0
  10. data/lib/rbbt/resource/path.rb +180 -0
  11. data/lib/rbbt/resource/rake.rb +48 -0
  12. data/lib/rbbt/resource/util.rb +111 -0
  13. data/lib/rbbt/resource/with_key.rb +28 -0
  14. data/lib/rbbt/tsv.rb +134 -0
  15. data/lib/rbbt/tsv/accessor.rb +345 -0
  16. data/lib/rbbt/tsv/attach.rb +183 -0
  17. data/lib/rbbt/tsv/attach/util.rb +277 -0
  18. data/lib/rbbt/{util/tsv/filters.rb → tsv/filter.rb} +76 -37
  19. data/lib/rbbt/tsv/index.rb +453 -0
  20. data/lib/rbbt/tsv/manipulate.rb +361 -0
  21. data/lib/rbbt/tsv/parser.rb +231 -0
  22. data/lib/rbbt/tsv/serializers.rb +79 -0
  23. data/lib/rbbt/tsv/util.rb +67 -0
  24. data/lib/rbbt/util/R.rb +3 -3
  25. data/lib/rbbt/util/chain_methods.rb +64 -0
  26. data/lib/rbbt/util/cmd.rb +17 -13
  27. data/lib/rbbt/util/excel2tsv.rb +4 -3
  28. data/lib/rbbt/util/log.rb +1 -0
  29. data/lib/rbbt/util/misc.rb +296 -285
  30. data/lib/rbbt/util/open.rb +9 -2
  31. data/lib/rbbt/util/persistence.rb +1 -1
  32. data/lib/rbbt/util/task/job.rb +3 -1
  33. data/lib/rbbt/workflow.rb +193 -0
  34. data/lib/rbbt/workflow/accessor.rb +249 -0
  35. data/lib/rbbt/workflow/annotate.rb +60 -0
  36. data/lib/rbbt/workflow/soap.rb +100 -0
  37. data/lib/rbbt/workflow/step.rb +102 -0
  38. data/lib/rbbt/workflow/task.rb +76 -0
  39. data/test/rbbt/resource/test_path.rb +12 -0
  40. data/test/rbbt/test_annotations.rb +106 -0
  41. data/test/rbbt/{util/test_fix_width_table.rb → test_fix_width_table.rb} +8 -9
  42. data/test/rbbt/test_resource.rb +66 -0
  43. data/test/rbbt/test_tsv.rb +332 -0
  44. data/test/rbbt/test_workflow.rb +102 -0
  45. data/test/rbbt/tsv/test_accessor.rb +163 -0
  46. data/test/rbbt/{util/tsv → tsv}/test_attach.rb +86 -43
  47. data/test/rbbt/{util/tsv/test_filters.rb → tsv/test_filter.rb} +31 -13
  48. data/test/rbbt/tsv/test_index.rb +284 -0
  49. data/test/rbbt/{util/tsv → tsv}/test_manipulate.rb +35 -105
  50. data/test/rbbt/util/test_R.rb +1 -1
  51. data/test/rbbt/util/test_chain_methods.rb +22 -0
  52. data/test/rbbt/util/test_filecache.rb +0 -1
  53. data/test/rbbt/util/test_misc.rb +97 -79
  54. data/test/rbbt/util/test_open.rb +1 -0
  55. data/test/rbbt/util/test_tmpfile.rb +1 -1
  56. data/test/rbbt/workflow/test_soap.rb +103 -0
  57. data/test/rbbt/workflow/test_step.rb +142 -0
  58. data/test/rbbt/workflow/test_task.rb +84 -0
  59. data/test/test_helper.rb +7 -7
  60. metadata +80 -54
  61. data/lib/rbbt/util/rake.rb +0 -176
  62. data/lib/rbbt/util/resource.rb +0 -355
  63. data/lib/rbbt/util/task.rb +0 -183
  64. data/lib/rbbt/util/tc_hash.rb +0 -324
  65. data/lib/rbbt/util/tsv.rb +0 -236
  66. data/lib/rbbt/util/tsv/accessor.rb +0 -312
  67. data/lib/rbbt/util/tsv/attach.rb +0 -416
  68. data/lib/rbbt/util/tsv/index.rb +0 -419
  69. data/lib/rbbt/util/tsv/manipulate.rb +0 -300
  70. data/lib/rbbt/util/tsv/misc.rb +0 -41
  71. data/lib/rbbt/util/tsv/parse.rb +0 -324
  72. data/lib/rbbt/util/tsv/resource.rb +0 -88
  73. data/lib/rbbt/util/workflow.rb +0 -135
  74. data/lib/rbbt/util/workflow/soap.rb +0 -116
  75. data/test/rbbt/util/test_persistence.rb +0 -201
  76. data/test/rbbt/util/test_rake.rb +0 -54
  77. data/test/rbbt/util/test_resource.rb +0 -77
  78. data/test/rbbt/util/test_task.rb +0 -133
  79. data/test/rbbt/util/test_tc_hash.rb +0 -144
  80. data/test/rbbt/util/test_tsv.rb +0 -221
  81. data/test/rbbt/util/test_workflow.rb +0 -135
  82. data/test/rbbt/util/tsv/test_accessor.rb +0 -150
  83. data/test/rbbt/util/tsv/test_index.rb +0 -241
  84. data/test/rbbt/util/tsv/test_parse.rb +0 -87
  85. data/test/rbbt/util/tsv/test_resource.rb +0 -9
@@ -1,135 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
- require 'rbbt/util/workflow'
3
-
4
- module MathWF
5
- extend WorkFlow
6
- self.tasks[:input] = Task.new(:input, :marshal, :value) do |value| value end
7
- self.tasks[:add_1] = Task.new(:add_1, :marshal) do input + 1 end
8
- self.tasks[:add_1].dependencies << tasks[:input]
9
- self.tasks[:times_2] = Task.new(:times_2, :marshal) do input * 2 end
10
- self.tasks[:times_2].dependencies << tasks[:add_1]
11
- self.tasks[:times_4] = Task.new(:times_4, :marshal) do input * 4 end
12
- self.tasks[:times_4].dependencies << tasks[:add_1]
13
- self.tasks.each do |name, task| task.workflow = self end
14
- end
15
-
16
-
17
- module MathWF2
18
- def mult(num, t)
19
- num * t
20
- end
21
-
22
- extend WorkFlow
23
-
24
-
25
- task_option :value
26
- task :input => :integer do |value| value end
27
-
28
- task :add_1 => :integer do input + 1 end
29
-
30
- task :times_2 => :integer do input * 2 end
31
-
32
- task_dependencies :add_1
33
- task :times_4 => :integer do input * 4 end
34
-
35
- task_option :times, "Times to multiply by", :integer, 10
36
- task_dependencies :add_1
37
- task :times => :integer do |times| mult(input, times) end
38
-
39
- task_dependencies []
40
- task :persist do
41
- task.workflow.methods.include?(:local_persist) or
42
- task.workflow.methods.include?("local_persist")
43
- end
44
- end
45
-
46
-
47
- module ConWF
48
- extend WorkFlow
49
- task :one => :integer do 1 end
50
-
51
- task_dependencies []
52
- task :two => :integer do 2 end
53
-
54
- task_option :init, "Initial value", :string
55
- task_dependencies Proc.new{|jobname,run_options|
56
- ConWF.job(run_options[:init].to_sym, jobname, {})
57
- }
58
- task :times_2 => :integer do
59
- input * 2
60
- end
61
- end
62
-
63
- MathWF2.jobdir = Rbbt.tmp.test.jobs.mathwf.find :user
64
- MathWF.jobdir = Rbbt.tmp.test.jobs.mathwf.find :user
65
- ConWF.jobdir = Rbbt.tmp.test.jobs.mathwf.find :user
66
-
67
- class TestWorkFlow < Test::Unit::TestCase
68
-
69
- def test_math_wf
70
- job = MathWF.tasks[:times_2].job(:job1, 1)
71
- assert_equal 4, job.recursive_clean.fork.join.load
72
- job = MathWF.tasks[:times_4].job(:job1, 1)
73
- assert_equal 8, job.recursive_clean.fork.join.load
74
- end
75
-
76
- def test_math_wf2
77
- job = MathWF2.tasks[:times_2].job(:job1, 1)
78
- job.recursive_clean.fork.join
79
- assert_equal 4, job.load
80
- job = MathWF2.tasks[:times_4].job(:job1, 1)
81
- assert_equal 8, job.recursive_clean.fork.join.load
82
- end
83
-
84
- def test_math_run
85
- job = MathWF2.job(:times_2, :job1, 1)
86
- job.recursive_clean.fork.join
87
- assert_equal 4, job.load
88
- end
89
-
90
- def test_math_defaults
91
- assert MathWF2.tasks[:times].option_defaults.include? :times
92
-
93
- job = MathWF2.job(:times, :job1, 1)
94
- job.recursive_clean.fork.join
95
- assert job.done?
96
- puts job.messages
97
- assert File.exists? job.path
98
- assert_equal 20, job.load
99
-
100
-
101
- job = MathWF2.job(:times, :job1, 1, :times => 20)
102
- job.recursive_clean.fork.join
103
- assert_equal 40, job.load
104
- end
105
-
106
- def test_recursive_clean
107
- job = MathWF2.job(:times, :job1, 1, :times => 20)
108
- job.fork.join
109
- assert File.exists?(job.path)
110
- job.clean
111
- assert (not File.exists?(job.path))
112
- assert File.exists?(job.previous_jobs.first.path)
113
- job.recursive_clean
114
- assert (not File.exists?(job.previous_jobs.first.path))
115
- end
116
-
117
- def test_local_persist
118
- MathWF2.job(:persist, :persist).clean
119
- assert MathWF2.run(:persist, :persist).load
120
- end
121
-
122
- def test_conditional
123
- assert_equal 2, ConWF.run(:times_2, "Test", "one").load
124
- assert_equal 4, ConWF.run(:times_2, "Test", "two").load
125
- end
126
-
127
- def test_option_summary
128
- assert MathWF.tasks[:times_2].option_summary.first.first[:name] == :value
129
- end
130
-
131
- def test_usage
132
- assert MathWF.tasks[:times_2].usage =~ /^Task:/
133
- end
134
- end
135
-
@@ -1,150 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/tsv/accessor'
3
- require 'rbbt/util/tsv'
4
-
5
- class TestTSVAcessor < Test::Unit::TestCase
6
-
7
- def test_zip_fields
8
- a = [%w(1 2), %w(a b)]
9
- assert_equal a, TSV.zip_fields(TSV.zip_fields(a))
10
- end
11
-
12
- def test_values_at
13
- content =<<-EOF
14
- #Id ValueA ValueB OtherID
15
- row1 a|aa|aaa b Id1|Id2
16
- row2 A B Id3
17
- EOF
18
-
19
- TmpFile.with_file(content) do |filename|
20
- tsv = TSV.new(File.open(filename), :list, :sep => /\s+/, :key => "OtherID", :persistence => true)
21
- assert_equal "row2", tsv.values_at("Id1", "Id3").last.first
22
- end
23
- end
24
-
25
- def test_to_s
26
- content =<<-EOF
27
- #Id ValueA ValueB OtherID
28
- row1 a|aa|aaa b Id1|Id2
29
- row2 A B Id3
30
- EOF
31
-
32
- content2 =<<-EOF
33
- #Id ValueA ValueB OtherID
34
- row1 a|aa|aaa b Id1|Id2
35
- row2 A B Id3
36
- EOF
37
-
38
- TmpFile.with_file(content) do |filename|
39
- tsv = TSV.new(File.open(filename), :sep => /\s+/)
40
- assert_equal content, tsv.to_s.sub(/^#: [^\n]*\n/s,'')
41
- end
42
- end
43
-
44
- def test_to_s_ordered
45
- content =<<-EOF
46
- #Id ValueA ValueB OtherID
47
- row1 a|aa|aaa b Id1|Id2
48
- row2 A B Id3
49
- EOF
50
-
51
- content2 =<<-EOF
52
- #Id ValueA ValueB OtherID
53
- row2 A B Id3
54
- row1 a|aa|aaa b Id1|Id2
55
- EOF
56
-
57
-
58
- TmpFile.with_file(content) do |filename|
59
- tsv = TSV.new(File.open(filename), :sep => /\s+/)
60
- assert_equal content, tsv.to_s(%w(row1 row2)).sub(/^#: [^\n]*\n/s,'')
61
- assert_not_equal content, tsv.to_s(%w(row2 row1)).sub(/^#: [^\n]*\n/s,'')
62
- assert_equal content2, tsv.to_s(%w(row2 row1)).sub(/^#: [^\n]*\n/s,'')
63
- end
64
- end
65
-
66
- def test_field_compare
67
- content =<<-EOF
68
- #Id Letter:LetterValue Other:LetterValue OtherID
69
- row1 a|aa|aaa b Id1|Id2
70
- row2 A B Id3
71
- row3 a C Id4
72
- EOF
73
-
74
- TmpFile.with_file(content) do |filename|
75
- tsv = TSV.new(filename + '#:sep=/\s+/')
76
-
77
- assert tsv.fields.include?("LetterValue")
78
- end
79
- end
80
-
81
- def test_indentify_fields
82
- content =<<-EOF
83
- #ID ValueA ValueB Comment
84
- row1 a b c
85
- row2 A B C
86
- EOF
87
-
88
- TmpFile.with_file(content) do |filename|
89
- tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
90
- assert_equal :key, tsv.identify_field("ID")
91
- end
92
- end
93
-
94
- def test_named_fields
95
- content =<<-EOF
96
- #ID ValueA ValueB Comment
97
- row1 a b c
98
- row2 A B C
99
- EOF
100
-
101
- TmpFile.with_file(content) do |filename|
102
- tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
103
-
104
- assert_equal "ValueA", tsv.fields["ValueA"]
105
- end
106
- end
107
-
108
- def test_namespace
109
- content =<<-EOF
110
- #ID ValueA ValueB Comment
111
- row1 a b c
112
- row2 A B C
113
- EOF
114
-
115
- TmpFile.with_file(content) do |filename|
116
- tsv = TSV.new(File.open(filename), :double, :sep => /\s/, :namespace => "TEST")
117
-
118
- assert_equal "TEST", tsv.fields.first.namespace
119
- assert_equal "TEST", tsv.key_field.namespace
120
- end
121
- end
122
-
123
- def test_delete
124
- content =<<-EOF
125
- #ID ValueA ValueB Comment
126
- row1 a b c
127
- row2 A B C
128
- EOF
129
-
130
- TmpFile.with_file(content) do |filename|
131
- tsv = TSV.new(File.open(filename), :double, :sep => /\s/)
132
- tsv.delete "row1"
133
-
134
- assert_equal 1, tsv.size
135
- assert_equal ["row2"], tsv.keys
136
-
137
- tsv = TSV.new(File.open(filename), :double, :sep => /\s/, :persistence => true)
138
- assert_equal 2, tsv.size
139
- assert_equal ["row1", "row2"], tsv.keys
140
- tsv.write
141
- tsv.delete "row1"
142
- tsv.read
143
-
144
- assert_equal 1, tsv.size
145
- assert_equal ["row2"], tsv.keys
146
- end
147
- end
148
-
149
- end
150
-
@@ -1,241 +0,0 @@
1
- require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
- require 'rbbt/util/tsv'
3
- require 'rbbt/util/tsv/index'
4
-
5
- class TestTSVManipulate < Test::Unit::TestCase
6
-
7
- def test_index
8
- content =<<-EOF
9
- #Id ValueA ValueB OtherID
10
- row1 a|aa|aaa b Id1|Id2
11
- row2 A B Id3
12
- EOF
13
-
14
- TmpFile.with_file(content) do |filename|
15
- tsv = TSV.new(File.open(filename), :sep => /\s+/, :key => "OtherID", :persistence => false)
16
- index = tsv.index(:case_insensitive => true, :persistence => true)
17
- assert index["row1"].include? "Id1"
18
- assert_equal "OtherID", index.fields.first
19
- end
20
-
21
- # TmpFile.with_file(content) do |filename|
22
- # tsv = TSV.new(File.open(filename), :sep => /\s+/, :key => "OtherID")
23
- # index = tsv.index(:case_insensitive => true)
24
- # assert index["row1"].include? "Id1"
25
- # assert_equal "OtherID", index.fields.first
26
- # end
27
- end
28
-
29
- def test_index_headerless
30
- content =<<-EOF
31
- row1 a|aa|aaa b Id1|Id2
32
- row2 A B Id3
33
- EOF
34
-
35
- TmpFile.with_file(content) do |filename|
36
- tsv = TSV.new(File.open(filename), :sep => /\s+/)
37
- index = tsv.index(:case_insensitive => true, :target => 2)
38
- assert index["row1"].include? "Id1"
39
- end
40
- end
41
-
42
-
43
- def test_best_index
44
- content =<<-EOF
45
- #Id ValueA ValueB OtherID
46
- row1 a|aa|aaa b|A Id1
47
- row2 A a|B Id3
48
- row3 A a|B Id4
49
- EOF
50
-
51
- TmpFile.with_file(content) do |filename|
52
- tsv = TSV.new(File.open(filename), :sep => /\s+/, :key => "OtherID", :persistence => true)
53
- index = tsv.index(:case_insensitive => false, :order => true, :persistence => true)
54
- assert_equal "Id1", index['a'].first
55
- assert_equal "Id3", index['A'].first
56
- assert_equal "OtherID", index.fields.first
57
- end
58
-
59
- TmpFile.with_file(content) do |filename|
60
- tsv = TSV.new(File.open(filename), :sep => /\s+/, :key => "OtherID")
61
- index = tsv.index(:case_insensitive => true)
62
- assert index["row1"].include? "Id1"
63
- assert_equal "OtherID", index.fields.first
64
- end
65
- end
66
-
67
- #{{{ Test Attach
68
-
69
- def test_smart_merge_single
70
- content1 =<<-EOF
71
- #: :case_insensitive=false
72
- #Id ValueA ValueB
73
- row1 a|aa|aaa b
74
- row2 A B
75
- EOF
76
-
77
- content2 =<<-EOF
78
- #: :case_insensitive=false
79
- #ValueC ValueB OtherID
80
- c|cc|ccc b Id1|Id2
81
- C B Id3
82
- EOF
83
-
84
- tsv1 = tsv2 = nil
85
- TmpFile.with_file(content1) do |filename|
86
- tsv1 = TSV.new(File.open(filename), :double, :sep => /\s+/)
87
- end
88
-
89
- TmpFile.with_file(content2) do |filename|
90
- tsv2 = TSV.new(File.open(filename), :double, :sep => /\s+/)
91
- end
92
-
93
- tsv1 = tsv1.smart_merge tsv2, "ValueB"
94
-
95
- assert_equal "C", tsv1["row2"]["ValueC"].first
96
- assert %w(c cc ccc).include? tsv1["row1"]["ValueC"].first
97
- assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"].sort
98
- end
99
-
100
- def test_index_to_key
101
- content =<<-EOF
102
- #: :sep=/\\s+/
103
- #Id ValueA ValueB
104
- row1 a|aa|aaa b
105
- row2 A B
106
- EOF
107
-
108
- tsv1 = tsv2 = nil
109
- TmpFile.with_file(content) do |filename|
110
- tsv1 = TSV.new(File.open(filename), :double, :sep => /\s+/, :key => "ValueA", :case_insensitive => true)
111
- end
112
- end
113
-
114
- # {{{ Test sorted index
115
-
116
- def load_data(data)
117
- Log.debug("Data:\n#{Open.read(data)}")
118
- tsv = TSV.new(data, :list, :sep=>":", :cast => proc{|e| e =~ /(\s*)(_*)/; ($1.length..($1.length + $2.length - 1))})
119
- tsv.add_field "Start" do |key, values|
120
- values["Range"].first
121
- end
122
- tsv.add_field "End" do |key, values|
123
- values["Range"].last
124
- end
125
-
126
- tsv = tsv.slice ["Start", "End"]
127
-
128
- tsv
129
- end
130
-
131
- def test_pos_index
132
- content =<<-EOF
133
- #Id ValueA ValueB Pos
134
- row1 a|aa|aaa b 0|10
135
- row2 A B 30
136
- EOF
137
-
138
- TmpFile.with_file(content) do |filename|
139
- tsv = TSV.new(File.open(filename), :double, :sep => /\s+/)
140
- index = tsv.pos_index("Pos")
141
- assert_equal ["row1"], index[10]
142
- end
143
- end
144
-
145
-
146
- def test_range_index
147
- content =<<-EOF
148
- #Id ValueA ValueB Pos1 Pos2
149
- row1 a|aa|aaa b 0|10 10|30
150
- row2 A B 30 35
151
- EOF
152
-
153
- TmpFile.with_file(content) do |filename|
154
- tsv = TSV.new(File.open(filename), :double, :sep => /\s+/)
155
- index = tsv.pos_index("Pos1")
156
- assert_equal ["row1"], index[10]
157
-
158
- index = tsv.range_index("Pos1", "Pos2")
159
- assert_equal ["row1"], index[20]
160
- end
161
- end
162
-
163
- def test_range_index2
164
- data =<<-EOF
165
- #ID:Range
166
- #:012345678901234567890
167
- a: ______
168
- b: ______
169
- c: _______
170
- d: ____
171
- e: ______
172
- f: ___
173
- g: ____
174
- EOF
175
- TmpFile.with_file(data) do |datafile|
176
- tsv = load_data(datafile)
177
- f = tsv.range_index("Start", "End")
178
-
179
- assert_equal %w(), f[0].sort
180
- assert_equal %w(b), f[1].sort
181
- assert_equal %w(), f[20].sort
182
- assert_equal %w(), f[(20..100)].sort
183
- assert_equal %w(a b d), f[3].sort
184
- assert_equal %w(a b c d e), f[(3..4)].sort
185
- end
186
- end
187
-
188
- def test_range_index_persistent
189
- data =<<-EOF
190
- #ID:Range
191
- #:012345678901234567890
192
- a: ______
193
- b: ______
194
- c: _______
195
- d: ____
196
- e: ______
197
- f: ___
198
- g: ____
199
- EOF
200
- TmpFile.with_file(data) do |datafile|
201
- TmpFile.with_file(load_data(datafile)) do |tsvfile|
202
- f = TSV.range_index(tsvfile, "Start", "End", :persistence => true)
203
-
204
- assert_equal %w(), f[0].sort
205
- assert_equal %w(b), f[1].sort
206
- assert_equal %w(), f[20].sort
207
- assert_equal %w(), f[(20..100)].sort
208
- assert_equal %w(a b d), f[3].sort
209
- assert_equal %w(a b c d e), f[(3..4)].sort
210
- end
211
- end
212
- end
213
-
214
- def test_range_index_persistent_with_filter
215
- data =<<-EOF
216
- #ID:Range
217
- #:012345678901234567890
218
- a: ______
219
- b: ______
220
- c: _______
221
- d: ____
222
- e: ______
223
- f: ___
224
- g: ____
225
- EOF
226
- TmpFile.with_file(data) do |datafile|
227
- TmpFile.with_file(load_data(datafile)) do |tsvfile|
228
- f = TSV.range_index(tsvfile, "Start", "End", :filters => [["field:Start", "3"]])
229
-
230
- assert_equal %w(), f[0].sort
231
- assert_equal %w(), f[1].sort
232
- assert_equal %w(), f[20].sort
233
- assert_equal %w(), f[(20..100)].sort
234
- assert_equal %w(a), f[3].sort
235
- assert_equal %w(a), f[(3..4)].sort
236
- end
237
- end
238
- end
239
-
240
- end
241
-