rbbt-util 5.7.0 → 5.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/annotations.rb +4 -1
  3. data/lib/rbbt/annotations/util.rb +11 -0
  4. data/lib/rbbt/persist.rb +8 -2
  5. data/lib/rbbt/resource/path.rb +1 -0
  6. data/lib/rbbt/tsv/accessor.rb +18 -15
  7. data/lib/rbbt/tsv/parallel.rb +89 -32
  8. data/lib/rbbt/tsv/util.rb +11 -0
  9. data/lib/rbbt/util/R.rb +0 -1
  10. data/lib/rbbt/util/concurrency.rb +2 -0
  11. data/lib/rbbt/util/concurrency/processes.rb +96 -0
  12. data/lib/rbbt/util/concurrency/processes/socket.rb +87 -0
  13. data/lib/rbbt/util/concurrency/processes/socket_old.rb +144 -0
  14. data/lib/rbbt/util/concurrency/processes/worker.rb +53 -0
  15. data/lib/rbbt/util/concurrency/threads.rb +76 -0
  16. data/lib/rbbt/util/log.rb +37 -5
  17. data/lib/rbbt/util/misc.rb +89 -4
  18. data/lib/rbbt/util/semaphore.rb +10 -4
  19. data/lib/rbbt/util/simpleopt/accessor.rb +5 -0
  20. data/lib/rbbt/util/simpleopt/doc.rb +2 -4
  21. data/lib/rbbt/workflow/accessor.rb +39 -12
  22. data/lib/rbbt/workflow/step.rb +5 -7
  23. data/share/rbbt_commands/benchmark/pthrough +18 -0
  24. data/share/rbbt_commands/color +41 -0
  25. data/share/rbbt_commands/stat/density +50 -0
  26. data/share/rbbt_commands/tsv/info +21 -3
  27. data/share/rbbt_commands/tsv/slice +46 -0
  28. data/share/rbbt_commands/tsv/subset +53 -0
  29. data/share/rbbt_commands/tsv/values +7 -1
  30. data/test/rbbt/annotations/test_util.rb +14 -0
  31. data/test/rbbt/tsv/test_parallel.rb +25 -3
  32. data/test/rbbt/tsv/test_util.rb +15 -0
  33. data/test/rbbt/util/concurrency/processes/test_socket.rb +37 -0
  34. data/test/rbbt/util/concurrency/test_processes.rb +53 -0
  35. data/test/rbbt/util/concurrency/test_threads.rb +42 -0
  36. data/test/rbbt/util/test_concurrency.rb +6 -0
  37. metadata +23 -2
@@ -0,0 +1,14 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/annotations'
3
+
4
+ class TestClass < Test::Unit::TestCase
5
+ def test_marshal
6
+ a = "STRING"
7
+ a.extend Annotated
8
+
9
+ assert_equal a, Marshal.load(Marshal.dump(a))
10
+ assert_equal Marshal.dump("STRING"), Marshal.dump(a)
11
+
12
+ end
13
+ end
14
+
@@ -4,17 +4,39 @@ require 'rbbt/tsv/parallel'
4
4
 
5
5
  class TestTSVParallel < Test::Unit::TestCase
6
6
 
7
- def test_pthrough
7
+ def _test_pthrough
8
8
  require 'rbbt/sources/organism'
9
- tsv = Organism.identifiers("Hsa").tsv :unnamed => true
9
+ tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
10
10
 
11
11
  h = {}
12
12
  tsv.monitor = true
13
+ tsv.unnamed = true
13
14
  tsv.pthrough do |k,v|
14
15
  h[k] = v.first
15
16
  end
16
17
 
17
- assert h.size > 0
18
+ assert_equal tsv.size, h.size
19
+ assert_equal tsv.keys.sort, h.keys.sort
18
20
  end
19
21
 
22
+
23
+ def test_ppthrough
24
+ require 'rbbt/sources/organism'
25
+ tsv = Organism.identifiers("Hsa").tsv :unnamed => true, :persist => false, :fields => ["Associated Gene Name"]
26
+
27
+ h = {}
28
+
29
+ tsv.ppthrough_callback do |k,v|
30
+ h[k] = v
31
+ end
32
+
33
+ tsv.unnamed = true
34
+ tsv.monitor = true
35
+ tsv.ppthrough(3) do |k,v|
36
+ [k,v.first]
37
+ end
38
+
39
+ assert_equal tsv.size, h.size
40
+ assert_equal tsv.keys.sort, h.keys.sort
41
+ end
20
42
  end
@@ -23,4 +23,19 @@ row2 A B Id3
23
23
 
24
24
  end
25
25
 
26
+ def test_marshal
27
+ content =<<-EOF
28
+ #Id ValueA ValueB OtherID
29
+ row1 a|aa|aaa b Id1|Id2
30
+ row2 A B Id3
31
+ EOF
32
+
33
+ TmpFile.with_file(content) do |filename|
34
+ tsv = TSV.open(filename, :sep => /\s+/)
35
+
36
+ assert TSV === Marshal.load(Marshal.dump(tsv))
37
+ assert_equal tsv.to_hash, Marshal.load(Marshal.dump(tsv)).to_hash
38
+ assert_equal({1 => 1}, Marshal.load(Marshal.dump({1 => 1})))
39
+ end
40
+ end
26
41
  end
@@ -0,0 +1,37 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../../..', 'test_helper.rb')
2
+
3
+ require 'rbbt-util'
4
+ require 'rbbt/util/log'
5
+ require 'rbbt/util/concurrency/processes/socket'
6
+
7
+ class TestConcurrency < Test::Unit::TestCase
8
+ def test_socket_push_pop
9
+ obj1 = [1,[2,3,4]] #* 1000
10
+ obj2 = ["1",["2","3","4"]] #* 1000
11
+ obj3 = "some string"
12
+ obj4 = TSV.setup({1 => 1})
13
+
14
+ socket = RbbtProcessQueue::RbbtProcessSocket.new
15
+ 10.times do
16
+
17
+ socket.push(obj1)
18
+ socket.push(obj2)
19
+ socket.push(obj3)
20
+ socket.push(obj4)
21
+
22
+ assert_equal obj1, socket.pop
23
+ assert_equal obj2, socket.pop
24
+ assert_equal obj3, socket.pop
25
+ assert_equal obj4, socket.pop
26
+
27
+
28
+ end
29
+
30
+ socket.swrite.close
31
+ assert ClosedStream === socket.pop
32
+
33
+ socket.clean
34
+ end
35
+ end
36
+
37
+
@@ -0,0 +1,53 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+
3
+ require 'rbbt-util'
4
+ require 'rbbt/util/log'
5
+ require 'rbbt/util/concurrency/processes'
6
+
7
+ class TestConcurrencyProcess < Test::Unit::TestCase
8
+
9
+ def test_process
10
+ q = RbbtProcessQueue.new 1
11
+
12
+ res = []
13
+
14
+ q.callback do |v|
15
+ res << v
16
+ end
17
+
18
+ q.init do |i|
19
+ i * 2
20
+ end
21
+
22
+ times = 500
23
+ t = TSV.setup({"a" => 1}, :type => :single)
24
+
25
+ Misc.benchmark do
26
+ times.times do |i|
27
+ q.process i
28
+ end
29
+
30
+ q.join
31
+ q.clean
32
+ end
33
+
34
+ assert_equal times, res.length
35
+ assert_equal [0, 2, 4], res.sort[0..2]
36
+
37
+ end
38
+
39
+ def test_each
40
+ times = 5000
41
+ elems = (0..times-1).to_a
42
+
43
+ TmpFile.with_file do |dir|
44
+ RbbtProcessQueue.each(elems) do |elem|
45
+ Open.write(File.join(dir, elem.to_s), "DONE")
46
+ end
47
+
48
+ assert_equal times, Dir.glob(File.join(dir, '*')).length
49
+ end
50
+ end
51
+ end
52
+
53
+
@@ -0,0 +1,42 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+ require 'rbbt/util/concurrency'
3
+
4
+ class TestConcurrencyThreads < Test::Unit::TestCase
5
+ def test_thread_queue
6
+ q = RbbtThreadQueue.new 10
7
+
8
+ res = []
9
+
10
+ q.init do |v|
11
+ res << v
12
+ end
13
+
14
+ times = 5_000_000
15
+ Misc.benchmark do
16
+ times.times do |i|
17
+ q.process [i*2]
18
+ end
19
+ end
20
+
21
+ q.join
22
+ q.clean
23
+
24
+ assert_equal times, res.length
25
+ assert_equal [0, 2, 4], res.sort[0..2]
26
+
27
+ end
28
+
29
+ def test_each
30
+ times = 50000
31
+ elems = (0..times-1).to_a
32
+
33
+ TmpFile.with_file do |dir|
34
+ elems.each do |elem|
35
+ Open.write(File.join(dir, elem.to_s), "DONE")
36
+ end
37
+
38
+ assert_equal times, Dir.glob(File.join(dir, '*')).length
39
+ end
40
+ end
41
+ end
42
+
@@ -0,0 +1,6 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/util/concurrency'
3
+
4
+ class TestConcurrency < Test::Unit::TestCase
5
+ end
6
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.7.0
4
+ version: 5.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-03-03 00:00:00.000000000 Z
11
+ date: 2014-03-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -161,6 +161,12 @@ files:
161
161
  - lib/rbbt/util/cmd.rb
162
162
  - lib/rbbt/util/color.rb
163
163
  - lib/rbbt/util/colorize.rb
164
+ - lib/rbbt/util/concurrency.rb
165
+ - lib/rbbt/util/concurrency/processes.rb
166
+ - lib/rbbt/util/concurrency/processes/socket.rb
167
+ - lib/rbbt/util/concurrency/processes/socket_old.rb
168
+ - lib/rbbt/util/concurrency/processes/worker.rb
169
+ - lib/rbbt/util/concurrency/threads.rb
164
170
  - lib/rbbt/util/excel2tsv.rb
165
171
  - lib/rbbt/util/filecache.rb
166
172
  - lib/rbbt/util/log.rb
@@ -193,6 +199,8 @@ files:
193
199
  - share/rbbt_commands/app/install
194
200
  - share/rbbt_commands/app/start
195
201
  - share/rbbt_commands/app/template
202
+ - share/rbbt_commands/benchmark/pthrough
203
+ - share/rbbt_commands/color
196
204
  - share/rbbt_commands/conf/web_user/add
197
205
  - share/rbbt_commands/conf/web_user/list
198
206
  - share/rbbt_commands/conf/web_user/remove
@@ -203,6 +211,7 @@ files:
203
211
  - share/rbbt_commands/resource/find
204
212
  - share/rbbt_commands/resource/get
205
213
  - share/rbbt_commands/resource/produce
214
+ - share/rbbt_commands/stat/density
206
215
  - share/rbbt_commands/study/task
207
216
  - share/rbbt_commands/system/purge
208
217
  - share/rbbt_commands/system/report
@@ -211,6 +220,8 @@ files:
211
220
  - share/rbbt_commands/tsv/get
212
221
  - share/rbbt_commands/tsv/info
213
222
  - share/rbbt_commands/tsv/json
223
+ - share/rbbt_commands/tsv/slice
224
+ - share/rbbt_commands/tsv/subset
214
225
  - share/rbbt_commands/tsv/unzip
215
226
  - share/rbbt_commands/tsv/values
216
227
  - share/rbbt_commands/workflow/cmd
@@ -226,6 +237,7 @@ files:
226
237
  - share/rbbt_commands/workflow/server
227
238
  - share/rbbt_commands/workflow/task
228
239
  - share/unicorn.rb
240
+ - test/rbbt/annotations/test_util.rb
229
241
  - test/rbbt/association/test_index.rb
230
242
  - test/rbbt/association/test_item.rb
231
243
  - test/rbbt/persist/test_tsv.rb
@@ -253,6 +265,9 @@ files:
253
265
  - test/rbbt/tsv/test_manipulate.rb
254
266
  - test/rbbt/tsv/test_parallel.rb
255
267
  - test/rbbt/tsv/test_util.rb
268
+ - test/rbbt/util/concurrency/processes/test_socket.rb
269
+ - test/rbbt/util/concurrency/test_processes.rb
270
+ - test/rbbt/util/concurrency/test_threads.rb
256
271
  - test/rbbt/util/simpleopt/test_get.rb
257
272
  - test/rbbt/util/simpleopt/test_parse.rb
258
273
  - test/rbbt/util/simpleopt/test_setup.rb
@@ -260,6 +275,7 @@ files:
260
275
  - test/rbbt/util/test_chain_methods.rb
261
276
  - test/rbbt/util/test_cmd.rb
262
277
  - test/rbbt/util/test_colorize.rb
278
+ - test/rbbt/util/test_concurrency.rb
263
279
  - test/rbbt/util/test_excel2tsv.rb
264
280
  - test/rbbt/util/test_filecache.rb
265
281
  - test/rbbt/util/test_log.rb
@@ -307,11 +323,15 @@ test_files:
307
323
  - test/rbbt/util/test_simpleDSL.rb
308
324
  - test/rbbt/util/test_log.rb
309
325
  - test/rbbt/util/test_open.rb
326
+ - test/rbbt/util/test_concurrency.rb
310
327
  - test/rbbt/util/test_R.rb
311
328
  - test/rbbt/util/test_colorize.rb
312
329
  - test/rbbt/util/test_simpleopt.rb
313
330
  - test/rbbt/util/test_excel2tsv.rb
314
331
  - test/rbbt/util/test_filecache.rb
332
+ - test/rbbt/util/concurrency/test_processes.rb
333
+ - test/rbbt/util/concurrency/test_threads.rb
334
+ - test/rbbt/util/concurrency/processes/test_socket.rb
315
335
  - test/rbbt/util/test_semaphore.rb
316
336
  - test/rbbt/util/test_misc.rb
317
337
  - test/rbbt/util/test_tmpfile.rb
@@ -319,6 +339,7 @@ test_files:
319
339
  - test/rbbt/test_resource.rb
320
340
  - test/rbbt/test_entity.rb
321
341
  - test/rbbt/test_knowledge_base.rb
342
+ - test/rbbt/annotations/test_util.rb
322
343
  - test/rbbt/association/test_index.rb
323
344
  - test/rbbt/association/test_item.rb
324
345
  - test/rbbt/test_tsv.rb