rbbt-util 5.14.3 → 5.14.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0861b4fc08580d2d67c2359eb41ba10d8f98fc54
-  data.tar.gz: 20f541a44866d348ebbde761c95b66acec879f5f
+  metadata.gz: de7c9170a1eadbc1ee1117b6f1930ad96d94b63b
+  data.tar.gz: 7763cbe7760e56fc1faeba8d117ec7deddd25f88
 SHA512:
-  metadata.gz: 3ca34de7e7851c59df081c5ca65647341a4688f597c79e14ca2b5ec19d742defbdb0dcf9200eaa48a937b670a5384ac1584e5cf6826414d947a86913c21ec115
-  data.tar.gz: 7815dd4a71975589f9d849950d2e0b4053cce9a65a798b98bf74184a3b9135906194bc752a72b3174d40ab221304dc2a43a5c7cf9f7d4f289b8c57bb4baf39d1
+  metadata.gz: e082be066b8392739566438b2c8ce7c8064aae3a29622702af4e34980fafeb53f566c6f80472d2d892862f84cef03b6138462ebe4f33aa97a9941734b97bbf75
+  data.tar.gz: 71e7400483d8ba1bc92bc804ccd96f9bd2ea0ddd906fa63e7599bfa2cdea3ec433a4a5ca95ccb147d4616e5693c7032d2708dc174ea54f0bee286dbf6ab618a7

data/lib/rbbt/persist.rb

@@ -132,7 +132,10 @@ module Persist
   end
 
   def self.save_file(path, type, content, lockfile = nil)
-    return if content.nil?
+    if content.nil?
+      lockfile.unlock if lockfile and lockfile.locked?
+      return
+    end
 
     case (type || :marshal).to_sym
     when :nil
@@ -292,8 +295,12 @@ module Persist
       retry
     rescue Exception
       Log.medium "Error in persist: #{path}#{Open.exists?(path) ? Log.color(:red, " Erasing") : ""}"
-      Log.exception $!
-      FileUtils.rm path if Open.exists? path 
+
+      begin
+        FileUtils.rm path 
+      rescue
+      end if Open.exists? path 
+
       raise $!
     end
   end

data/lib/rbbt/persist/tsv/adapter.rb

@@ -55,7 +55,7 @@ module Persist
     def write_and_read
       lock_filename = Persist.persistence_path(persistence_path + '.write', {:dir => TSV.lock_dir})
       #mutex.synchronize do
-        Misc.lock(lock_filename, true, :max_age => 600) do
+        Misc.lock(lock_filename) do
           write if closed? or not write?
           res = begin
                   yield

data/lib/rbbt/persist/tsv/cdb.rb

@@ -3,7 +3,7 @@ require 'libcdb'
 module Persist
 
   module CDBAdapter
-    attr_accessor :persistence_path, :closed
+    include Persist::TSVAdapter
 
     def self.open(path, write)
       write = true unless File.exists? path
@@ -19,6 +19,10 @@ module Persist
       database
     end
 
+    def serializer
+      :clean
+    end
+
     def include?(k)
       not write? and super(k) 
     end
@@ -38,9 +42,8 @@ module Persist
     end
 
     def fix_io
+      ddd instance_variable_get(:@io)
       if instance_variable_get(:@io) != persistence_path
-        #close_read if read?
-        #close_write if write?
         instance_variable_set(:@io, File.open(persistence_path))
       end
     end
@@ -129,10 +132,10 @@ module Persist
 
     database = Persist::CDBAdapter.open(path, write)
 
-    unless serializer == :clean
-      TSV.setup database
-      database.serializer = serializer || database.serializer
-    end
+    #unless serializer == :clean
+    #  TSV.setup database
+    #  database.serializer = serializer if serializer
+    #end
 
     database
   end

data/lib/rbbt/persist/tsv/sharder.rb

@@ -124,6 +124,13 @@ module Persist
       ! write?
     end
 
+    def range(*args)
+      databases.values.inject([]) do |acc,database|
+        acc.concat database.range(*args) if TokyoCabinet::BDB === database
+        acc
+      end
+    end
+
     def each
       databases.values.each do |database|
         database.each do |k,v|

data/lib/rbbt/tsv/manipulate.rb

@@ -417,7 +417,7 @@ module TSV
       case
       when (Array === method and (key == :key or key_field == key))
         with_unnamed do
-          TSV.traverse(Annotated.purge(method).uniq, :bar => true){|key| 
+          Annotated.purge(method).each{|key| 
             new[key] = self[key] if invert ^ (self.include? key)
           }
         end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -6,7 +6,7 @@ module TSV
       nil
     when (defined? Step and Step)
       obj.result
-    when IO, File
+    when IO, File, Zlib::GzipReader, Bgzf
       obj
     when TSV::Dumper
       obj.stream
@@ -24,14 +24,19 @@ module TSV
         else
           nil
         end
+      when TSV
+        obj.length
       when Array, Hash
         obj.size
       when File
+        return nil if Misc.gzip?(file) or Misc.bgzip?(file)
         CMD.cmd("wc -l '#{obj.filename}'").read.to_i
       when Path
+        return nil if Misc.gzip?(file) or Misc.bgzip?(file)
         CMD.cmd("wc -l '#{obj.find}'").read.to_i
       when String
         if File.exists? obj
+          return nil if Misc.gzip?(file) or Misc.bgzip?(file)
           CMD.cmd("wc -l '#{obj}'").read.to_i
         else
           nil
@@ -144,6 +149,12 @@ module TSV
 
     if callback
       while line = io.gets
+        if line[-1] != "\n"
+          while c = io.getc
+            line << c
+            break if c=="\n"
+          end
+        end
         begin
           callback.call yield line.strip
         ensure
@@ -213,7 +224,7 @@ module TSV
         else
           obj.traverse(options, &block)
         end
-      when IO, File, StringIO
+      when IO, File, Zlib::GzipReader, Bgzf, StringIO
         begin
           if options[:type] == :array
             traverse_io_array(obj, options, &block)

data/lib/rbbt/tsv/parser.rb

@@ -551,7 +551,6 @@ module TSV
             raise $!
           end
         end
-
       ensure
         stream.close unless stream.closed?
         stream.join if stream.respond_to? :join

data/lib/rbbt/tsv/stream.rb

@@ -112,8 +112,10 @@ module TSV
         last_min = nil
         while lines.compact.any?
           min = keys.compact.sort.first
+          break if min.nil?
           str = []
           keys.each_with_index do |key,i|
+
             case key
             when min
               str << parts[i] * sep

data/lib/rbbt/tsv/util.rb

@@ -82,6 +82,10 @@ module TSV
 
   def self.get_stream(file, open_options = {})
     case file
+    when Zlib::GzipReader
+      file
+    when (defined? Bgzf and Bgzf)
+      file
     when TSV::Parser
       file.stream
     when Path

data/lib/rbbt/util/R.rb

@@ -61,6 +61,10 @@ source('#{UTIL}');
       "'#{ object }'"
     when Fixnum, Float
       object
+    when TrueClass
+      "TRUE"
+    when FalseClass
+      "FALSE"
     when Array
       "c(#{object.collect{|e| ruby2R(e) } * ", "})"
     else

data/lib/rbbt/util/log/progress.rb

@@ -24,6 +24,7 @@ module Log
     end
 
     def tick(step = nil)
+      return if ENV["RBBT_NO_PROGRESS"] == "true"
       @ticks += 1
 
       time = Time.now

data/lib/rbbt/util/misc/bgzf.rb

@@ -0,0 +1,154 @@
+
+module Bgzf
+  attr_accessor :data_offset, :compressed_stream, :block_cache_size
+
+  def self.setup(compressed_stream)
+    require 'bio-bgzf'
+    reader = Bio::BGZF::Reader.new(compressed_stream)
+    reader.extend Bgzf
+    reader.compressed_stream = compressed_stream
+    reader.data_offset = 0
+    reader
+  end
+
+  def filename
+    @filename ||= begin
+                  compressed_stream.respond_to?(:filename) ? compressed_stream.filename : nil
+                end
+  end
+
+  def closed?
+    @compressed_stream.closed?
+  end
+
+  def close
+    @compressed_stream.close unless @compressed_stream.closed?
+    @access.clear
+    @blocks.clear
+  end
+
+  def seek(off)
+    @data_offset = off
+  end
+
+  def _index
+    @_index ||= begin
+                 index = []
+                 pos = 0
+                 while true do
+                   blockdata_offset = tell
+                   block = read_block
+                   break unless block
+                   index << [pos, blockdata_offset]
+                   pos += block.length
+                 end
+                 @block_cache_size = Math.log(index.length).to_i + 1
+                 index
+               end
+  end
+
+  def init
+    _index
+  end
+
+  def closest_page(pos)
+    upper = _index.size - 1
+    lower = 0
+    @_index_pos ||= _index.collect{|v| v.first }
+
+    return -1 if upper < lower
+
+    while(upper >= lower) do
+      idx = (idx.nil? and @last_idx) ? @last_idx : (lower + (upper - lower) / 2)
+      pos_idx = @_index_pos[idx]
+
+      case pos <=> pos_idx
+      when 0
+        break
+      when -1
+        upper = idx - 1
+      when 1
+        lower = idx + 1
+      end
+    end
+
+    @last_idx = idx
+
+    if pos_idx > pos
+      idx = idx - 1
+    end
+
+
+    idx.to_i
+  end
+
+  def block_offset
+    pos = data_offset
+    i = closest_page(data_offset)
+    page = _index[i][1]
+    offset = pos - _index[i][0]
+    [page, offset]
+  end
+
+  def _purge_cache
+    if @blocks.length > @block_cache_size
+      @access.uniq!
+      oldest = @access.last
+      @blocks.delete oldest
+    end
+  end
+
+  def _get_block(vo)
+    @blocks ||= {}
+    @access ||= []
+    @access << vo
+    if @blocks.include? vo
+      @blocks[vo]
+    else
+      _purge_cache
+      @blocks[vo] ||= read_block_at vo
+    end
+  end
+
+  def get_block
+    block_vo, offset = block_offset
+    block = _get_block block_vo
+    block[offset..-1]
+  end
+
+  def read(size)
+    block = get_block 
+    return "" if block.nil? or block.empty?
+    len = block.length
+    if len >= size
+      @data_offset += size
+      return block[0..size-1]
+    else
+      @data_offset += len
+      str = block
+      str << read(size - len)
+      str
+    end
+  end
+
+  def gets
+    str = nil
+    current = @data_offset
+    while true
+      block = read(1024)
+      break if block.empty?
+      str = "" if str.nil?
+      if i = block.index("\n")
+        str << block[0..i]
+        break
+      else
+        str << block
+      end
+    end
+    return nil if str.nil?
+
+    @data_offset = current + str.length
+
+    str
+  end
+end

data/lib/rbbt/util/misc/pipes.rb

@@ -214,13 +214,23 @@ module Misc
 
   def self.sensiblewrite(path, content = nil, options = {}, &block)
     force = Misc.process_options options, :force
-    return if Open.exists? path and not force
+
+    if Open.exists? path and not force
+      Misc.consume_stream content 
+      return
+    end
+
     lock_options = Misc.pull_keys options.dup, :lock
     lock_options = lock_options[:lock] if Hash === lock_options[:lock]
     tmp_path = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_dir})
     tmp_path_lock = Persist.persistence_path(path, {:dir => Misc.sensiblewrite_lock_dir})
     Misc.lock tmp_path_lock, lock_options do
-      return if Open.exists? path and not force
+
+      if Open.exists? path and not force
+        Misc.consume_stream content 
+        return
+      end
+
       FileUtils.mkdir_p File.dirname(tmp_path) unless File.directory? File.dirname(tmp_path)
       FileUtils.rm_f tmp_path if File.exists? tmp_path
       begin

data/lib/rbbt/util/open.rb

@@ -1,6 +1,7 @@
 require 'rbbt/util/cmd'
 require 'rbbt/util/tmpfile'
 require 'rbbt/util/misc'
+require 'rbbt/util/misc/bgzf'
 
 require 'zlib'
 
@@ -285,13 +286,14 @@ module Open
 
 
   # Decompression
+  
+  def self.bgunzip(stream)
+    Bgzf.setup stream
+  end
    
   def self.gunzip(stream)
-    if String === stream
-      Zlib::Inflate.inflate(stream)
-    else
-      CMD.cmd("gunzip", :pipe => true, :in => stream, :post => proc{stream.force_close if stream.respond_to? :force_close})
-    end
+    #Zlib::GzipReader.new(stream)
+    CMD.cmd('zcat', :in => stream, :pipe => true)
   end
 
   def self.unzip(stream)
@@ -310,6 +312,10 @@ module Open
     !! (file =~ /\.gz$/)
   end
 
+  def self.bgzip?(file)
+    !! (file =~ /\.bgz$/)
+  end
+
   def self.zip?(file)
     !! (file =~ /\.zip/)
   end
@@ -375,6 +381,7 @@ module Open
          end
     io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
     io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
+    io = bgunzip(io) if ((String === url and bgzip?(url)) and not options[:noz]) or options[:bgzip]
 
     if block_given?
       begin

data/lib/rbbt/util/semaphore.rb

@@ -68,11 +68,11 @@ void post_semaphore(char* name){
     end
 
     begin
-      Log.warn "Creating semaphore (#{ size }): #{file}"
+      Log.low "Creating semaphore (#{ size }): #{file}"
       RbbtSemaphore.create_semaphore(file, size)
       yield file
     ensure
-      Log.error "Removing semaphore #{ file }"
+      Log.low "Removing semaphore #{ file }"
       RbbtSemaphore.delete_semaphore(file)
     end
   end

data/lib/rbbt/workflow/accessor.rb

@@ -386,6 +386,7 @@ module Workflow
   def task_info(name)
     name = name.to_sym
     task = tasks[name]
+    raise "No '#{name}' task in '#{self.to_s}' Workflow" if task.nil?
     description = task.description
     result_description = task.result_description
     result_type = task.result_type
@@ -503,7 +504,7 @@ module Workflow
       when Symbol
         job(dependency, jobname, inputs)
       when Proc
-        dependency.call jobname, inputs
+        dependency.call jobname, inputs, real_dependencies
       end
     end
     real_dependencies.flatten.compact

data/test/rbbt/persist/test_tsv.rb

@@ -21,9 +21,11 @@ class TestPersistTSV < Test::Unit::TestCase
     Log.info "Testing #{ Term::ANSIColor.red(engine) }"
     TmpFile.with_file nil, false do |tmp_file|
       db= nil
+
       Misc.benchmark(1, "Build database with #{MAX - 2} entries") do
         db = TSV.open(file, :fields => [1], :persist => true, :persist_engine => engine, :persist_dir => tmp_file, :type => :single, :unnamed => true)
       end
+
       _test = db.keys.sort{rand}[1..100000]
       Misc.benchmark(5, "Access #{test.length} random entries") do
         _test.each do |k| db[k] end
@@ -36,32 +38,32 @@ class TestPersistTSV < Test::Unit::TestCase
     end
   end
 
-  def _test_benchmark_tch
+  def __test_benchmark_tch
     engine = "HDB"
     run_bechmark(tsv_path, engine)
   end
 
-  def _test_benchmark_tcb
+  def __test_benchmark_tcb
     engine = "BDB"
     run_bechmark(tsv_path, engine)
   end
 
-  def _test_benchmark_kch
+  def __test_benchmark_kch
     engine = "kch"
     run_bechmark(tsv_path, engine)
   end
 
-  def _test_benchmark_kcb
+  def __test_benchmark_kcb
     engine = "kct"
     run_bechmark(tsv_path, engine)
   end
 
-  def _test_benchmark_cdb
+  def __test_benchmark_cdb
     engine = "CDB"
     run_bechmark(tsv_path, engine)
   end
 
-  def _test_benchmark_leveldb
+  def __test_benchmark_leveldb
     engine = "LevelDB"
     run_bechmark(tsv_path, engine)
   end

data/test/rbbt/test_packed_index.rb

@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'test_helper.r
 require 'rbbt/packed_index'
 
 class TestPackedIndex < Test::Unit::TestCase
-  def test_index
+  def _test_index
 
     TmpFile.with_file do |tmpfile|
       pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
@@ -12,7 +12,7 @@ class TestPackedIndex < Test::Unit::TestCase
       pi << nil
       pi << nil
       pi.close
-      pi = PackedIndex.new tmpfile, false
+      pi = PackedIndex.new(tmpfile, false)
       Misc.benchmark(1000) do
       100.times do |i|
         assert_equal i, pi[i][0] 
@@ -21,7 +21,47 @@ class TestPackedIndex < Test::Unit::TestCase
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
+    end
+  end
+
+  def test_bgzip
+
+    size = 1000000
+    density = 0.1
+
+    access = []
+    (size * density).to_i.times do
+      access << rand(size-1) + 1
+    end
+    access.sort!
+    access.uniq!
+
+    TmpFile.with_file do |tmpfile|
+      pi = PackedIndex.new tmpfile, true, %w(i i 23s f f f f f)
+      size.times do |i|
+        pi << [i, i+2, i.to_s * 10, rand, rand, rand, rand, rand]
+      end
+      pi << nil
+      pi << nil
+      pi.close
 
+      pi = PackedIndex.new(tmpfile, false)
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
+
+      `bgzip #{tmpfile} `
+      `mv #{tmpfile}.gz #{tmpfile}.bgz`
+
+      pi = PackedIndex.new(tmpfile + '.bgz', false)
+      pi[0]
+      Misc.benchmark do
+        access.each do |point|
+          assert_equal point+2, pi[point][1]
+        end
+      end
     end
   end
 end

data/test/rbbt/util/misc/test_bgzf.rb

@@ -0,0 +1,39 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/bgzf'
+
+class TestBgzf < Test::Unit::TestCase
+  def _test_Bgzf
+    content = "1234567890" * 1000000
+    TmpFile.with_file(content) do |file|
+      compressed = file + '.gz'
+      `bgzip #{file} -c > #{compressed}`
+      stream = Bgzf.setup File.open(compressed)
+      stream.seek 500003
+      assert_equal "4567", stream.read(4)
+      assert_equal "89", stream.read(2)
+    end
+  end
+
+  def test_tsv
+    require 'rbbt/tsv'
+    TmpFile.with_file(datafile_test(:identifiers).read) do |file|
+      Misc.benchmark do
+        tsv = TSV.open(Open.open(file))
+      end
+      compressed = file + '.bgz'
+
+      `bgzip #{file} -c > #{compressed}`
+      stream = Bgzf.setup File.open(compressed)
+      Misc.benchmark do
+        tsv = TSV.open(stream)
+      end
+
+      `gzip #{file}`
+      stream = Open.open(file + '.gz')
+      Misc.benchmark do
+        tsv = TSV.open(stream)
+      end
+    end
+  end
+end
+

data/test/rbbt/util/test_open.rb

@@ -7,11 +7,11 @@ require 'iconv'
 
 class TestOpen < Test::Unit::TestCase
 
-  def test_wget
+  def _test_wget
     assert(Misc.fixutf8(Open.wget('http://google.com', :quiet => true).read) =~ /html/)
   end
 
-  def test_nice
+  def _test_nice
     nice =  0.5
 
     t = Time.now
@@ -24,16 +24,16 @@ class TestOpen < Test::Unit::TestCase
     assert(Time.now - t + 0.5 >= nice)
   end
 
-  def test_remote?
+  def _test_remote?
     assert(Open.remote?('http://google.com'))
     assert(! Open.remote?('~/.bashrc'))
   end
 
-  def test_open
+  def _test_open
     assert(Open.read('http://google.com', :quiet => true) =~ /html/)
   end
 
-  def test_read
+  def _test_read
     content =<<-EOF
 1
 2
@@ -48,7 +48,7 @@ class TestOpen < Test::Unit::TestCase
     end
   end
 
-  def test_read_grep
+  def _test_read_grep
     content =<<-EOF
 1
 2
@@ -68,7 +68,7 @@ class TestOpen < Test::Unit::TestCase
     end
   end
 
-  def test_read_grep_invert
+  def _test_read_grep_invert
     content =<<-EOF
 1
 2
@@ -98,13 +98,14 @@ class TestOpen < Test::Unit::TestCase
 4
     EOF
     TmpFile.with_file(content) do |file|
-      `gzip #{file}`
+      `bgzip #{file}`
       assert_equal(content, Open.read(file + '.gz'))
+      puts content
       FileUtils.rm file + '.gz'
     end
   end
 
-  def test_repo_dir
+  def _test_repo_dir
     file1 = "TEST"
     file2 = "TEST" * 1000
     TmpFile.with_file do |tmpdir|
@@ -132,7 +133,7 @@ class TestOpen < Test::Unit::TestCase
     end
   end
   
-  def test_repo_dir
+  def _test_repo_dir
     TmpFile.with_file do |tmpdir|
       tmpdir = "/home/mvazquezg/tmp/repo_dir"
       repo = File.join(tmpdir, 'repo')

data/test/test_helper.rb

@@ -28,6 +28,6 @@ class Test::Unit::TestCase
   end
 
   def datafile_test(file)
-    Path.setup(File.join(File.dirname(__FILE__), 'data', file))
+    Path.setup(File.join(File.dirname(__FILE__), 'data', file.to_s))
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.3
+  version: 5.14.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-23 00:00:00.000000000 Z
+date: 2014-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -185,6 +185,7 @@ files:
 - lib/rbbt/util/log/progress/report.rb
 - lib/rbbt/util/log/progress/util.rb
 - lib/rbbt/util/misc.rb
+- lib/rbbt/util/misc/bgzf.rb
 - lib/rbbt/util/misc/concurrent_stream.rb
 - lib/rbbt/util/misc/development.rb
 - lib/rbbt/util/misc/exceptions.rb
@@ -320,6 +321,7 @@ files:
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/simpleopt/test_get.rb
@@ -379,6 +381,7 @@ test_files:
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_open.rb
 - test/rbbt/util/misc/test_lock.rb
+- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_R.rb