rbbt-util 5.5.49 → 5.5.50

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2MyZWMzOTc4ODRkYTkzZDg1MTY2ZDlhZGMzN2NkMWE5YTJiN2JiZQ==
+    NGM4ZjliYTA2Y2UxYzNiZWY2NWM0N2Q5ZTI3MTFhNGU2ODI2Nzk3ZQ==
   data.tar.gz: !binary |-
-    YjhmN2VjNzljNDMzOTlkNzRiMmNhOWE1ZDM0YmI4ZWIxYTkwNzE4NQ==
+    NmI0MTM4MjRhYTM3NjJmMWRiN2NjOTg3NDNjYTYxYjhjN2IzNDRhOQ==
 SHA512:
   metadata.gz: !binary |-
-    YTVkM2M1MjdkMzJjYmZlMmJiM2NlM2UwYzNjNDg2ZWQxMDkxZjRiMTEwMWZk
-    Yjg5YWMxNGUzN2VlOGU4YmFlMDc5ZGZmZTdlOTRkMmI1NWI2NzQ4OTI0ODZm
-    YWZmMzE1ODJmZWM4MTBlNWE0M2U4YWNkNzI0YWEyYWQ2ZmJmZjI=
+    NTc0MThjZDQ5ZTMyMjM0MjM2ZTFhMDZiYTg5YzU5ZDJmZGE1Y2ExNTZmZWU0
+    M2RkOWI3NTJkNmU3NWVkYzFiZDRiY2M2YTExMTg4MmQ1M2U4N2Y3YWRmN2U4
+    NDFiMjFiNzRhMmJmNzgwOTEyYzM2MDk2YjkxNzIxMTY5YjA0YmY=
   data.tar.gz: !binary |-
-    OGI2OTYyYzRkZmE0Y2ZhMjY2OWFhZDAwNGU0MzVkOTk3MzQwNWY2YzY0ZGM0
-    ZTAxNzEzNzkxODZhYjBiZmMzNDc0MjBmM2RhM2MwMTRlYjYwZDc3NmRkZDE1
-    ODU0MjhiNmM0ZWRlMjc5Mzc4Y2FlMjU2NDczODU2N2Y5YmJlM2Y=
+    M2UyM2MyMmZjODBhZjU4N2MzNzA1MTcyMWZlYTEzYTU3MmVmNWRlZGY5ZmU1
+    MWU1MTIyMzNiNTQyMzBiNDc4OGZjNmE3NDZhNjk3YTk3MTU2ODNmMjY1NmRj
+    NTliMWJiOTBiZDRjYmQwZTg1NWEyZDM3NmZlMzUzZTQ3NzIwNTI=

data/lib/rbbt/resource.rb

@@ -58,27 +58,34 @@ module Resource
   def get_from_server(path, final_path)
     url = File.join(remote_server, '/resource/', self.to_s, 'get_file')
     url << "?" << Misc.hash2GET_params(:file => path, :create => false)
+
     begin
-      response = Net::HTTP.get_response(URI(url))
-
-      case response
-      when Net::HTTPSuccess then
-        #Misc.sensiblewrite(final_path, response.body)
-        Misc.sensiblewrite(final_path){ Net::HTTP.get_response(URI(url)).body }
-      when Net::HTTPRedirection then
-        location = response['location']
-        Log.debug("Feching directory from: #{location}. Into: #{final_path}")
-        FileUtils.mkdir_p final_path unless File.exists? final_path
-        Misc.in_dir final_path do
-          CMD.cmd('tar xvfz -', :in => Open.open(location))
-        end
-      else
-        raise "Response not understood: #{response.inspect}"
+      Net::HTTP.get_response URI(url) do |response|
+          case response
+          when Net::HTTPSuccess, Net::HTTPOK
+
+            Misc.sensiblewrite(final_path) do |file|
+              response.read_body do |chunk|
+                file.write chunk
+              end
+            end
+
+            when Net::HTTPRedirection, Net::HTTPFound
+              location = response['location']
+              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+              FileUtils.mkdir_p final_path unless File.exists? final_path
+              Misc.in_dir final_path do
+                CMD.cmd('tar xvfz -', :in => Open.open(location))
+              end
+          else
+            exit
+            raise "Response not understood: #{response.inspect}"
+          end
       end
-      return true
     rescue
       Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
       Log.error $!.message
+      Log.error $!.backtrace * "\n"
       FileUtils.rm_rf final_path if File.exists? final_path
       return false
     end

data/lib/rbbt/util/cmd.rb

@@ -67,13 +67,12 @@ module CMD
     end
 
     def read(*args)
-      data = original_read(*args)
+      data = original_read(*args) unless self.eof?
 
       self.close if self.eof?
 
       data
     end
-
   end
 
 

data/lib/rbbt/util/misc.rb

@@ -3,6 +3,9 @@ require 'net/smtp'
 require 'digest/md5'
 require 'narray'
 require 'cgi'
+require 'zlib'
+require 'rubygems/package'
+require 'rbbt/util/tar'
 
 class Hash
   def chunked_values_at(keys, max = 5000)
@@ -1017,16 +1020,17 @@ end
     end
   end
 
-  def self.sensiblewrite(path, content = nil)
+  def self.sensiblewrite(path, content = nil, &block)
     return if File.exists? path
     Misc.lock path + '.sensible_write' do
       if not File.exists? path
         begin
-          tmp_path = path + '.tmp'
-          content = yield if block_given?
+          tmp_path = path + '.sensible_write'
           case
+          when block_given?
+            File.open(tmp_path, 'w', &block)
           when String === content
-            File.open(tmp_path, 'w') do |f|  f.write content  end
+            File.open(tmp_path, 'w') do |f| f.write content end
           when (IO === content or StringIO === content)
             File.open(tmp_path, 'w') do |f|  
               while l = content.gets; f.write l; end  

data/lib/rbbt/util/tar.rb

@@ -0,0 +1,126 @@
+require 'rubygems/package'
+require 'zlib'
+require 'fileutils'
+ 
+# SOURCE: https://gist.github.com/sinisterchipmunk/1335041
+# Adapted for Rbbt 
+
+module Misc
+  # Creates a tar file in memory recursively
+  # from the given path.
+  #
+  # Returns a StringIO whose underlying String
+  # is the contents of the tar file.
+  def self.tar(path, tarfile = nil)
+    tarfile ||= StringIO.new("")
+
+    Gem::Package::TarWriter.new(tarfile) do |tar|
+      Dir[File.join(path, "**/*")].each do |file|
+        mode = File.stat(file).mode
+        relative_file = file.sub /^#{Regexp::escape path}\/?/, ''
+
+        if File.directory?(file)
+          tar.mkdir relative_file, mode
+        else
+          tar.add_file relative_file, mode do |tf|
+            File.open(file, "rb") { |f| tf.write f.read }
+          end
+        end
+      end
+    end
+
+    tarfile.rewind
+
+    tarfile
+  end
+
+  def self.tarize(path, gz = nil)
+    gz ||= StringIO.new('wb')
+
+    tar =  Misc.in_dir(path) do self.tar('.') end
+
+    tar.rewind
+
+    string = tar.string
+    ddd string.size
+
+    #chunk_size = 1024
+    #Zlib::GzipWriter.wrap(gz) do |writer|
+    #  while chunk = tar.read(chunk_size) do
+    #    writer.write chunk
+    #  end
+    #  writer.close
+    #end
+    z = Zlib::GzipWriter.new(gz)
+    z.write string
+    z.close
+
+    gz.reopen('read')
+    gz.rewind
+
+
+    gz
+  end
+
+  def self.tarize(path)
+    Misc.in_dir(path) do
+      CMD.cmd("tar cvfz - '.'", :pipe => true)
+    end
+  end
+
+  # gzips the underlying string in the given StringIO,
+  # returning a new StringIO representing the 
+  # compressed file.
+  def self.gzip(tarfile)
+    gz = StringIO.new("")
+    z = Zlib::GzipWriter.new(gz)
+    z.write tarfile.string
+    z.close # this is necessary!
+
+    # z was closed to write the gzip footer, so
+    # now we need a new StringIO
+    StringIO.new gz.string
+  end
+
+  # un-gzips the given IO, returning the
+  # decompressed version as a StringIO
+  def ungzip(tarfile)
+    z = Zlib::GzipReader.new(tarfile)
+    unzipped = StringIO.new(z.read)
+    z.close
+    unzipped
+  end
+
+  # untars the given IO into the specified
+  # directory
+  def untar(io, destination)
+    Gem::Package::TarReader.new io do |tar|
+      tar.each do |tarfile|
+        destination_file = File.join destination, tarfile.full_name
+
+        if tarfile.directory?
+          FileUtils.mkdir_p destination_file
+        else
+          destination_directory = File.dirname(destination_file)
+          FileUtils.mkdir_p destination_directory unless File.directory?(destination_directory)
+          File.open destination_file, "wb" do |f|
+            f.print tarfile.read
+          end
+        end
+      end
+    end
+  end
+end
+
+
+### Usage Example: ###
+#
+# include Util::Tar
+# 
+# io = tar("./Desktop")   # io is a TAR of files
+# gz = gzip(io)           # gz is a TGZ
+# 
+# io = ungzip(gz)         # io is a TAR
+# untar(io, "./untarred") # files are untarred
+#
+

data/share/unicorn.rb

@@ -1,5 +1,6 @@
 worker_processes 4
 timeout 30
+preload_app true
 
 @app_path = Dir.pwd
 @port = "2887"

data/test/rbbt/util/test_misc.rb

@@ -6,11 +6,11 @@ require 'rbbt/entity'
 
 class TestMisc < Test::Unit::TestCase
 
-  def test_humanize
+  def _test_humanize
     assert_equal "mutation_enrichment", Misc.humanize("MutationEnrichment")
   end
 
-  def test_fixutf8
+  def _test_fixutf8
     string = "abc\xffdef"
     string = string.force_encoding("UTF-8") if string.respond_to? :force_encoding
     assert(! string.valid_encoding?) if string.respond_to? :valid_encoding?
@@ -19,37 +19,37 @@ class TestMisc < Test::Unit::TestCase
     assert( Misc.fixutf8(string).valid_encoding) if string.respond_to? :valid_encoding
   end
 
-  def test_colors_for
+  def _test_colors_for
     colors, used = Misc.colors_for([1,2,2,1,2,1,2,2,3,3,2,3,2])
     assert_equal Misc::COLOR_LIST[1], used[2]
   end
 
-  def test_total_length
+  def _test_total_length
     ranges = [(0..100), (50..150), (120..160)]
     ranges = [(0..100), (50..150), (120..160), (51..70)]
     assert_equal 161, Misc.total_length(ranges)
   end
 
-  def test_id_filename?
+  def _test_id_filename?
     TmpFile.with_file("") do |file|
       assert Misc.is_filename?(file)
       assert ! Misc.is_filename?("TEST STRING")
     end
   end
 
-  def test_merge_sorted_arrays
+  def _test_merge_sorted_arrays
     assert_equal [1,2,3,4], Misc.merge_sorted_arrays([1,3], [2,4])
   end
 
-  def test_intersect_sorted_arrays
+  def _test_intersect_sorted_arrays
     assert_equal [2,4], Misc.intersect_sorted_arrays([1,2,3,4], [2,4])
   end
 
-  def test_sorted_array_matches
+  def _test_sorted_array_matches
     assert_equal [1,3], Misc.sorted_array_hits(%w(a b c d e), %w(b d))
   end
 
-  def test_binary_include?
+  def _test_binary_include?
     a = %w(a b c d e).sort
     assert Misc.binary_include?(a, "a")
     assert(!Misc.binary_include?(a, "z"))
@@ -58,24 +58,24 @@ class TestMisc < Test::Unit::TestCase
     assert(Misc.binary_include?(a, "d"))
   end
 
-  def test_process_to_hash
+  def _test_process_to_hash
     list = [1,2,3,4]
     assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
   end
 
-#  def test_pdf2text_example
+#  def _test_pdf2text_example
 #    assert PDF2Text.pdf2text(datafile_test('example.pdf')).read =~ /An Example Paper/i
 #  end
 #
-#  def test_pdf2text_EPAR
+#  def _test_pdf2text_EPAR
 #    assert PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB/document_library/EPAR_-_Scientific_Discussion/human/000402/WC500033103.pdf").read =~ /Tamiflu/i
 #  end
 #
-#  def test_pdf2text_wrong
+#  def _test_pdf2text_wrong
 #    assert_raise CMD::CMDError do PDF2Text.pdf2text("http://www.ema.europa.eu/docs/en_GB#").read end
 #  end
 
-  def test_string2hash
+  def _test_string2hash
     assert(Misc.string2hash("--user-agent=firefox").include? "--user-agent")
     assert_equal(true, Misc.string2hash(":true")[:true])
     assert_equal(true, Misc.string2hash("true")["true"])
@@ -93,17 +93,17 @@ class TestMisc < Test::Unit::TestCase
     end
   end
   
-  def test_named_array
+  def _test_named_array
     a = NamedArray.setup([1,2,3,4], %w(a b c d))
     assert_equal(1, a['a'])
   end
 
-#  def test_path_relative_to
+#  def _test_path_relative_to
 #    assert_equal "test/foo", Misc.path_relative_to('test/test/foo', 'test')
 #  end
 
-#  def test_chunk
-#    test =<<-EOF
+#  def _test_chunk
+#    _test =<<-EOF
 #This is an example file. Entries are separated by Entry
 #-- Entry
 #1
@@ -118,7 +118,7 @@ class TestMisc < Test::Unit::TestCase
 #    assert_equal "1\n2\n3", Misc.chunk(test, /^-- Entry/).first.strip
 #  end
 
-  def test_hash2string
+  def _test_hash2string
     hash = {}
     assert_equal hash, Misc.string2hash(Misc.hash2string(hash))
 
@@ -136,14 +136,14 @@ class TestMisc < Test::Unit::TestCase
  
   end
 
-  def test_merge
+  def _test_merge
     a = [[1],[2]]
     a = NamedArray.setup a, %w(1 2)
     a.merge [3,4]
     assert_equal [1,3], a[0]
   end
 
-  def test_indiferent_hash
+  def _test_indiferent_hash
     a = {:a => 1, "b" => 2}
     a.extend IndiferentHash
 
@@ -153,7 +153,7 @@ class TestMisc < Test::Unit::TestCase
     assert_equal 2, a[:b]
   end
 
-  def test_lockfile
+  def _test_lockfile
 
     TmpFile.with_file do |tmpfile|
       pids = []
@@ -177,7 +177,7 @@ class TestMisc < Test::Unit::TestCase
     end
   end
 
-  def test_positions2hash
+  def _test_positions2hash
     inputs = Misc.positional2hash([:one, :two, :three], 1, :two => 2, :four => 4)
     assert_equal 1, inputs[:one]
     assert_equal 2, inputs[:two]
@@ -185,7 +185,7 @@ class TestMisc < Test::Unit::TestCase
     assert_equal nil, inputs[:four]
   end
 
-  def test_mean
+  def _test_mean
     assert_equal 2, Misc.mean([1,2,3])
     assert_equal 3, Misc.mean([1,2,3,4,5])
   end
@@ -194,32 +194,32 @@ class TestMisc < Test::Unit::TestCase
     assert_equal Math.sqrt(2), Misc.sd([1,3])
   end
 
-  def test_align_small
+  def _test_align_small
     reference = "AABCDEBD"
     sequence  = "ABCD"
     assert_equal '-ABCD---', Misc.fast_align(reference, sequence).last
   end
 
-  def test_align_real
+  def _test_align_real
     reference = "SGNECNKAIDGNKDTFWHTFYGANGDPKPPPHTYTIDMKTTQNVNGLSMLPRQDGNQNGWIGRHEVYLSSDGTNW"
     sequence  = "TYTIDMKTTQNVNGLSML"
     assert_equal "--------------------------------TYTIDMKTTQNVNGLSML-------------------------", Misc.fast_align(reference, sequence).last
   end
 
-  def test_divide
+  def _test_divide
     assert_equal 2, Misc.divide(%w(1 2 3 4 5 6 7 8 9),2).length
   end
 
-  def test_ordered_divide
+  def _test_ordered_divide
     assert_equal 5, Misc.ordered_divide(%w(1 2 3 4 5 6 7 8 9),2).length
   end
 
-  def test_collapse_ranges
+  def _test_collapse_ranges
     ranges = [(0..100), (50..150), (51..61),(200..250), (300..324),(320..350)]
     assert_equal [(0..150),(200..250), (300..350)], Misc.collapse_ranges(ranges)
   end
 
-  def test_humanize
+  def _test_humanize
     str1 = "test_string"
     str2 = "TEST_string"
     str3 = "test"
@@ -229,18 +229,28 @@ class TestMisc < Test::Unit::TestCase
     assert_equal "Test", Misc.humanize(str3)
   end
 
-  def test_snake_case
+  def _test_snake_case
     str1 = "ACRONIMTest"
     str2 = "ACRONIM_test"
     assert_equal "ACRONIM_test", Misc.snake_case(str1)
     assert_equal "ACRONIM_test", Misc.snake_case(str2)
   end
 
-  def test_correct_vcf_mutations
+  def _test_correct_vcf_mutations
     assert_equal [737407, ["-----", "-----G", "-----GTTAAT"]], Misc.correct_vcf_mutation(737406, "GTTAAT", "G,GG,GGTTAAT")
   end
 
-  def test_fingerprint
+  def _test_fingerprint
     puts Misc.fingerprint({:a => 1})
   end
+
+  def test_tarize
+    path = File.expand_path('/home/mvazquezg/git/rbbt-util/lib')
+    stream = Misc.tarize(path)
+    TmpFile.with_file do |res|
+      Misc.in_dir(res) do
+        CMD.cmd("tar xvfz - ", :in => stream)
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.5.49
+  version: 5.5.50
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -163,6 +163,7 @@ files:
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
 - lib/rbbt/util/simpleopt.rb
+- lib/rbbt/util/tar.rb
 - lib/rbbt/util/task/job.rb
 - lib/rbbt/util/tmpfile.rb
 - lib/rbbt/workflow.rb