rbbt-util 5.14.16 → 5.14.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70d99fc7279b9731d3f52b47a88b98e754f50376
-  data.tar.gz: a44903e02e4cd08f4780b7a64526f14335fcd661
+  metadata.gz: 5c5725c2ab4aa762ca676d5d4e18f17dee2e8e92
+  data.tar.gz: 088c84611fccd379434220853ce601256540b9cb
 SHA512:
-  metadata.gz: 8f6e582804b9c76c255ef1a7acc1ab8bf261e75a239c6b51f85de1e37baea9e0bd688da035e9999963a8d744e8f327039af4c0fe20e8b57e3f965097be418b7c
-  data.tar.gz: e655675f21100fcb903238b74c0e721743d9a78ae2bd7f17e1238ac1de9439b293b712760d781ba05aafd2da601f9b874e64da14848d373c023f0c817ab66025
+  metadata.gz: 4aa7267702a7587187a85c30fb4bf646efb298ff8d92d2f9aa23daeb58e19ce89dddd470f9d36fcb4258f5bda0e37a6d6bb98327c5308f33b12e79dcaa935e72
+  data.tar.gz: 60796a47e6344334dc576c850df51d405796d8e98f881fcf487f53a23eeaa6db54e0f79207682c28f59b8b694eab063dfbe1395ec114167e3cca9c70b5984116

data/bin/rbbt

@@ -206,7 +206,6 @@ rescue ParameterException
   puts
   exit -1
 rescue SystemExit
-  Log.error "Command error" unless $?.success?
 rescue Exception
   Log.exception $!
 ensure

data/etc/app.d/base.rb

@@ -23,7 +23,9 @@ use Rack::Session::Cookie, :key => 'rack.session',
 
 
 #{{{ DIRECTORIES
-local_var = Rbbt.var#.find(:current)
+global_var = Rbbt.var
+local_var = Rbbt.var.app[$class_name]
+
 set :cache_dir           , local_var.sinatra.cache.find
 set :persist_dir         , local_var.sinatra.cache.persistence.find
 set :persist_options     , {:persist => true, :persist_dir => :persist_dir}

data/lib/rbbt/annotations/util.rb

@@ -50,10 +50,13 @@ module Annotated
     info = {}
     fields.each_with_index do |field,i|
       next if field == "literal"
-      if field == "JSON"
+      case field
+      when "JSON"
         JSON.parse(values[i]).each do |key, value|
           info[key.to_sym] = value
         end
+      when nil
+        next
       else
         info[field.to_sym] = resolve_array(values[i])
       end

data/lib/rbbt/association/index.rb

@@ -74,7 +74,7 @@ module Association
     end
 
     def subset(source, target)
-      return [] if source.nil? or target.nil?
+      return [] if source.nil? or target.nil? or source.empty? or target.empty?
 
       if source == :all or source == "all"
         if target == :all or target == "all"

data/lib/rbbt/resource/path.rb

@@ -40,7 +40,7 @@ module Path
 
   def glob(pattern = '*')
     if self.include? "*"
-      Dir.glob(self).collect{|f| Path.setup(f, self.resource, self.pkgdir)}
+      self.glob_all
     else
       return [] unless self.exists? 
       exp = File.join(self.find, pattern)
@@ -156,6 +156,14 @@ module Path
       compact.select{|file| file.exists? }.uniq
   end
 
+  def glob_all(caller_lib = nil, search_paths = nil)
+    search_paths ||= self.search_paths || SEARCH_PATHS
+    search_paths = search_paths.dup
+
+    search_paths.keys.
+      collect{|where| Dir.glob(find(where, Path.caller_lib_dir, search_paths))}.
+      compact.flatten.collect{|path| Path.setup(path, self.resource, self.pkgdir)}
+  end
   #{{{ Methods
 
   def in_dir?(dir)

data/lib/rbbt/tsv/stream.rb

@@ -20,8 +20,6 @@ module TSV
     options = Misc.add_defaults options, :sep => "\t", :sort => true
     sort, sep, preamble = Misc.process_options options, :sort, :sep, :preamble
 
-
-
     out = Misc.open_pipe do |sin|
 
       streams = streams.collect do |stream|
@@ -72,6 +70,7 @@ module TSV
       key_field = key_fields.compact.first
       fields = fields.compact.flatten
       options = options.merge(input_options.first)
+      options[:type] = :list if options[:type] == :single
 
       preamble_txt = case preamble
                      when TrueClass
@@ -115,7 +114,6 @@ module TSV
           break if min.nil?
           str = []
           keys.each_with_index do |key,i|
-
             case key
             when min
               str << parts[i] * sep

data/lib/rbbt/util/misc/bgzf.rb

@@ -23,8 +23,8 @@ module Bgzf
 
   def close
     @compressed_stream.close unless @compressed_stream.closed?
-    @access.clear
-    @blocks.clear
+    @access.clear if @access
+    @blocks.clear if @blocks
   end
 
   def seek(off)

data/lib/rbbt/util/misc/development.rb

@@ -275,7 +275,7 @@ module Misc
     num = :current if num.nil?
     cpus = case num
            when :current
-            4 
+            10
            when String
              num.to_i
            when Integer
@@ -285,10 +285,18 @@ module Misc
                32000 / num
              end
            end
-    file = caller.first + rand(1000000).to_s if file.nil?
+
+    #file = caller.first + rand(1000000).to_s if file.nil?
     index = (0..elems.length-1).to_a.collect{|v| v.to_s }
-    RbbtSemaphore.fork_each_on_semaphore index, cpus, file do |i|
-      yield elems[i.to_i]
+    TSV.traverse index, :cpus => cpus, :bar => "Bootstrap in #{ cpus } cpus: #{ Misc.fingerprint elems }", :into => Set.new do |pos|
+      elem = elems[pos.to_i]
+      elems.annotate elem if elems.respond_to? :annotate
+      begin
+        yield elem
+      rescue Interrupt
+        Log.warn "Process #{Process.pid} was aborted"
+      end
+      nil
     end
   end
 end

data/lib/rbbt/util/semaphore.rb

@@ -78,21 +78,17 @@ void post_semaphore(char* name){
   end
 
   def self.fork_each_on_semaphore(elems, size, file = nil)
-    with_semaphore(size, file) do |file|
 
-      TSV.traverse elems, :cpus => size*2, :bar => "Fork each on semaphore: #{ file }" do |elem|
-        elems.annotate elem if elems.respond_to? :annotate
-        begin
-          RbbtSemaphore.synchronize(file) do
-            yield elem
-          end
-        rescue Interrupt
-          Log.warn "Process #{Process.pid} was aborted"
-        end
+    TSV.traverse elems, :cpus => size, :bar => "Fork each on semaphore: #{ Misc.fingerprint elems }", :into => Set.new do |elem|
+      elems.annotate elem if elems.respond_to? :annotate
+      begin
+        yield elem
+      rescue Interrupt
+        Log.warn "Process #{Process.pid} was aborted"
       end
-
+      nil
     end
-    true
+    nil
   end
 
   def self.thread_each_on_semaphore(elems, size)

data/lib/rbbt/workflow/accessor.rb

@@ -500,7 +500,7 @@ module Workflow
         options.each{|i,v|
           case v
           when Symbol
-            rec_dependency = real_dependencies.collect{|d| [d, d.dependencies].flatten}.flatten.select{|d| d.task.name == v }.first
+            rec_dependency = real_dependencies.collect{|d| [d, d.rec_dependencies].compact.flatten}.flatten.select{|d| d.task.name == v }.first
             if (dependency.first.tasks[dependency[1]].input_options[i] || {})[:stream]
               inputs[i] = rec_dependency.run(true).grace.join.path 
             else

data/share/rbbt_commands/tsv/assemble_pdf_table

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Assemble the TSV from a table extracted from a PDF
+
+$ rbbt tsv assemble_pdf_table file.txt
+
+When extracting tables from PDF they are often laid out one column at a time, divided by pages.
+This command takes a file with the following structure:
+
+1 A few lines containing table headers, one per line
+2 A group of lines containing the values for the first column of the first page, ending in an empty line
+3 More groups of lines corresponding to other columns
+4 Repetitions of 2 and 3 for more pages
+
+This script will take care of matching the columns read with the headers specified
+
+-h--help Help
+EOF
+
+SOPT.usage if options[:help]
+
+file = ARGV.shift
+
+file = STDIN if file == '-' or file.nil?
+
+
+txt = Misc.fixutf8(TSV.get_stream(file).read)
+
+header, _sep, rest = txt.strip.partition("\n\n")
+fields = header.split("\n")
+num_columns = fields.length
+
+columns = {}
+
+num_columns.times do |i|
+  columns[i] = []
+end
+
+lines = rest.split("\n")
+
+while lines and lines.any?
+  first_block = lines[0..lines.index("")-1]
+  block_size = first_block.length
+  lines = lines[block_size+1..-1]
+  columns[0] << first_block
+  (1..num_columns-1).each do |pos|
+    block = lines[0..block_size-1]
+    lines = lines[block_size+1..-1]
+    columns[pos] << block
+  end
+end
+
+full_columns = []
+num_columns.times do |i|
+  column = columns[i]
+  full_columns << column.flatten
+end
+
+puts "#" << fields * "\t"
+Misc.zip_fields(full_columns).zip do |values|
+  puts values * "\t"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.16
+  version: 5.14.17
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-15 00:00:00.000000000 Z
+date: 2014-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -257,6 +257,7 @@ files:
 - share/rbbt_commands/system/purge
 - share/rbbt_commands/system/report
 - share/rbbt_commands/system/status
+- share/rbbt_commands/tsv/assemble_pdf_table
 - share/rbbt_commands/tsv/attach
 - share/rbbt_commands/tsv/change_id
 - share/rbbt_commands/tsv/get