rbbt-util 5.6.17 → 5.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9cd5bcdf21e90d1e62413b0c5f05c2fe92cd43cc
-  data.tar.gz: 0e5f75492b51edf63e76b32624d4a4dc39092a6e
+  metadata.gz: 2d4025fea0b4bfd41f869ae4282cd268ebe5f876
+  data.tar.gz: 194ead2946a9a3c215a9136bc2b0c615bf73488a
 SHA512:
-  metadata.gz: e2b471f1a98ae3ddfe81fd90675cc05a8d1cbbd24da1ce2d08955f476038681f87bebfd979d9798241c0cee531e28c1c18e4faa868a9d7f7c24dfc2831742bdf
-  data.tar.gz: 4db6e2a91bc5c1cf7370feba6494ffef580f80b4b0d67d7c27be5b8f83a2a9f429e73244001541a40252ca9c7a218eaae8c75a46e76352b20e0deb5f565d145f
+  metadata.gz: 0df26701e5d7e3842a9ddd939c385b7c16884f81eee2fde670d6c7b1c0fe7cc371d05acaa65967eda8716fe7301c5e83e79a8b189e07341597b23052d648c060
+  data.tar.gz: afcd1c9b2c88cb5858278ae50401a5140d729aa2007629f99adff8d3239f3d28d2df4c4791362a450d1892d273bef352941ab14578a6031fc7e43a9f8404a3fd

data/lib/rbbt/persist/tsv.rb

@@ -7,25 +7,25 @@ end
 begin
   require 'rbbt/persist/tsv/lmdb'
 rescue Exception
-  Log.warn "The lmdb gem could not be loaded. Persistance using this engine will fail."
+  Log.debug "The lmdb gem could not be loaded. Persistance using this engine will fail."
 end
 
 begin
   require 'rbbt/persist/tsv/leveldb'
 rescue Exception
-  Log.warn "The LevelDB gem could not be loaded. Persistance using this engine will fail."
+  Log.debug "The LevelDB gem could not be loaded. Persistance using this engine will fail."
 end
 
 begin
   require 'rbbt/persist/tsv/cdb'
 rescue Exception
-  Log.warn "The CDB gem could not be loaded. Persistance using this engine will fail."
+  Log.debug "The CDB gem could not be loaded. Persistance using this engine will fail."
 end
 
 begin
   require 'rbbt/persist/tsv/kyotocabinet'
 rescue Exception
-  Log.warn "The kyotocabinet gem could not be loaded. Persistance using this engine will fail."
+  Log.debug "The kyotocabinet gem could not be loaded. Persistance using this engine will fail."
 end
 
 module Persist

data/lib/rbbt/resource/path.rb

@@ -108,16 +108,21 @@ module Path
     else
       where = where.to_sym
       raise "Did not recognize the 'where' tag: #{where}. Options: #{paths.keys}" unless paths.include? where
-      libdir = where == :lib ? Path.caller_lib_dir(caller_lib) : ""
-      libdir ||= ""
+
+      if where == :lib
+        libdir = Path.caller_lib_dir(caller_lib) || "NOLIBDIR"
+      else
+        libdir = "NOLIBDIR"
+      end
       pwd = FileUtils.pwd
-      self.annotate paths[where].
+      path = paths[where].
         sub('{PKGDIR}', pkgdir).
         sub('{PWD}', pwd).
         sub('{TOPLEVEL}', toplevel).
         sub('{SUBPATH}', subpath).
         sub('{PATH}', self).
         sub('{LIBDIR}', libdir) #, @pkgdir, @resource, @search_paths
+      self.annotate path
     end
   end
 

data/lib/rbbt/resource/util.rb

@@ -8,12 +8,13 @@ module Path
       l =~ /progress-monitor\.rb/ 
     }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
 
-    file = File.expand_path file
     return Path.setup(file) if File.exists? File.join(file, relative_to)
 
     while file != '/'
       dir = File.dirname file
+
       return Path.setup(dir) if File.exists? File.join(dir, relative_to)
+
       file = File.dirname file
     end
 

data/lib/rbbt/tsv.rb

@@ -14,6 +14,7 @@ require 'rbbt/tsv/index'
 require 'rbbt/tsv/attach'
 require 'rbbt/tsv/filter'
 require 'rbbt/tsv/field_index'
+require 'rbbt/tsv/parallel'
 
 module TSV
   class << self
@@ -123,7 +124,7 @@ module TSV
 
     line = parser.rescue_first_line
 
-    if TokyoCabinet::HDB === data and parser.straight and
+    if defined? TokyoCabinet and TokyoCabinet::HDB === data and parser.straight and
       data.close
       begin
         bin = 'tchmgr'

data/lib/rbbt/tsv/manipulate.rb

@@ -172,29 +172,28 @@ module TSV
       next if value.nil?
 
       keys, value = traverser.process(key, value)
+
+      next if keys.nil?
       
       keys = [keys].compact unless Array === keys
 
       # Annotated with Entity and NamedArray
-      if not @unnamed
-        if not traverser.new_field_names.nil? 
-          case type
-          when :double, :list
-            if value.frozen?
-              Log.warn "Value frozen: #{ value }"
-            end
-            if value.nil?
-              nil
-            else
-              NamedArray.setup value, traverser.new_field_names, key, entity_options, entity_templates
-            end
-          when :flat, :single
-            prepare_entity(value, traverser.new_field_names.first, entity_options)
-          end
+      if not @unnamed and not traverser.new_field_names.nil? 
+
+        case type
+        when :double, :list
+          Log.warn "Value frozen: #{ value }" if value.frozen?
+
+          value.nil? ?
+            nil :
+            NamedArray.setup(value, traverser.new_field_names, key, entity_options, entity_templates)
+
+        when :flat, :single
+          prepare_entity(value, traverser.new_field_names.first, entity_options)
         end
+
       end
 
-      next if keys.nil?
 
       if zipped
 
@@ -214,6 +213,7 @@ module TSV
         end
 
       else
+
         keys.each do |key|
           if not @unnamed
             k = Misc.prepare_entity(k, traverser.new_key_field_name, entity_options)
@@ -221,6 +221,7 @@ module TSV
           value.key = key if NamedArray === value
           yield key, value
         end
+
       end
     end
 
@@ -239,6 +240,7 @@ module TSV
         data = {}
       end
 
+      new_key_field_name, new_field_names = nil, nil
       with_unnamed do
         if zipped or (type != :double and type != :flat)
           new_key_field_name, new_field_names = through new_key_field, new_fields, uniq, zipped do |key, value|
@@ -268,23 +270,23 @@ module TSV
             end
           end
         end
+      end
 
-        if real_data and real_data.respond_to? :persistence_path
-          real_data.serializer = type if real_data.respond_to? :serializer
-          real_data.merge!(data)
-          data = real_data
-        end
+      if real_data and real_data.respond_to? :persistence_path
+        real_data.serializer = type if real_data.respond_to? :serializer
+        real_data.merge!(data)
+        data = real_data
+      end
 
-        data.extend TSV unless TSV === data
-        self.annotate(data)
+      data.extend TSV unless TSV === data
+      self.annotate(data)
 
-        data.key_field = new_key_field_name
-        data.fields = new_field_names
-        data.fields.each do |field|
-          data.entity_templates[field] = entity_templates[field] if entity_templates.include? field
-        end
-        data.type = zipped ? :list : type
+      data.key_field = new_key_field_name
+      data.fields = new_field_names
+      data.fields.each do |field|
+        data.entity_templates[field] = entity_templates[field] if entity_templates.include? field
       end
+      data.type = zipped ? :list : type
     end
   end
 

data/lib/rbbt/tsv/parallel.rb

@@ -0,0 +1,47 @@
+module TSV
+
+  def pthrough(num_threads = 100, new_key_field = nil, new_fields = nil, uniq = false, zipped = false)
+    q = Queue.new
+    mutex = Mutex.new
+
+    threads = []
+
+    done = false
+    num_threads.times do |i|
+      threads << Thread.new(Thread.current) do |current|
+        begin
+          loop do
+            p = q.pop
+            p << mutex
+            yield *p
+            next if q.length == 0 and done
+          end
+        rescue Exception
+          current.raise $!
+        end
+      end
+    end
+
+    max = 10_000_000
+    res = through(new_key_field, new_fields, uniq, zipped) do |*p|
+      if q.length >= max
+        Thread.pass
+        q << p
+      end
+      q << p
+    end
+
+    done == true
+
+    Thread.pass while q.length > 0
+
+
+    threads.each{|t| t.kill }
+
+    res
+  end
+
+  def _pthrough(num_threads = 1, new_key_field = nil, new_fields = nil, uniq = false, zipped = false, &block)
+    through(new_key_field, new_fields, uniq, zipped, &block) 
+  end
+end

data/lib/rbbt/util/simpleopt/parse.rb

@@ -15,7 +15,7 @@ module SOPT
     end
 
     while shortcuts.include?(short) and not shortcuts[short] == long
-      while (long.length - current.length > 2) and shortcuts[short].index current * ""
+      while shortcuts[short].index current * ""
         next_letter = chars.shift
         return nil if next_letter.nil?
         current << next_letter
@@ -23,6 +23,8 @@ module SOPT
       short = current * ""
     end
 
+    return nil if shortcuts.include? short
+
     short
   end
 

data/lib/rbbt/workflow.rb

@@ -28,7 +28,10 @@ module Workflow
   def self.load_workflow_file(filename)
     begin
       $LOAD_PATH.unshift(File.join(File.dirname(File.expand_path(filename)), 'lib'))
-      require File.expand_path(filename)
+
+      filename = File.expand_path(filename)
+
+      require filename
       Log.debug{"Workflow loaded from: #{ filename }"}
       return true
     rescue Exception
@@ -80,7 +83,6 @@ module Workflow
   end
 
   def self.require_workflow(wf_name)
-
     # Already loaded
     begin
       workflow = Misc.string2const wf_name

data/lib/rbbt/workflow/accessor.rb

@@ -97,9 +97,9 @@ class Step
   def log(status, message = nil, do_log = true)
 
     if message
-      Log.low "[#{ status }] #{ message }: #{path}"
+      Log.medium "[#{ status }] #{ message }: #{path}"
     else
-      Log.low "[#{ status }]: #{path}"
+      Log.medium "[#{ status }]: #{path}"
     end if do_log
 
     self.status = status

data/test/rbbt/tsv/test_parallel.rb

@@ -0,0 +1,20 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/parallel'
+
+class TestTSVParallel < Test::Unit::TestCase
+
+  def test_pthrough
+    require 'rbbt/sources/organism'
+    tsv = Organism.identifiers("Hsa").tsv :unnamed => true
+
+    h = {}
+    tsv.monitor = true
+    tsv.pthrough do |k,v|
+      h[k] = v.first
+    end
+
+    assert h.size > 0
+  end
+
+end

metadata

@@ -1,99 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.6.17
+  version: 5.7.0
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-02 00:00:00.000000000 Z
+date: 2014-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: progress-monitor
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: progress-monitor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: lockfile
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: lockfile
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: spreadsheet
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: highline
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: highline
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: term-ansicolor
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
   type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez@cnio.es
 executables:
@@ -152,6 +152,7 @@ files:
 - lib/rbbt/tsv/filter.rb
 - lib/rbbt/tsv/index.rb
 - lib/rbbt/tsv/manipulate.rb
+- lib/rbbt/tsv/parallel.rb
 - lib/rbbt/tsv/parser.rb
 - lib/rbbt/tsv/serializers.rb
 - lib/rbbt/tsv/util.rb
@@ -250,6 +251,7 @@ files:
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_util.rb
 - test/rbbt/util/simpleopt/test_get.rb
 - test/rbbt/util/simpleopt/test_parse.rb
@@ -274,39 +276,33 @@ homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.2.2
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
 - test/rbbt/test_workflow.rb
-- test/rbbt/test_association.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_entity.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_fix_width_table.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_cmd.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_parse.rb
 - test/rbbt/util/test_chain_methods.rb
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_log.rb
@@ -319,13 +315,17 @@ test_files:
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/test_association.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_entity.rb
+- test/rbbt/test_knowledge_base.rb
 - test/rbbt/association/test_index.rb
 - test/rbbt/association/test_item.rb
+- test/rbbt/test_tsv.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/workflow/test_step.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/test_annotations.rb
 - test/rbbt/persist/test_tsv.rb
 - test/rbbt/persist/tsv/test_lmdb.rb
 - test/rbbt/persist/tsv/test_kyotocabinet.rb
@@ -333,6 +333,7 @@ test_files:
 - test/rbbt/persist/tsv/test_tokyocabinet.rb
 - test/rbbt/persist/tsv/test_leveldb.rb
 - test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_util.rb
@@ -340,3 +341,5 @@ test_files:
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/test_fix_width_table.rb
+- test/test_helper.rb