rbbt-util 5.18.1 → 5.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f4bb8ad71b9080b39bc7217d9f4a5dc0da2719d
-  data.tar.gz: 94b7dfcbd524c888c73f945cf95e223b318205a7
+  metadata.gz: 9bf01122d1f1460989b9d2527d19cd87942f3d07
+  data.tar.gz: e1575d23a2667e28938be48e396e6d7c4f605c45
 SHA512:
-  metadata.gz: 17e4da0d2455c8c93de87d51b4166b2156552611c3bafc17ce561fcb6d278e3ad16df23ece0727fea9a783c66d97b528e889246cd5da51a183ba368ba8c7ecff
-  data.tar.gz: 2c981aee80405b0d9794c4568114da6c3d2916f1fac755357f081566ad9d40c17346d5fe14c3388cb66f3a490ede94ab261bb1cb5e2d79cc0f51584d8fbd7b23
+  metadata.gz: e327f95dcdd0f274b53a1001fe65ef4ccd0411a3cc1ab852b70f0e7e609d9835476768d4873dde308080871e8ebcc1d4bcb406f0ac93b58ca815a85c2c10aa0f
+  data.tar.gz: 9d0616c4bd35dc136da7019ffd913b6b08e1f76a36c9e6bca096b2491bdafb171dba9031abedc4c67782e8f8b81f77dd323a4966fb829f62f57d760aa47a95f4

data/bin/rbbt

@@ -43,6 +43,7 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --nocolor               #{Log.color :yellow, "Disable colored output"}
 --nobar                 #{Log.color :yellow, "Disable progress report"}
 --nostream              #{Log.color :yellow, "Disable persistance/job streaming"}
+--update_persist        #{Log.color :yellow, "Update persisted TSV files if source has been updated"}
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
 --dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
 -nolock--no_lock_id            #{Log.color :yellow, "Do not track lockfiles with ids (prevent stale file handlers for high-througput and high-concurrency)"}
@@ -54,6 +55,7 @@ locate = options.delete :locate_file
 if options[:log_file]
   Log.logfile(options[:log_file])
 end
+
 if options[:log]
   Log.severity = options[:log].to_i
 else
@@ -76,6 +78,10 @@ if mem_dump = options.delete(:dump_mem)
 end
 
 
+if options.delete(:update_persist)
+  ENV["RBBT_UPDATE_TSV_PERSIST"] = "true"
+end
+
 if options.delete :nostream
   ENV["RBBT_NO_STREAM"] = "true"
 end

data/lib/rbbt/persist.rb

@@ -27,6 +27,8 @@ module Persist
 
   def self.newer?(path, file)
     return true if not Open.exists? file
+    path = path.find if Path === path
+    file = file.find if Path === file
     return File.mtime(path) - File.mtime(file) if File.mtime(path) < File.mtime(file)
     return false
   end
@@ -82,7 +84,7 @@ module Persist
     options_md5 = Misc.hash2md5 clean_options
     filename  << ":" << options_md5 unless options_md5.empty?
 
-    persistence_dir[filename].find
+    persistence_dir[filename]
   end
 
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
@@ -333,10 +335,17 @@ module Persist
       other_options = Misc.process_options persist_options, :other
       path = persistence_path(name, persist_options, other_options || {})
 
+      if ENV["RBBT_UPDATE_TSV_PERSIST"] == 'true' and name and Open.exists?(name)
+        persist_options[:check] ||= []
+        persist_options[:check] << name
+      else
+        check_options = {}
+      end
+
       case 
       when type.to_sym == :memory
         repo = persist_options[:repo] || Persist::MEMORY
-        repo[path] ||= yield
+        repo[path.find] ||= yield
 
       when (type.to_sym == :annotations and persist_options.include? :annotation_repo)
 
@@ -419,7 +428,7 @@ module Persist
         end
 
       else
-        persist_file(path, type, persist_options, &block)
+        persist_file(path.find, type, persist_options, &block)
       end
 
     end

data/lib/rbbt/persist/tsv.rb

@@ -98,9 +98,14 @@ module Persist
 
     path = persistence_path(filename, persist_options, options)
 
-    lock_filename = Persist.persistence_path(path, {:dir => TSV.lock_dir})
+    if ENV["RBBT_UPDATE_TSV_PERSIST"] == 'true' and filename
+      check_options = {:check => [filename]}
+    else
+      check_options = {}
+    end
 
-    if is_persisted?(path) and not persist_options[:update]
+    if is_persisted?(path, check_options) and not persist_options[:update]
+      path = path.find if Path === path
       Log.debug "TSV persistence up-to-date: #{ path }"
       if persist_options[:shard_function]
         return open_sharder(path, false, nil, persist_options[:engine], persist_options, &persist_options[:shard_function]) 
@@ -109,9 +114,11 @@ module Persist
       end
     end
 
+    lock_filename = Persist.persistence_path(path, {:dir => TSV.lock_dir})
     Misc.lock lock_filename do
       begin
-        if is_persisted?(path) and not persist_options[:update]
+        if is_persisted?(path, check_options) and not persist_options[:update]
+          path = path.find if Path === path
           Log.debug "TSV persistence (suddenly) up-to-date: #{ path }"
 
           if persist_options[:shard_function]
@@ -120,6 +127,7 @@ module Persist
             return open_database(path, false, nil, persist_options[:engine] || TokyoCabinet::HDB, persist_options) 
           end
         end
+        path = path.find if Path === path
 
         FileUtils.rm_rf path if File.exists? path
 

data/lib/rbbt/resource/path.rb

@@ -120,6 +120,7 @@ module Path
                          path = find(w, caller_lib, paths)
                          return path if File.exists? path
                        end
+
                        if paths.include? :default
                          find((paths[:default] || :user), caller_lib, paths)
                        else

data/lib/rbbt/tsv/accessor.rb

@@ -415,12 +415,14 @@ module TSV
         sorted = elems.sort do |a, b| 
           a_value = a.last
           b_value = b.last
+          a_empty = a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?)
+          b_empty = b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?)
           case
-          when ((a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?)) and (b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?)))
+          when (a_empty and b_empty)
             0
-          when (a_value.nil? or (a_value.respond_to?(:empty?) and a_value.empty?))
+          when a_empty
             -1
-          when (b_value.nil? or (b_value.respond_to?(:empty?) and b_value.empty?))
+          when b_empty
             1
           when Array === a_value
             if a_value.length == 1 and b_value.length == 1

data/lib/rbbt/tsv/attach.rb

@@ -237,9 +237,13 @@ module TSV
         attach_index other, index, fields
       end
     rescue Exception
-      Log.exception $!
-      same_key = false
-      retry
+      if same_key
+        Log.warn "Could not translate identifiers with same_key"
+        same_key = false
+        retry
+      else
+        raise $!
+      end
     end
     Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 

data/lib/rbbt/util/misc/development.rb

@@ -298,7 +298,6 @@ module Misc
            end
 
 
-    #options = Misc.add_defaults options, :respawn => true, :cpus => cpus, :into => Set.new 
     options = Misc.add_defaults options, :respawn => true, :cpus => cpus
     options = Misc.add_defaults options, :bar => "Bootstrap in #{ options[:cpus] } cpus: #{ Misc.fingerprint Annotated.purge(elems) }"
     respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
@@ -311,6 +310,7 @@ module Misc
         res = yield elem
       rescue Interrupt
         Log.warn "Process #{Process.pid} was aborted"
+        raise $!
       end
       res = nil unless options[:into]
       raise RbbtProcessQueue::RbbtProcessQueueWorker::Respawn, res if respawn == :always and cpus > 1

data/lib/rbbt/util/misc/inspect.rb

@@ -237,7 +237,7 @@ module Misc
 
     if defined? Annotated and Annotated === _obj and not (defined? AssociationItem and AssociationItem === _obj)
       info = Annotated.purge(_obj.info)
-      str << "_" << hash2str(info) 
+      str << "_" << obj2str(info) 
     end
 
     str

data/lib/rbbt/util/open.rb

@@ -270,6 +270,7 @@ module Open
       dir_sub_path.push file
       exists_in_repo(*dir_sub_path)
     else
+      file = file.find if Path === file
       File.exists? file
     end
   end

data/lib/rbbt/util/tc_cache.rb

@@ -0,0 +1,18 @@
+require 'rbbt/tsv'
+require 'rbbt/persist'
+module TCCache
+  def self.open(file, type = :single)
+    database = Persist.open_tokyocabinet(file, true, type, "HDB")
+    database.extend TCCache
+  end
+
+  def cache(key)
+    if self.include? key
+      return self[key]
+    else
+      self.write_and_read do
+        self[key] = yield
+      end
+    end
+  end
+end

data/lib/rbbt/workflow/step/run.rb

@@ -285,6 +285,7 @@ class Step
 
   def produce(force=true)
     return self if done? and not dirty?
+
     if error? or aborted?
       if force
         clean
@@ -292,9 +293,13 @@ class Step
         raise "Error in job: #{status}"
       end
     end
+
     clean if dirty? or not running?
+
     run(true) unless started?
+
     join unless done?
+
     self
   end
 

data/share/rbbt_commands/resource/produce

@@ -7,6 +7,7 @@ require 'rbbt/workflow'
 options = SOPT.get <<EOF
 -w--workflows* Workflows to use; 'all' for all in Rbbt.etc.workflows:
 -r--requires* Files to require; 'all' for all in Rbbt.etc.requires:
+-f--force Force the production if the file is already present
 -h--help Help
 EOF
 
@@ -41,7 +42,9 @@ end
 
 resource, path = ARGV
 
+force = options[:force]
+
 resource = Kernel.const_get(resource)
 
-puts resource[path].produce.find
+puts resource[path].produce(force).find
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.18.1
+  version: 5.19.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-22 00:00:00.000000000 Z
+date: 2015-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -263,6 +263,7 @@ files:
 - lib/rbbt/util/simpleopt/setup.rb
 - lib/rbbt/util/tar.rb
 - lib/rbbt/util/task/job.rb
+- lib/rbbt/util/tc_cache.rb
 - lib/rbbt/util/tmpfile.rb
 - lib/rbbt/workflow.rb
 - lib/rbbt/workflow/accessor.rb