rbbt-util 5.21.66 → 5.21.67

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ff21babc0504e17df0c48429ee4db81aebbac9a
-  data.tar.gz: 0c6ba9a55916f59bdc371510a696ec2a49582cbd
+  metadata.gz: 6fdfbe2c9f05d2feec86f7af0493388d3fd82ae6
+  data.tar.gz: 76fdd311a0785941e491837ab1672c74ac4b1bf2
 SHA512:
-  metadata.gz: b34c51ad0a84893b4d809234d99e935a2d58c88fe89e04b76f9dee30c49d39f04844a7ad02dea30ac18bd98be611e226f500e2bb95248778f4689e437620a873
-  data.tar.gz: 50cb3f8b234f73a38bc406a2d328fafd2b87b83e9121e39a3885b52727c8649bcb0c456ae334f5a617343e595532d685aa567ba462942cfdc39e46c855fce68d
+  metadata.gz: 5ff39d846bfe555f00ae2643a8510636d626044b7c93186fb30d3d10141fa3c24777aa4426e4fea0ffc7ccdffeb89d70ff7138b757f9cd714bf7ebf681119089
+  data.tar.gz: 131a08fa2c1e7741e9542b508075021830dd708fd62f05e473b6f91c08a79670d669c2791daf31f1f4f11c351dd501702b2155f9b88dc39e686c552b91f21683

data/lib/rbbt/association/database.rb

@@ -72,14 +72,26 @@ module Association
     info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
     options = options.merge({:key_field => source_field, :fields =>  info_fields})
 
-    tsv.with_monitor(options[:monitor]) do
-      tsv = tsv.reorder source_field, fields if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
-    end
+    data = options[:data] || {}
+    TmpFile.with_file do |tmpfile|
+      tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
+
+      tsv.with_monitor(options[:monitor]) do
+        tsv = tsv.reorder source_field, fields, :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
+      end
 
-    tsv.key_field = source_header
-    tsv.fields = field_headers
+      tsv.key_field = source_header
+      tsv.fields = field_headers
 
-    tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
+      if source_format or target_format
+        tsv = translate tsv, source_format, target_format, :persist => true, :persist_data => data, :data => data
+      else
+        tsv.through do |k,v|
+          data[k] = v
+        end
+        tsv.annotate data
+      end
+    end
 
     tsv
   end
@@ -87,7 +99,7 @@ module Association
   def self.open_stream(stream, options = {})
     fields, persist = Misc.process_options options, :fields, :persist
 
-    parser = TSV::Parser.new stream, options.merge(:fields => nil, :key_field => nil)
+    parser = TSV::Parser.new stream, options.merge(:fields => nil, :key_field => nil, :type => :double)
 
     key_field, *_fields = all_fields = parser.all_fields
 
@@ -128,13 +140,25 @@ module Association
     open_options = options.merge(parser.options).merge(:parser => parser)
     open_options = Misc.add_defaults open_options, :monitor => {:desc => "Parsing #{ Misc.fingerprint stream }"}
 
-    tsv = TSV.parse parser.stream, {}, open_options
-    tsv.key_field = source_header
-    tsv.fields = field_headers
+    data = open_options[:data] || {}
+    tsv = nil
+    TmpFile.with_file do |tmpfile|
+      tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
+
+      tsv = TSV.parse parser.stream, tmp_data, open_options
+      tsv.key_field = source_header
+      tsv.fields = field_headers
 
-    tsv = tsv.to_double unless tsv.type == :double
+      if source_format or target_format
+        tsv = translate tsv, source_format, target_format, :persist => true, :persist_data => data, :data => data
+      else
+        tsv.through do |k,v|
+          data[k] = v
+        end
+        tsv.annotate data
+      end
 
-    tsv = translate tsv, source_format, target_format, :persist => persist if source_format or target_format
+    end
 
     tsv
   end
@@ -148,7 +172,18 @@ module Association
                  open_stream(TSV.get_stream(file), options.dup)
                when TSV
                  file = file.to_double unless file.type == :double
-                 reorder_tsv(file, options.dup)
+                 tsv = reorder_tsv(file, options.dup)
+                 if options[:data]
+                   data = options[:data]
+                   tsv.with_unnamed do
+                     tsv.with_monitor("Saving database #{Misc.fingerprint file}") do
+                       tsv.through do |k,v|
+                         data[k] = v
+                       end
+                     end
+                   end
+                 end
+                 tsv
                when IO
                  open_stream(file, options.dup)
                else

data/lib/rbbt/association/open.rb

@@ -22,19 +22,17 @@ module Association
       file = file.call if Proc === file
 
       options = options.dup
-      tsv = Association.database(file, options.merge(:persist => persist, :unnamed => true))
-      tsv = tsv.to_double unless tsv.type == :double
+      data.serializer = :double if data.respond_to? :serializer
 
-      tsv.annotate data
+      tsv = Association.database(file, options.merge(:persist => true, :unnamed => true, :data => data, :type => :double))
 
-      data.serializer = :double if data.respond_to? :serializer
-      tsv.with_unnamed do
-        tsv.with_monitor("Saving database #{Misc.fingerprint file}") do
-          tsv.through do |k,v|
-            data[k] = v
-          end
-        end
-      end
+      #tsv.with_unnamed do
+      #  tsv.with_monitor("Saving database #{Misc.fingerprint file}") do
+      #    tsv.through do |k,v|
+      #      data[k] = v
+      #    end
+      #  end
+      #end
 
       data
     end

data/lib/rbbt/tsv/manipulate.rb

@@ -249,10 +249,10 @@ module TSV
     persist_options[:prefix] = "Reorder"
 
     Persist.persist_tsv self, self.filename, self.options.merge({:key_field => new_key_field, :fields => new_fields}), persist_options do |data|
-      if data.respond_to? :persistence_path
-        real_data = data 
-        data = {}
-      end
+      #if data.respond_to? :persistence_path
+      #  real_data = data 
+      #  data = {}
+      #end
 
       new_key_field_name, new_field_names = nil, nil
       with_unnamed do
@@ -304,11 +304,11 @@ module TSV
         end
       end
 
-      if real_data and real_data.respond_to? :persistence_path
-        real_data.serializer = type if real_data.respond_to? :serializer
-        real_data.merge!(data)
-        data = real_data
-      end
+      #if real_data and real_data.respond_to? :persistence_path
+      #  real_data.serializer = type if real_data.respond_to? :serializer
+      #  real_data.merge!(data)
+      #  data = real_data
+      #end
 
       data.extend TSV unless TSV === data
       self.annotate(data)

data/lib/rbbt/workflow/accessor.rb

@@ -325,7 +325,7 @@ class Step
       begin
         klass = Kernel.const_get(ex_class)
         ex = klass.new ex_message
-        #ex.set_backtrace ex_backtrace
+        ex.set_backtrace ex_backtrace unless ex_backtrace.nil? or ex_backtrace.empty?
         ex
       rescue
         Log.exception $!
@@ -338,10 +338,11 @@ class Step
     return true if aborted?
     return false unless error?
     begin
+      return true unless info[:exception]
       klass = Kernel.const_get(info[:exception][:class])
       not (RbbtException == klass or RbbtException === klass)
     rescue Exception
-      false
+      true
     end
   end
 
@@ -350,6 +351,7 @@ class Step
   end
 
   def dirty?
+    return true if done? and not status == :done
     dirty_files = rec_dependencies.collect{|dependency| dependency.path unless dependency.error? and not dependency.recoverable_error? }.compact.uniq.reject{|path| not (Path === path) or path.exists?}
     if dirty_files.any?
       true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.66
+  version: 5.21.67
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-26 00:00:00.000000000 Z
+date: 2017-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake