rbbt-util 5.34.14 → 5.34.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2e50428f936aab209874c5f26e7504aa14da9a30bc7e2b13575432527a5b6a4
-  data.tar.gz: e522b76bb910b477e303e2cfc323c8708fa4d3bbecc0a7c46de473fe81f1ef6a
+  metadata.gz: 5b76f18c79a3b52708203f69eb7cd9baff8cb07feb2a4b26850094cfda9b9edc
+  data.tar.gz: 8511317ae202283e8138746485e74d39b28881b78b85539c521e94309337782e
 SHA512:
-  metadata.gz: e73a4ceda5eca11497ea4f0fe91fda7275375aaf8aeb6f9f70a45a46d4c71022527d7cd32c0cb30700a6bc2724bd1e2d2e0bd808a553ee3c8d6e729baa1dc2c0
-  data.tar.gz: a050c5defff3e3d36fe5682c117802d54cfeb930e3e61ff96da29e326719bf97af6fe85b5c30812dd91c5c6a447201164c15a3e1714aa1226690a0c49782ba9a
+  metadata.gz: 71dd9c6806e8129fa08b466ddd3e8ea7172a0938000f399b030ef89a5f40321b08f24f6b98ba823ac72317460f45dbc9a8fc88c65899582766a542fde55adb24
+  data.tar.gz: 0b6d4b0aea41c92161193ffb379ee86e44715298ebf6d26d585ffb6292ae89a77e82a9764f07630106510c45bc5b56193b716de9edd71ed52cb4046f711d2e24

data/lib/rbbt/hpc/slurm.rb

@@ -10,7 +10,9 @@ module HPC
       <<-EOF
 let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
 let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
-[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" || MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" 
+[ ! -z $SLURM_MEM_PER_NODE ] && MAX_MEMORY="$SLURM_MEM_PER_NODE"
 export MAX_MEMORY_DEFAULT
 export MAX_MEMORY
 export BATCH_JOB_ID=$SLURM_JOB_ID

data/lib/rbbt/persist/tsv.rb

@@ -146,9 +146,22 @@ module Persist
           data.serializer = :type 
         end
 
-        data.write_and_read do
-          yield data
+        if persist_options[:persist] == :preload
+          tmp_tsv = yield({})
+          tmp_tsv.annotate data
+          data.serializer = tmp_tsv.type
+          data.write_and_read do
+            tmp_tsv.each do |k,v|
+              data[k] = v
+            end
+          end
+        else
+          data.write_and_read do
+            yield data
+          end
+        end
 
+        data.write_and_read do
           FileUtils.mv data.persistence_path, path if File.exist? data.persistence_path and not File.exist? path
           tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
           tsv.persistence_path = path
@@ -165,6 +178,7 @@ module Persist
       end
     end
   end
+
 end
 
 require 'rbbt/persist/tsv/sharder'

data/lib/rbbt/persist.rb

@@ -84,6 +84,8 @@ module Persist
       perfile = prefix.to_s + ":" + file.to_s.gsub(/\//, '>') 
     end
 
+    perfile.sub!(/\.b?gz$/,'')
+
     if options.include? :filters
       options[:filters].each do |match,value|
         perfile = perfile + "&F[#{match}=#{Misc.digest(value.inspect)}]"

data/lib/rbbt/tsv/manipulate.rb

@@ -360,7 +360,7 @@ module TSV
     new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
     self.with_unnamed do
       keys.each do |k|
-        new[k] = self[k]
+        new[k] = self[k] if self.include?(k)
       end
     end
     new

data/lib/rbbt/tsv/parser.rb

@@ -135,7 +135,7 @@ module TSV
                  []
                else
                  parts.values_at *field_positions
-               end.collect{|value| (value.empty? || value.nil?) ? [nil] : value.split(@sep2, -1) }
+               end.collect{|value| (value.nil? || value.empty?) ? [] : value.split(@sep2, -1) }
       [keys, values]
     end
 

data/lib/rbbt/tsv.rb

@@ -137,12 +137,12 @@ module TSV
     case
     when Path === stream 
       stream.open do |f|
-        f.no_fail = false if ConcurrentStream === f
+        f.no_fail = true if ConcurrentStream === f
         Parser.new f, options
       end
     when (String === stream and stream.length < 300 and (Open.exists? stream or Open.remote? stream))
       Open.open(stream) do |f|
-        f.no_fail = false if ConcurrentStream === f
+        f.no_fail = true if ConcurrentStream === f
         Parser.new f, options
       end
     else

data/lib/rbbt/util/cmd.rb

@@ -163,6 +163,7 @@ module CMD
 
     if in_content.respond_to?(:read)
       in_thread = Thread.new(Thread.current) do |parent|
+        Thread.current.report_on_exception = false if no_fail
         begin
           begin
             while c = in_content.readpartial(Misc::BLOCK_SIZE)
@@ -177,7 +178,7 @@ module CMD
             in_content.join if in_content.respond_to? :join 
           end
         rescue
-          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}"
+          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
           raise $!
         end
       end

data/lib/rbbt/util/misc/math.rb

@@ -39,17 +39,17 @@ module Misc
   end
 
   def self.sum(list)
-    list.compact.inject(0.0){|acc,e| acc += e}
+    list.compact.inject(0.0){|acc,e| acc += e }
   end
 
   def self.mean(list)
-    sum(list).to_f / list.compact.length
+    sum(list.compact.collect{|v| v.to_f } ) / list.compact.length
   end
 
   def self.median(array)
     sorted = array.sort
     len = sorted.length
-    (sorted[(len - 1) / 2] + sorted[len / 2]) / 2.0
+    (sorted[(len - 1) / 2] + sorted[len / 2]).to_f / 2
   end
 
   def self.variance(list)

data/share/rbbt_commands/tsv/info

@@ -18,6 +18,7 @@ Display summary information. Works with Tokyocabinet HDB and BDB as well.
 -hh--header_hash* Change the character used to mark the header line (defaults to #)
 -k--key_field* Change the key field
 -f--fields* Change the fields to load
+-fp--fingerprint Fingerprint values
 -s--sep* Change the fields separator (default TAB)
 -h--help Help
 EOF
@@ -70,7 +71,11 @@ else
   
   puts "Rows: #{Log.color :blue, rows}" 
   parts = []
-  header.first_line.split(header.sep, -1).each_with_index{|p,i| parts << (Log.color(:cyan, "(#{i}) ") << p.strip) }
+  header.first_line.split(header.sep, -1).each_with_index do |p,i| 
+   p.strip!
+   p = Misc.fingerprint p.split("|") if %w(double flat).include?(header.options[:type].to_s) && options[:fingerprint]
+   parts << (Log.color(:cyan, "(#{i}) ") << p.strip) 
+  end
   puts parts * "\t"
   puts
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.14
+  version: 5.34.16
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-28 00:00:00.000000000 Z
+date: 2022-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake