rbbt-util 5.28.12 → 5.28.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 889338e97d8b2f2467dcbbb7d7ecfb1a4904a317702c2b7c7164757efd1f55dc
-  data.tar.gz: 0e71e707ea5d0bab991aaf25f9dd5bf1d371d36ed6267694934aab92218ba293
+  metadata.gz: 74253d97dc17c890ce9022e38ccd514847e785b69f4cfd74a8a5333a9aa1c97d
+  data.tar.gz: d908fecdec1c5e6ccce81b3190883011d775f38cab689c136e7c90808c3c93ac
 SHA512:
-  metadata.gz: 173386b238dda5361b1c473c90708b783736f3726f868e86dd64f732d3e9474fc9987b133fdfc3d19538ccc19b31eb2f91ae29fa1fedaf150e5d46619ef69c76
-  data.tar.gz: 306f1058d7248387a7887bed09c4bd21541e6f52462de6b17d2d4b04bb73ebe4f0cdb3a9245e1a24f61fec252587d3b0669623d8544a6fc0d250eeebdad8c47e
+  metadata.gz: 7c6b8f81e5f814a35ed3b738ad85d4e7314b9fb24b36ecdb3828d1d4644cc8b9ce4117c862ae7baa0bd7f60362e93be2aff282a2d24c474e327d1d5948f56fff
+  data.tar.gz: 3a6235b141f139f245935b8858b175b2d54302b89dc3ac10101ffddfd10d1161760aaf0f176114a4fc6d8f0b8aa17556b8637e2f2ab815dcf875ea98c4825921

data/lib/rbbt/hpc.rb

@@ -98,6 +98,8 @@ module Marenostrum
 #SBATCH --nodes="#{nodes}"
       EOF
 
+      prep  = ""
+
       if highmem
         header +=<<-EOF
 #SBATCH --constraint=highmem 
@@ -146,8 +148,6 @@ SINGULARITY_RUBY_INLINE="$HOME/.singularity_ruby_inline"
 mkdir -p "$SINGULARITY_RUBY_INLINE"
         EOF
 
-        prep  = ""
-
         if contain
           scratch_group_dir = File.join('/gpfs/scratch/', group)
           projects_group_dir = File.join('/gpfs/projects/', group)
@@ -214,6 +214,8 @@ singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv .rbbt/var/
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rbbt system clean -f &>> #{fsync}
 singularity exec -e -C -H "$CONTAINER_DIR" "$SINGULARITY_IMG" rm -Rfv tmp/ &>> #{fsync}
 EOF
+            else
+              prep = ""
             end
           end
         end

data/lib/rbbt/tsv/attach.rb

@@ -243,6 +243,7 @@ module TSV
     Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
 
     if complete
+      Log.warn "Attaching through index and completing empty rows; keys with wrong format may appear (#{other.key_field} insted of #{self.key_field})" if index
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
       common_fields = (other.fields & self.fields)
@@ -255,11 +256,11 @@ module TSV
       case type
       when :single
         missing.each do |k|
-          self[k] = nil
+          self[k] = fill
         end
       when :list
         missing.each do |k|
-          values = [nil] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -267,8 +268,9 @@ module TSV
           self[k] = values
         end
       when :double
+        fill = [] if fill.nil?
         missing.each do |k|
-          values = [[]] * field_length
+          values = [fill] * field_length
           other_values = other[k]
           other_common_pos.zip(this_common_pos).each do |o,t|
             values[t] = other_values[o]
@@ -276,8 +278,9 @@ module TSV
           self[k] = values
         end
       when :flat
+        fill = [] if fill.nil?
         missing.each do |k|
-          self[k] = []
+          self[k] = fill
         end
       end
     end

data/lib/rbbt/tsv/parallel.rb

@@ -2,6 +2,3 @@ require 'rbbt/util/concurrency'
 
 require 'rbbt/tsv/parallel/through'
 require 'rbbt/tsv/parallel/traverse'
-
-module TSV
-end

data/lib/rbbt/workflow/definition.rb

@@ -77,6 +77,7 @@ module Workflow
     task name do
       raise RbbtException, "dependency not found in dep_task" if dependencies.empty?
       dep = dependencies.last.join
+      raise dep.get_exception if dep.error?
       set_info :result_type, dep.info[:result_type]
       forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
       if forget

data/test/rbbt/tsv/test_attach.rb

@@ -566,7 +566,7 @@ row2,CC
       tsv3.keys.each{|k| tsv3[k] = nil if tsv3[k] == ""}
     end
 
-    assert_equal tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"], [nil, "B", nil]
+    assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
   end
 
   def test_attach_index_both_non_key
@@ -597,10 +597,10 @@ A    Id3
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
-    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce.find #.to_s
+    tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce(true).produce.find #.to_s
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
@@ -627,12 +627,92 @@ E    B
 
     tsv1 = tsv2 = nil
 
-    tsv1 = Rbbt.tmp.test.test1.data.tsv :double,  :sep => /\s+/
-    tsv2 = Rbbt.tmp.test.test2.data.tsv :double,  :sep => /\s+/
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
 
     tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
     Log.tsv tsv1
     
   end
+
+  def test_attach_complete
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueB
+row1    b
+row3    C
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [[], ["C"]], tsv1["row3"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+
+    ppp tsv1.attach tsv2, :complete => ["AA"]
+    tsv1.attach tsv2, :complete => ["AA"]
+    assert_equal [["AA"], ["C"]], tsv1["row3"]
+  end
+
+  def test_attach_complete_identifiers
+    content1 =<<-EOF
+#: :sep=/\\s+/
+#Id    ValueA
+row1    a|aa|aaa
+row2    A
+    EOF
+
+    content2 =<<-EOF
+#: :sep=/\\s+/
+#Id2    ValueB
+ROW_1    b
+ROW_2    C
+    EOF
+
+    identifiers =<<-EOF
+#: :sep=/\\s+/
+#Id    Id2
+row1    ROW_1
+row2    ROW_2
+row3    ROW_3
+    EOF
+    Rbbt.claim Rbbt.tmp.test.test1.data, :string, content1
+    Rbbt.claim Rbbt.tmp.test.test2.data, :string, content2
+    Rbbt.claim Rbbt.tmp.test.identifiers.data, :string, identifiers
+
+    tsv1 = tsv2 = nil
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+
+    tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double,  :sep => /\s+/
+    tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double,  :sep => /\s+/
+    ids = Rbbt.tmp.test.identifiers.data.produce(true).tsv :double,  :sep => /\s+/
+
+    tsv1.identifiers = ids
+
+    tsv1.attach tsv2, :complete => true
+    assert_equal [["A"], ["C"]], tsv1["row2"]
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.12
+  version: 5.28.14
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-03 00:00:00.000000000 Z
+date: 2020-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake