scout-gear 10.3.0 → 10.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a7adf9c59ab3ce69e89dd2e02089b5287605a1b74c6a435a5bfa4ece17c1f75
-  data.tar.gz: 65f9fa08d1b4a99e973d33eb5a7c0ddab85a98b5d6a675f8a2f5cfe381e664aa
+  metadata.gz: 4e4dbf9f73608dbc264f7f7584af04ac87bbeb2175b04a7ef50d2ef72cea8592
+  data.tar.gz: d423876c0f4cd1132128c519e9576afa7933f5c2f50b2f3498009a7dcba8ea30
 SHA512:
-  metadata.gz: cd78580f6d0fd2a417f15f425602131d4e3150fc8a0f35a1d2f63f5868e29cae8a9f59244a2fdc5d642441051b8528bcd2f8596aaf4bc11414437e6d4dc4cd14
-  data.tar.gz: ad61b8616e5029ff0ee748b48ef78928dc862bfbb0ba763610ad9f89f7e865975d816318ea388d30d774f1088894b9c2a3d42487b9a3c01c795c45cd5b37f2d7
+  metadata.gz: 8d53af5f2c84362e664ff33f0afafafd5bb92756e5843c68466b81b09b51830a03a99093c18103a3f6a7d26a70099a283fc4841c346b223691b9afd6774399ac
+  data.tar.gz: 3c3fbc75f0ad79395211eb198198aa5e6e31de6e00a5d54375cfff5197d5cacfb6e0755889d157e4671b43edbbb1535b020814881df83a34227d231c623b0461

data/.vimproject

@@ -74,7 +74,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    tsv=tsv{
     util.rb
     util=util{
-     process.rb
      select.rb
      process.rb
      unzip.rb

data/VERSION

@@ -1 +1 @@
-10.3.0
+10.4.0

data/lib/scout/tsv/attach.rb

@@ -40,12 +40,18 @@ module TSV
 
   def self.attach(source, other, target: nil, fields: nil, match_key: nil, other_key: nil, one2one: true, complete: false, insitu: nil, persist_input: false, bar: nil)
     source = TSV::Transformer.new source unless TSV === source || TSV::Parser === source
-    other = TSV.open other, persist: persist_input unless TSV === other 
+    other = TSV::Parser.new other unless TSV === other || TSV::Parser === other
 
     fields = [fields] if String === fields
 
     match_key, other_key = TSV.match_keys(source, other, match_key: match_key, other_key: other_key)
 
+    if ! (TSV === other)
+      other_key_name = other_key == :key ? other.key_field : other.fields[other_key]
+      other = TSV.open other, key_field: other_key_name, fields: fields, one2one: true, persist: persist_input
+      other_key = :key if other.key_field == source.key_field
+    end
+
     if TSV::Transformer === source
       source.dumper = case target
                       when :stream
@@ -117,6 +123,7 @@ module TSV
                 current_values[overlap] = v if current_values[overlap].nil? || String === current_values[overlap] && current_values[overlap].empty?
               else
                 current_values[overlap] ||= []
+                next if v.nil?
                 current_values[overlap].concat (v - current_values[overlap])
               end
             end
@@ -154,6 +161,7 @@ module TSV
                          end
 
             other_values.zip(overlaps).each do |v,overlap|
+              next if v.nil?
               if false && overlap == :key
                 other_key = Array === v ? v : v.first
               elsif source.type == :list

data/lib/scout/tsv/change_id.rb

@@ -1,10 +1,10 @@
 module TSV
-  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, stream: false, keep: false, persist_identifiers: nil)
+  def self.change_key(source, new_key_field, identifiers: nil, one2one: false, merge: true, stream: false, keep: false, persist_identifiers: nil)
     source = TSV::Parser.new source if String === source
     if identifiers && source.identify_field(new_key_field, strict: true).nil?
       identifiers = identifiers.nil? ? source.identifiers : identifiers
       new = source.attach(identifiers, fields: [new_key_field], insitu: false, one2one: true, persist_input: persist_identifiers)
-      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one)
+      new = new.change_key(new_key_field, keep: keep, stream: stream, one2one: one2one, merge: merge)
       return new
     end
 
@@ -17,7 +17,7 @@ module TSV
       [k, v]
     end
 
-    stream ? transformer : transformer.tsv
+    stream ? transformer : transformer.tsv(merge: merge, one2one: one2one)
   end
 
   def change_key(*args, **kwargs)

data/lib/scout/tsv/index.rb

@@ -2,7 +2,7 @@ require_relative 'parser'
 require_relative 'transformer'
 require_relative 'persist/fix_width_table'
 module TSV
-  def self.index(tsv_file, target: 0, fields: nil, order: true, bar: nil, **kwargs)
+  def self.index(tsv_file, target: :key, fields: nil, order: true, bar: nil, **kwargs)
     persist, type, persist_update, data_persist = IndiferentHash.process_options kwargs,
       :persist, :persist_type, :persist_update, :data_persist,
       :persist => false, :persist_type => "HDB"
@@ -10,7 +10,7 @@ module TSV
 
     fields = :all if fields.nil?
 
-    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, :prefix => "Index", :other_options => kwargs)) do |filename|
+    Persist.persist(tsv_file, type, kwargs.merge(target: target, fields: fields, persist: persist, update: persist_update, prefix: "Index", other_options: {fields: fields, target: target, order: order})) do |filename|
       if filename
         index = ScoutCabinet.open(filename, true, type)
         TSV.setup(index, :type => :single)

data/lib/scout/tsv/parser.rb

@@ -276,6 +276,7 @@ module TSV
       end
       @fix = fix
       @options, @key_field, @fields, @first_line, @preamble = TSV.parse_header(@stream, fix:fix, header_hash:header_hash, sep:sep)
+      @options[:filename] = file if Path.is_filename?(file)
       @options[:sep] = sep if @options[:sep].nil?
       @options.merge!(:key_field => @key_field, :fields => @fields)
       @type = @options[:type] || type
@@ -374,7 +375,7 @@ module TSV
   end
 
   def self.parse(stream, fix: true, header_hash: "#", sep: "\t", filename: nil, namespace: nil, unnamed: false, serializer: nil, **kwargs, &block)
-    parser = TSV::Parser.new stream, fix: fix, header_hash: header_hash, sep: sep
+    parser = TSV::Parser === stream ? stream : TSV::Parser.new(stream, fix: fix, header_hash: header_hash, sep: sep)
 
     cast = kwargs[:cast]
     cast = parser.options[:cast] if cast.nil?
@@ -407,8 +408,9 @@ module TSV
 
     data = parser.traverse **kwargs, &block
     data.type = type
-    data.filename = filename
-    data.namespace = namespace
+    data.filename = filename || parser.options[:filename]
+    data.namespace = namespace || parser.options[:namespace]
+    data.identifiers = parser.options[:identifiers]
     data.unnamed = unnamed
     data.save_extension_attr_hash if data.respond_to?(:save_extension_attr_hash)
     data

data/lib/scout/tsv/persist.rb

@@ -3,12 +3,12 @@ require_relative 'persist/adapter'
 
 begin
   require_relative 'persist/tokyocabinet'
-rescue
+rescue Exception
 end
 
 begin
   require_relative 'persist/tkrzw'
-rescue
+rescue Exception
 end
 
 Persist.save_drivers[:tsv] = proc do |file,content| 

data/lib/scout/tsv/transformer.rb

@@ -130,6 +130,27 @@ module TSV
     res
   end
 
+  def to_double
+    return self if self.type == :double
+    res = self.annotate({})
+    self.with_unnamed do
+      transformer = Transformer.new self, res
+      transformer.type = :double
+      transformer.traverse do |k,v|
+        case self.type
+        when :single
+          [k, [[v]]]
+        when :list
+          [k, v.collect{|v| [v] }]
+        when :flat
+          [k, [v]]
+        end
+      end
+    end
+    res
+  end
+
+
   def to_single
     res = self.annotate({})
     transformer = Transformer.new self, res

data/lib/scout/tsv/util/process.rb

@@ -46,7 +46,7 @@ module TSV
   def add_field(name = nil)
     through do |key, values|
       new_values = yield(key, values)
-      new_values = [new_values] if type == :double and not Array === new_values
+      new_values = [new_values].compact if type == :double and not Array === new_values
 
       case
       when (values.nil? and (fields.nil? or fields.empty?))

data/lib/scout/tsv.rb

@@ -28,9 +28,43 @@ module TSV
     {:key_field => key, :fields => fields}.merge(rest_options)
   end
 
+  class << self
+    alias original_setup setup
+
+    def setup(obj, *rest, &block)
+
+      if rest.length == 1 && String === rest.first
+        options = TSV.str2options(rest.first)
+        if Array === obj
+          default_value = case options[:type]
+                          when :double, :flat, :list, nil
+                            []
+                          when :single
+                            nil
+                          end
+          obj = Misc.array2hash(obj, default_value)
+        end
+        original_setup(obj, options, &block)
+      else
+        if Array === obj
+          options = rest.first if Hash === rest.first
+          options ||= {}
+          default_value = case options[:type]
+                          when :double, :flat, :list, nil
+                            []
+                          when :single
+                            nil
+                          end
+          obj = Misc.array2hash(obj, default_value)
+        end
+        original_setup(obj, *rest, &block)
+      end
+    end
+  end
+
   def self.str_setup(option_str, obj)
     options = TSV.str2options(option_str) 
-    setup(obj, options)
+    setup(obj, **options)
   end
 
   def self.open(file, options = {})
@@ -40,7 +74,17 @@ module TSV
     persist_options = IndiferentHash.add_defaults persist_options, :prefix => "TSV", :type => :HDB
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
-    Persist.persist(file, persist_options[:type], persist_options.merge(:other_options => options)) do |filename|
+
+    source_name, other_options = case file
+                  when StringIO
+                    [file.inspect, options]
+                  when TSV::Parser
+                    [file.options[:filename], file.options]
+                  else
+                    [file, options]
+                  end
+
+    Persist.persist(source_name, persist_options[:type], persist_options.merge(:other_options => other_options)) do |filename|
       if filename
         data = case persist_options[:type]
                when :HDB, :BDB
@@ -52,7 +96,7 @@ module TSV
         data = nil
       end
       options[:data] = data if data
-      options[:filename] = file
+      options[:filename] = TSV::Parser === file ? file.options[:filename] : file
 
       if data
         Log.debug "TSV open #{Log.fingerprint file} into #{Log.fingerprint data}"
@@ -60,8 +104,12 @@ module TSV
         Log.debug "TSV open #{Log.fingerprint file}"
       end
 
-      Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
-        TSV.parse(f, **options)
+      if TSV::Parser === file
+        TSV.parse(file, **options)
+      else
+        Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
+          TSV.parse(f, **options)
+        end
       end
     end
   end

data/lib/scout/workflow/definition.rb

@@ -103,7 +103,14 @@ module Workflow
   end
 
   def task(name_and_type, &block)
-    name, type = name_and_type.collect.first
+    case name_and_type
+    when Hash
+      name, type = name_and_type.collect.first
+    when Symbol
+      name, type = [name_and_type, :binary]
+    when String
+      name, type = [name_and_type, :binary]
+    end
     type = type.to_sym if String === type
     name = name.to_sym if String === name
     @tasks ||= IndiferentHash.setup({})

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.3.0 ruby lib
+# stub: scout-gear 10.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.3.0"
+  s.version = "10.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-08-01"
+  s.date = "2023-08-09"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -167,7 +167,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.4.13".freeze
+  s.rubygems_version = "3.5.0.dev".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/test/scout/test_tsv.rb

@@ -310,6 +310,9 @@ row2   b  bbb bbbb bb
 
     tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
     assert_equal "ID", tsv.key_field
+
+    tsv = TSV.setup({}, "ID~ValueA,ValueB#:type=:flat")
+    assert_equal "ID", tsv.key_field
   end
 
   def test_cast_in_header
@@ -328,4 +331,31 @@ c 3
       assert_include tsv.to_s, ":cast=:to_f"
     end
   end
+
+  def test_open_persist_parser
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      parser = TSV::Parser.new filename
+      tsv = TSV.open(parser, :persist => true)
+      tsv.close
+      Persist::CONNECTIONS.clear
+      TSV.open(filename, :persist => true)
+    end
+
+    assert_equal "Id", tsv.key_field
+
+    assert tsv.respond_to?(:persistence_class)
+    assert_equal TokyoCabinet::HDB, tsv.persistence_class
+
+    assert_include tsv.keys, 'row1'
+    assert_include tsv.keys, 'row2'
+  end
+
 end

data/test/scout/tsv/persist/test_tkrzw.rb

@@ -1,123 +1,129 @@
 require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
 
 require 'scout/tsv'
 
-class TestScoutTKRZW < Test::Unit::TestCase
-  def _test_open
-    TmpFile.with_file nil do |tmp|
-      db = ScoutTKRZW.open(tmp, true)
-      1000.times do |i|
-        db["foo#{i}"] = "bar#{i}"
-      end
-      assert_include db, 'foo1'
-      assert_equal 1000, db.keys.length
-
-      db.close
-      TmpFile.with_file nil do |tmp_alt|
-        Open.cp tmp, tmp_alt
-        db2 = ScoutTKRZW.open(tmp_alt, false)
-        assert_include db2, 'foo1'
-        assert_equal 1000, db2.keys.length
+begin
+  require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+  class TestScoutTKRZW < Test::Unit::TestCase
+    def setup
+    end
+
+    def test_open
+      TmpFile.with_file nil do |tmp|
+        db = ScoutTKRZW.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        assert_include db, 'foo1'
+        assert_equal 1000, db.keys.length
+
+        db.close
+        TmpFile.with_file nil do |tmp_alt|
+          Open.cp tmp, tmp_alt
+          db2 = ScoutTKRZW.open(tmp_alt, false)
+          assert_include db2, 'foo1'
+          assert_equal 1000, db2.keys.length
+        end
       end
     end
-  end
 
-  def _test_tkrzw
-    content =<<-'EOF'
+    def test_tkrzw
+      content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
 row2    A    B    Id3
 row2    a    a    id3
-    EOF
+      EOF
 
-    tsv = TmpFile.with_file(content) do |filename|
-      Persist.persist(__method__, 'tkh') do
-        TSV.open(filename)
+      tsv = TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, 'tkh') do
+          TSV.open(filename)
+        end
       end
-    end
 
 
-    assert_equal %w(a aa aaa), tsv["row1"][0]
+      assert_equal %w(a aa aaa), tsv["row1"][0]
 
-    assert TSVAdapter === tsv
-    assert TSV === tsv
-    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
-    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+      assert TSVAdapter === tsv
+      assert TSV === tsv
+      assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+      assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
 
-    tsv.close
-    tsv_loaded = assert_nothing_raised do
-      TmpFile.with_file(content) do |filename|
-        Persist.persist(__method__, 'tkh') do
-          raise
+      tsv.close
+      tsv_loaded = assert_nothing_raised do
+        TmpFile.with_file(content) do |filename|
+          Persist.persist(__method__, 'tkh') do
+            raise
+          end
         end
       end
+
+      assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
     end
 
-    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
-  end
+    def __test_benchmark1
+      TmpFile.with_file nil do |tmp|
+
+        Misc.benchmark(1000) do
+          db = ScoutTKRZW.open(tmp, true)
+          1000.times do |i|
+            db["foo#{i}"] = "bar#{i}"
+          end
+          10.times do
+            db.keys
+          end
+          10.times do |i|
+            db["foo#{i}"]
+          end
+          Open.rm tmp
+        end
+
+        Misc.benchmark(1000) do
+          db = ScoutCabinet.open(tmp, true)
+          1000.times do |i|
+            db["foo#{i}"] = "bar#{i}"
+          end
+          10.times do
+            db.keys
+          end
+          10.times do |i|
+            db["foo#{i}"]
+          end
+          db.close
+          Open.rm tmp
+        end
+      end
+    end
 
-  def __test_benchmark1
-    TmpFile.with_file nil do |tmp|
+    def test_benchmark2
+      TmpFile.with_file nil do |tmp|
 
-      Misc.benchmark(1000) do
         db = ScoutTKRZW.open(tmp, true)
-        1000.times do |i|
+        10000.times do |i|
           db["foo#{i}"] = "bar#{i}"
         end
-        10.times do
-          db.keys
-        end
-        10.times do |i|
-          db["foo#{i}"]
+
+        Misc.benchmark(1000) do
+          100.times do |i|
+            db["foo#{i}"]
+          end
         end
-        Open.rm tmp
-      end
 
-      Misc.benchmark(1000) do
+        Open.rm tmp
         db = ScoutCabinet.open(tmp, true)
-        1000.times do |i|
+        10000.times do |i|
           db["foo#{i}"] = "bar#{i}"
         end
-        10.times do
-          db.keys
-        end
-        10.times do |i|
-          db["foo#{i}"]
+        Misc.benchmark(1000) do
+          100.times do |i|
+            db["foo#{i}"]
+          end
         end
-        db.close
-        Open.rm tmp
       end
     end
-  end
 
-  def test_benchmark2
-    TmpFile.with_file nil do |tmp|
-
-      db = ScoutTKRZW.open(tmp, true)
-      10000.times do |i|
-        db["foo#{i}"] = "bar#{i}"
-      end
-
-      Misc.benchmark(1000) do
-        100.times do |i|
-          db["foo#{i}"]
-        end
-      end
-
-      Open.rm tmp
-      db = ScoutCabinet.open(tmp, true)
-      10000.times do |i|
-        db["foo#{i}"] = "bar#{i}"
-      end
-      Misc.benchmark(1000) do
-        100.times do |i|
-          db["foo#{i}"]
-        end
-      end
-    end
   end
-
+rescue Exception
 end
 

data/test/scout/tsv/test_transformer.rb

@@ -117,4 +117,16 @@ row2    A2|A22    B2|B22
     tsv = TSV.open(content)
     assert_equal %w(A1 A11 B1 B11), tsv.to_flat["row1"]
   end
+
+  def test_to_double
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1    B1
+row2    A2    B2
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal %w(A1), tsv.to_double["row1"]["ValueA"]
+  end
 end

data/test/scout/tsv/util/test_process.rb

@@ -43,5 +43,25 @@ row2    AA    BB    Id33
     assert_equal %w(c cc ccc), tsv["row1"]["ValueC"]
     assert_equal %w(C CC), tsv["row2"]["ValueC"]
   end
+
+  def test_add_field_double_empty
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    tsv.add_field "ValueC" do |k,v|
+      nil
+    end
+
+    assert_equal %w(), tsv["row1"]["ValueC"]
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.3.0
+  version: 10.4.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-01 00:00:00.000000000 Z
+date: 2023-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -295,7 +295,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.13
+rubygems_version: 3.5.0.dev
 signing_key: 
 specification_version: 4
 summary: basic gear for scouts