scout-gear 10.2.0 → 10.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e807e8eea8b8ff3b40be002fcd9cdfa9fbc8d3fdb447f20b1b3c56f1c318de8
-  data.tar.gz: 8838111c9867b63671060c82b7224f2ac310b64d375f0cc62c26b42092da6f1d
+  metadata.gz: 2a7adf9c59ab3ce69e89dd2e02089b5287605a1b74c6a435a5bfa4ece17c1f75
+  data.tar.gz: 65f9fa08d1b4a99e973d33eb5a7c0ddab85a98b5d6a675f8a2f5cfe381e664aa
 SHA512:
-  metadata.gz: 2dcc68c9d612b208d6b6b94dd366ef37f2467b364d1da8367b1f6833d305d6ef8c35a1dc0f0e27c6e00ce481344e230e20ac6eb85236dd27c75c59e8ad3f0ec5
-  data.tar.gz: 85d8e7458a1453966e2e123fc8ced3a0c2bc8be884a7570f1bb57f4d260db582e90380d2ea0f8a91c91b68decb8439e8b556d294b9d71258c522fa06628ec22b
+  metadata.gz: cd78580f6d0fd2a417f15f425602131d4e3150fc8a0f35a1d2f63f5868e29cae8a9f59244a2fdc5d642441051b8528bcd2f8596aaf4bc11414437e6d4dc4cd14
+  data.tar.gz: ad61b8616e5029ff0ee748b48ef78928dc862bfbb0ba763610ad9f89f7e865975d816318ea388d30d774f1088894b9c2a3d42487b9a3c01c795c45cd5b37f2d7

data/.vimproject

@@ -89,9 +89,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     persist=persist{
      adapter.rb
      serialize.rb
-     sdbm.rb
-     tkrzw.rb
      tokyocabinet.rb
+     tkrzw.rb
      fix_width_table.rb
     }
     index.rb

data/VERSION

@@ -1 +1 @@
-10.2.0
+10.3.0

data/lib/scout/tsv/persist/tkrzw.rb

@@ -0,0 +1,84 @@
+require 'tkrzw'
+require_relative 'adapter'
+
+module ScoutTKRZW
+  attr_accessor :persistence_path, :persistence_class, :open_options
+
+  def self.open(path, write = true, persistence_class = 'tkh', options = {})
+    open_options = IndiferentHash.add_defaults options, truncate: true, num_buckets: 100, dbm: "HashDBM", sync_hard: true, encoding: "UTF-8"
+  
+    path = path.find if Path === path
+
+    dir = File.dirname(File.expand_path(path))
+    Open.mkdir(dir) unless File.exist?(dir)
+
+    database = Persist::CONNECTIONS[[persistence_class, path]*":"] ||= Tkrzw::DBM.new
+
+    database.close if database.open?
+
+    database.open(path, write, open_options)
+
+    database.extend ScoutTKRZW
+    database.persistence_path ||= path
+    database.open_options = open_options
+
+    #Persist::CONNECTIONS[[persistence_class, path]*":"] = database
+
+    database
+  end
+
+  def close
+    @closed = true
+    @writable = false
+    super
+  end
+
+  def read(force = false)
+    return if open? && ! writable? && ! force
+    self.close if open?
+    if !self.open(@persistence_path, false, @open_options)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = false
+    @closed = false
+
+    self
+  end
+
+  def write(force = true)
+    return if open? && writable? && ! force
+    self.close if self.open?
+
+    if !self.open(@persistence_path, true, @open_options)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = true
+    @closed = false
+
+    self
+  end
+
+  def keys
+    search("contain", "")
+  end
+end
+
+Persist.save_drivers[:tkh] = proc do |file, content|
+  data = ScoutTKRZW.open(file, true, "tkh")
+  content.annotate(data)
+  data.extend TSVAdapter
+  data.merge!(content)
+  data.close
+  data.read
+  data
+end
+
+Persist.load_drivers[:tkh] = proc do |file| 
+  data = ScoutTKRZW.open(file, false, "tkh")
+  data.extend TSVAdapter unless TSVAdapter === data
+  data
+end

data/lib/scout/tsv/persist.rb

@@ -1,6 +1,15 @@
 require 'scout/persist'
 require_relative 'persist/adapter'
-require_relative 'persist/tokyocabinet'
+
+begin
+  require_relative 'persist/tokyocabinet'
+rescue
+end
+
+begin
+  require_relative 'persist/tkrzw'
+rescue
+end
 
 Persist.save_drivers[:tsv] = proc do |file,content| 
   stream = if IO === content

data/lib/scout/tsv.rb

@@ -37,11 +37,20 @@ module TSV
     grep, invert_grep = IndiferentHash.process_options options, :grep, :invert_grep, :persist => false
 
     persist_options = IndiferentHash.pull_keys options, :persist
-    persist_options = IndiferentHash.add_defaults persist_options, :prefix => "TSV", :type => "HDB"
+    persist_options = IndiferentHash.add_defaults persist_options, :prefix => "TSV", :type => :HDB
 
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
     Persist.persist(file, persist_options[:type], persist_options.merge(:other_options => options)) do |filename|
-      data = filename ? ScoutCabinet.open(filename, true, persist_options[:type]) : nil
+      if filename
+        data = case persist_options[:type]
+               when :HDB, :BDB
+                 ScoutCabinet.open(filename, true, persist_options[:type])
+               when :tkh, :tkt, :tks
+                 ScoutTKRZW.open(filename, true, persist_options[:type])
+               end
+      else
+        data = nil
+      end
       options[:data] = data if data
       options[:filename] = file
 

data/lib/scout/workflow/definition.rb

@@ -107,7 +107,7 @@ module Workflow
     type = type.to_sym if String === type
     name = name.to_sym if String === name
     @tasks ||= IndiferentHash.setup({})
-    block = self.method(name) if block.nil?
+    block = lambda &self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
       @annotate_next_task[:extension] ||=  
@@ -137,7 +137,7 @@ module Workflow
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
     task_proc = workflow.tasks[oname]
-    raise "Task #{name} not found" if task_proc.nil?
+    raise "Task #{oname} not found" if task_proc.nil?
     returns task_proc.returns if @returns.nil?
     type = task_proc.type 
     task name => type do

data/lib/scout/workflow/task/dependencies.rb

@@ -19,7 +19,7 @@ module Task
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || step_inputs[v] || k
+          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || k
         else
           resolved_inputs[k] = v
         end
@@ -79,7 +79,7 @@ module Task
         when Step
           dep = res
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Hash
           step_options = block_options.merge(res)

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder, :binary].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.2.0 ruby lib
+# stub: scout-gear 10.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.2.0"
+  s.version = "10.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-07-27"
+  s.date = "2023-08-01"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -51,6 +51,7 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/persist/adapter.rb",
     "lib/scout/tsv/persist/fix_width_table.rb",
     "lib/scout/tsv/persist/serialize.rb",
+    "lib/scout/tsv/persist/tkrzw.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
     "lib/scout/tsv/stream.rb",
     "lib/scout/tsv/transformer.rb",
@@ -124,6 +125,7 @@ Gem::Specification.new do |s|
     "test/scout/test_workflow.rb",
     "test/scout/tsv/persist/test_adapter.rb",
     "test/scout/tsv/persist/test_fix_width_table.rb",
+    "test/scout/tsv/persist/test_tkrzw.rb",
     "test/scout/tsv/persist/test_tokyocabinet.rb",
     "test/scout/tsv/test_attach.rb",
     "test/scout/tsv/test_change_id.rb",
@@ -165,7 +167,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.5.0.dev".freeze
+  s.rubygems_version = "3.4.13".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/test/scout/tsv/persist/test_tkrzw.rb

@@ -0,0 +1,123 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestScoutTKRZW < Test::Unit::TestCase
+  def _test_open
+    TmpFile.with_file nil do |tmp|
+      db = ScoutTKRZW.open(tmp, true)
+      1000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+      assert_include db, 'foo1'
+      assert_equal 1000, db.keys.length
+
+      db.close
+      TmpFile.with_file nil do |tmp_alt|
+        Open.cp tmp, tmp_alt
+        db2 = ScoutTKRZW.open(tmp_alt, false)
+        assert_include db2, 'foo1'
+        assert_equal 1000, db2.keys.length
+      end
+    end
+  end
+
+  def _test_tkrzw
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      Persist.persist(__method__, 'tkh') do
+        TSV.open(filename)
+      end
+    end
+
+
+    assert_equal %w(a aa aaa), tsv["row1"][0]
+
+    assert TSVAdapter === tsv
+    assert TSV === tsv
+    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
+    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
+
+    tsv.close
+    tsv_loaded = assert_nothing_raised do
+      TmpFile.with_file(content) do |filename|
+        Persist.persist(__method__, 'tkh') do
+          raise
+        end
+      end
+    end
+
+    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
+  end
+
+  def __test_benchmark1
+    TmpFile.with_file nil do |tmp|
+
+      Misc.benchmark(1000) do
+        db = ScoutTKRZW.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        10.times do
+          db.keys
+        end
+        10.times do |i|
+          db["foo#{i}"]
+        end
+        Open.rm tmp
+      end
+
+      Misc.benchmark(1000) do
+        db = ScoutCabinet.open(tmp, true)
+        1000.times do |i|
+          db["foo#{i}"] = "bar#{i}"
+        end
+        10.times do
+          db.keys
+        end
+        10.times do |i|
+          db["foo#{i}"]
+        end
+        db.close
+        Open.rm tmp
+      end
+    end
+  end
+
+  def test_benchmark2
+    TmpFile.with_file nil do |tmp|
+
+      db = ScoutTKRZW.open(tmp, true)
+      10000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+
+      Misc.benchmark(1000) do
+        100.times do |i|
+          db["foo#{i}"]
+        end
+      end
+
+      Open.rm tmp
+      db = ScoutCabinet.open(tmp, true)
+      10000.times do |i|
+        db["foo#{i}"] = "bar#{i}"
+      end
+      Misc.benchmark(1000) do
+        100.times do |i|
+          db["foo#{i}"]
+        end
+      end
+    end
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.2.0
+  version: 10.3.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-27 00:00:00.000000000 Z
+date: 2023-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -163,6 +163,7 @@ files:
 - lib/scout/tsv/persist/adapter.rb
 - lib/scout/tsv/persist/fix_width_table.rb
 - lib/scout/tsv/persist/serialize.rb
+- lib/scout/tsv/persist/tkrzw.rb
 - lib/scout/tsv/persist/tokyocabinet.rb
 - lib/scout/tsv/stream.rb
 - lib/scout/tsv/transformer.rb
@@ -236,6 +237,7 @@ files:
 - test/scout/test_workflow.rb
 - test/scout/tsv/persist/test_adapter.rb
 - test/scout/tsv/persist/test_fix_width_table.rb
+- test/scout/tsv/persist/test_tkrzw.rb
 - test/scout/tsv/persist/test_tokyocabinet.rb
 - test/scout/tsv/test_attach.rb
 - test/scout/tsv/test_change_id.rb
@@ -293,7 +295,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.0.dev
+rubygems_version: 3.4.13
 signing_key: 
 specification_version: 4
 summary: basic gear for scouts