scout-gear 10.4.0 → 10.6.1

data/test/scout/workflow/test_definition.rb

@@ -14,5 +14,57 @@ class TestWorkflowDefinition < Test::Unit::TestCase
     end
     assert_equal 5, wf.job(:str_length, :string => "12345").run
   end
+
+  def test_as_jobname
+    wf = Workflow.annonymous_workflow do
+      self.name = "CallName"
+      input :name, :string, "Name to call", nil, :jobname => true
+      task :call_name => :string do |name|
+        "Hi #{name}"
+      end
+    end
+    assert_equal "Hi Miguel", wf.job(:call_name, "Miguel").run
+    assert_equal "Hi Cleia", wf.job(:call_name, "Miguel", name: "Cleia").run
+    assert_equal "Miguel", wf.job(:call_name, "Miguel", name: "Cleia").clean_name
+    assert_equal "Cleia", wf.job(:call_name, name: "Cleia").clean_name
+  end
+
+  def test_task_alias
+    wf = Workflow.annonymous_workflow do
+      self.name = "CallName"
+      input :name, :string, "Name to call", nil, :jobname => true
+      task :call_name => :string do |name|
+        "Hi #{name}"
+      end
+
+      task_alias :call_miguel, self, :call_name, name: "Miguel"
+    end
+
+    job = wf.job(:call_miguel)
+    assert_equal "Hi Miguel", job.run
+  end
+
+  def test_task_alias_remove_dep
+    wf = Workflow.annonymous_workflow do
+      self.name = "CallName"
+      input :name, :string, "Name to call", nil, :jobname => true
+      task :call_name => :string do |name|
+        "Hi #{name}"
+      end
+
+      task_alias :call_miguel, self, :call_name, name: "Miguel"
+    end
+
+    old_cache = Scout::Config::CACHE.dup
+    Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => true}, 'task:CallName#call_miguel') 
+    job = wf.job(:call_miguel)
+    dep_path = job.step(:call_name).path
+    assert_equal "Hi Miguel", job.run
+    refute job.dependencies.any?
+    refute Open.exist?(dep_path)
+    Scout::Config::CACHE.replace old_cache
+    assert_include job.archived_info, dep_path
+    assert_equal :done, job.archived_info[dep_path][:status]
+  end
 end
 

data/test/scout/workflow/test_step.rb

@@ -31,6 +31,29 @@ class TestWorkflowStep < Test::Unit::TestCase
     assert_equal "12 has 2 characters", step2.run
   end
 
+  def test_dependency_load
+    tmpfile = tmpdir
+    step1 = Step.new tmpdir.test_task1.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpdir.test_task2.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    step2.run
+
+    assert_equal "12 has 2 characters", step2.run
+
+    new_step = Step.new step2.path
+
+    assert_equal "12 has 2 characters", new_step.run
+    assert_equal 2, new_step.step(:test_task1).run
+  end
+
   def test_streaming
     tmpfile = tmpdir.test_step
 
@@ -58,6 +81,11 @@ class TestWorkflowStep < Test::Unit::TestCase
     step1.join
     step1.clean
 
+    step1.clean
+    res =  step1.run(:no_load)
+    assert res.nil?
+    step1.clean
+
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
       stream = step1.stream
@@ -95,6 +123,7 @@ class TestWorkflowStep < Test::Unit::TestCase
       lines << line
     end
 
+    stream.join
     assert step1.path.read.end_with? "line-#{times-1}\n"
     assert_equal times/2, lines.length
     assert_equal times/2, step2.join.path.read.split("\n").length
@@ -338,4 +367,32 @@ class TestWorkflowStep < Test::Unit::TestCase
     assert_equal "12 has 2 characters", step2.run
   end
 
+  def test_semaphore
+
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1 do |s|
+      sleep 2
+      "done1"
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      sleep 2
+      "done2"
+    end
+
+    step1.dependencies = []
+    step2.dependencies = []
+
+    ScoutSemaphore.with_semaphore(1) do |sem|
+      step1.fork(false, sem)
+      step2.fork(false, sem)
+      sleep 1
+      assert((step1.status == :queue) || (step2.status == :queue))
+      step1.join
+      step2.join
+      assert_equal "done2", step2.run
+    end
+
+  end
+
 end

data/test/scout/workflow/test_task.rb

@@ -21,5 +21,30 @@ class TestTask < Test::Unit::TestCase
     assert_equal 2, s.run
   end
 
-end
+  def __test_benchmark
+    tasks = []
+    wf = Module.new do
+      extend Workflow
+      self.name = "TestWF"
+
+      500.times do |i|
+        task_name = "task_#{i}"
+        last_task_name = "task_#{i-1}"
+        if i == 0
+          task task_name => :array do
+            [task_name]
+          end
+        else
+          dep last_task_name
+          task task_name => :array do
+            step(last_task_name).load.push(task_name)
+          end
+        end
+      end
+    end
 
+    Misc.benchmark(1000) do
+      wf.job(:task_499)
+    end
+  end
+end

data/test/scout/workflow/test_usage.rb

@@ -53,10 +53,10 @@ class TestWorkflowUsage < Test::Unit::TestCase
   end
 
   def test_task_input_fixed
-    assert_match /Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
-    refute_match /--array/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow)
-    assert_match /Desc3/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
-    refute_match /--array/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow)
+    assert_match(/Desc2_fixed/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow))
+    refute_match(/--array/, UsageWorkflow.tasks[:step2_fixed].usage(UsageWorkflow))
+    assert_match(/Desc3/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow))
+    refute_match(/--array/, UsageWorkflow.tasks[:step3_fixed].usage(UsageWorkflow))
   end
 end
 

data/test/test_helper.rb

@@ -32,11 +32,16 @@ class Test::Unit::TestCase
     assert_equal File.expand_path(path1), File.expand_path(path2)
   end
 
+  def self.tmpdir
+    @@tmpdir ||= Path.setup('tmp/test_tmpdir').find
+  end
+
   def tmpdir
-    @tmpdir ||= Path.setup('tmp/test_tmpdir').find
+    @tmpdir ||= Test::Unit::TestCase.tmpdir
   end
 
   setup do
+    Open.rm_rf tmpdir
     TmpFile.tmpdir = tmpdir.tmpfiles
     Log::ProgressBar.default_severity = 0
     Persist.cache_dir = tmpdir.var.cache
@@ -44,9 +49,26 @@ class Test::Unit::TestCase
     Open.remote_cache_dir = tmpdir.var.cache
     Workflow.directory = tmpdir.var.jobs
     Workflow.workflows.each{|wf| wf.directory = Workflow.directory[wf.name] }
+    Entity.entity_property_cache = tmpdir.entity_properties if defined?(Entity)
   end
   
   teardown do
     Open.rm_rf tmpdir
   end
+
+  def self.datadir_test
+    Path.setup(File.join(File.dirname(__FILE__), 'data'))
+  end
+
+  def self.datafile_test(file)
+    datadir_test[file.to_s]
+  end
+
+  def datadir_test
+    Test::Unit::TestCase.datadir_test
+  end
+
+  def datafile_test(file)
+    Test::Unit::TestCase.datafile_test(file)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.4.0
+  version: 10.6.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-09 00:00:00.000000000 Z
+date: 2024-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: scout-essentials
@@ -145,26 +145,50 @@ files:
 - lib/rbbt-scout.rb
 - lib/scout-gear.rb
 - lib/scout.rb
+- lib/scout/association.rb
+- lib/scout/association/fields.rb
+- lib/scout/association/index.rb
+- lib/scout/association/item.rb
+- lib/scout/association/util.rb
+- lib/scout/entity.rb
+- lib/scout/entity/format.rb
+- lib/scout/entity/identifiers.rb
+- lib/scout/entity/object.rb
+- lib/scout/entity/property.rb
 - lib/scout/offsite.rb
 - lib/scout/offsite/exceptions.rb
 - lib/scout/offsite/ssh.rb
 - lib/scout/offsite/step.rb
 - lib/scout/offsite/sync.rb
+- lib/scout/persist/engine.rb
+- lib/scout/persist/engine/fix_width_table.rb
+- lib/scout/persist/engine/packed_index.rb
+- lib/scout/persist/engine/sharder.rb
+- lib/scout/persist/engine/tkrzw.rb
+- lib/scout/persist/engine/tokyocabinet.rb
+- lib/scout/persist/tsv.rb
+- lib/scout/persist/tsv/adapter.rb
+- lib/scout/persist/tsv/adapter/base.rb
+- lib/scout/persist/tsv/adapter/fix_width_table.rb
+- lib/scout/persist/tsv/adapter/packed_index.rb
+- lib/scout/persist/tsv/adapter/sharder.rb
+- lib/scout/persist/tsv/adapter/tkrzw.rb
+- lib/scout/persist/tsv/adapter/tokyocabinet.rb
+- lib/scout/persist/tsv/serialize.rb
 - lib/scout/semaphore.rb
 - lib/scout/tsv.rb
+- lib/scout/tsv/annotation.rb
+- lib/scout/tsv/annotation/repo.rb
 - lib/scout/tsv/attach.rb
 - lib/scout/tsv/change_id.rb
+- lib/scout/tsv/change_id/translate.rb
+- lib/scout/tsv/csv.rb
 - lib/scout/tsv/dumper.rb
+- lib/scout/tsv/entity.rb
 - lib/scout/tsv/index.rb
 - lib/scout/tsv/open.rb
 - lib/scout/tsv/parser.rb
 - lib/scout/tsv/path.rb
-- lib/scout/tsv/persist.rb
-- lib/scout/tsv/persist/adapter.rb
-- lib/scout/tsv/persist/fix_width_table.rb
-- lib/scout/tsv/persist/serialize.rb
-- lib/scout/tsv/persist/tkrzw.rb
-- lib/scout/tsv/persist/tokyocabinet.rb
 - lib/scout/tsv/stream.rb
 - lib/scout/tsv/transformer.rb
 - lib/scout/tsv/traverse.rb
@@ -184,8 +208,11 @@ files:
 - lib/scout/workflow/definition.rb
 - lib/scout/workflow/deployment.rb
 - lib/scout/workflow/deployment/orchestrator.rb
+- lib/scout/workflow/deployment/trace.rb
 - lib/scout/workflow/documentation.rb
 - lib/scout/workflow/step.rb
+- lib/scout/workflow/step/archive.rb
+- lib/scout/workflow/step/children.rb
 - lib/scout/workflow/step/config.rb
 - lib/scout/workflow/step/dependencies.rb
 - lib/scout/workflow/step/file.rb
@@ -226,26 +253,53 @@ files:
 - share/software/install_helpers
 - share/templates/command
 - share/templates/workflow.rb
+- test/data/person/brothers
+- test/data/person/identifiers
+- test/data/person/marriages
+- test/data/person/parents
+- test/scout/association/test_fields.rb
+- test/scout/association/test_index.rb
+- test/scout/association/test_item.rb
+- test/scout/entity/test_format.rb
+- test/scout/entity/test_identifiers.rb
+- test/scout/entity/test_object.rb
+- test/scout/entity/test_property.rb
 - test/scout/offsite/test_ssh.rb
 - test/scout/offsite/test_step.rb
 - test/scout/offsite/test_sync.rb
 - test/scout/offsite/test_task.rb
+- test/scout/persist/engine/test_fix_width_table.rb
+- test/scout/persist/engine/test_packed_index.rb
+- test/scout/persist/engine/test_sharder.rb
+- test/scout/persist/engine/test_tkrzw.rb
+- test/scout/persist/engine/test_tokyocabinet.rb
+- test/scout/persist/test_tsv.rb
+- test/scout/persist/tsv/adapter/test_base.rb
+- test/scout/persist/tsv/adapter/test_fix_width_table.rb
+- test/scout/persist/tsv/adapter/test_packed_index.rb
+- test/scout/persist/tsv/adapter/test_serialize.rb
+- test/scout/persist/tsv/adapter/test_sharder.rb
+- test/scout/persist/tsv/adapter/test_tkrzw.rb
+- test/scout/persist/tsv/adapter/test_tokyocabinet.rb
+- test/scout/persist/tsv/test_serialize.rb
+- test/scout/test_association.rb
+- test/scout/test_entity.rb
 - test/scout/test_offsite.rb
 - test/scout/test_semaphore.rb
 - test/scout/test_tsv.rb
 - test/scout/test_work_queue.rb
 - test/scout/test_workflow.rb
-- test/scout/tsv/persist/test_adapter.rb
-- test/scout/tsv/persist/test_fix_width_table.rb
-- test/scout/tsv/persist/test_tkrzw.rb
-- test/scout/tsv/persist/test_tokyocabinet.rb
+- test/scout/tsv/annotation/test_repo.rb
+- test/scout/tsv/change_id/test_translate.rb
+- test/scout/tsv/test_annotation.rb
 - test/scout/tsv/test_attach.rb
 - test/scout/tsv/test_change_id.rb
+- test/scout/tsv/test_csv.rb
 - test/scout/tsv/test_dumper.rb
+- test/scout/tsv/test_entity.rb
 - test/scout/tsv/test_index.rb
 - test/scout/tsv/test_open.rb
 - test/scout/tsv/test_parser.rb
-- test/scout/tsv/test_persist.rb
 - test/scout/tsv/test_stream.rb
 - test/scout/tsv/test_transformer.rb
 - test/scout/tsv/test_traverse.rb
@@ -260,6 +314,9 @@ files:
 - test/scout/work_queue/test_socket.rb
 - test/scout/work_queue/test_worker.rb
 - test/scout/workflow/deployment/test_orchestrator.rb
+- test/scout/workflow/deployment/test_trace.rb
+- test/scout/workflow/step/test_archive.rb
+- test/scout/workflow/step/test_children.rb
 - test/scout/workflow/step/test_dependencies.rb
 - test/scout/workflow/step/test_info.rb
 - test/scout/workflow/step/test_load.rb
@@ -295,7 +352,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.0.dev
+rubygems_version: 3.5.10
 signing_key: 
 specification_version: 4
 summary: basic gear for scouts

data/lib/scout/tsv/persist.rb

@@ -1,27 +0,0 @@
-require 'scout/persist'
-require_relative 'persist/adapter'
-
-begin
-  require_relative 'persist/tokyocabinet'
-rescue Exception
-end
-
-begin
-  require_relative 'persist/tkrzw'
-rescue Exception
-end
-
-Persist.save_drivers[:tsv] = proc do |file,content| 
-  stream = if IO === content
-             content
-           elsif content.respond_to?(:stream)
-             content.stream
-           elsif content.respond_to?(:dumper_stream)
-             content.dumper_stream
-           else
-             content
-           end
-  Open.sensible_write(file, stream)
-end
-
-Persist.load_drivers[:tsv] = proc do |file| TSV.open file end

data/test/scout/tsv/persist/test_tokyocabinet.rb

@@ -1,120 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-require 'scout/tsv'
-
-class TestTSVTokyo < Test::Unit::TestCase
-  def test_tokyo
-    content =<<-'EOF'
-#: :sep=/\s+/#:type=:double#:merge=:concat
-#Id    ValueA    ValueB    OtherID
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-row2    a    a    id3
-    EOF
-
-    tsv = TmpFile.with_file(content) do |filename|
-      Persist.persist(__method__, :HDB) do
-        TSV.open(filename)
-      end
-    end
-
-    assert_equal %w(a aa aaa), tsv["row1"][0]
-
-    assert TSVAdapter === tsv
-    assert TSV === tsv
-    assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
-    assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
-
-    tsv_loaded = assert_nothing_raised do
-      TmpFile.with_file(content) do |filename|
-        Persist.persist(__method__, :HDB) do
-          raise
-        end
-      end
-    end
-
-
-    assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
-  end
-
-  def test_custom_load
-    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
-
-    size = 100_000
-    (0..size).each do |i|
-      k = "key-#{i}"
-      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
-      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
-
-      tsv[k] = [values1, values2]
-    end
-
-    tc = Persist.persist(__method__, :HDB) do |file|
-      tsv
-    end
-
-    100.times do
-      i = rand(size).floor
-      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
-    end
-  end
-
-  def __test_speed
-    tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
-
-    size = 100_000
-    (0..size).each do |i|
-      k = "key-#{i}"
-      values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
-      values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
-
-      tsv[k] = [values1, values2]
-    end
-
-    tc = Persist.persist(__method__, :HDB) do |file|
-      data = ScoutCabinet.open(file, true, "HDB")
-      TSV.setup(data, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
-      data.extend TSVAdapter
-      Log::ProgressBar.with_bar size do |b|
-        (0..size).each do |i|
-          b.tick
-          k = "key-#{i}"
-          values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
-          values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
-
-          data[k] = [values1, values2]
-        end
-      end
-      data
-    end
-
-    100.times do
-      i = rand(size).floor
-      assert_equal tc["key-#{i}"], tsv["key-#{i}"]
-    end
-  end
-
-  def test_float_array
-    content =<<-EOF
-#Id    ValueA    ValueB    OtherID
-row1   0.2   0.3 0
-row2    0.1  4.5 0
-    EOF
-
-    TmpFile.with_file(content) do |filename|
-      tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
-      tsv.save_extension_attr_hash
-      assert_equal [0.2, 0.3, 0], tsv["row1"]
-      assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
-      Open.cp tsv.persistence_path, tmpdir.persistence.foo
-      tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
-      tsv2.extend TSVAdapter
-      assert_equal [0.2, 0.3, 0], tsv2["row1"]
-      assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
-    end
- 
-  end
-
-end
-

data/test/scout/tsv/test_persist.rb

@@ -1,45 +0,0 @@
-require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
-require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
-
-require 'scout/tsv'
-
-class TestTSVPersist < Test::Unit::TestCase
-  def test_persist
-    content =<<-'EOF'
-#: :sep=/\s+/#:type=:double#:merge=:concat
-#Id    ValueA    ValueB    OtherID
-row1    a|aa|aaa    b    Id1|Id2
-row2    A    B    Id3
-row2    a    a    id3
-    EOF
-
-
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
-      TmpFile.with_file(content) do |filename|
-        TSV.open(filename)
-      end
-    end
-
-    assert_include tsv.keys, 'row1'
-    assert_include tsv.keys, 'row2'
-
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
-      TmpFile.with_file(content) do |filename|
-        TSV.open(filename)
-      end
-    end
-
-    assert_include tsv.keys, 'row1'
-    assert_include tsv.keys, 'row2'
-
-    assert_nothing_raised do
-      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
-        raise
-      end
-    end
-
-    assert_include tsv.keys, 'row1'
-    assert_include tsv.keys, 'row2'
-  end
-end
-