scout-gear 10.1.0 → 10.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab089af23ec3cb6d23db1190f712d2a981db96e818572aa5b561227a9601b9b8
-  data.tar.gz: d72662def6d6b7207d015d7cea613a1882578efe201ed391f79d37027ab7c70e
+  metadata.gz: 2a7adf9c59ab3ce69e89dd2e02089b5287605a1b74c6a435a5bfa4ece17c1f75
+  data.tar.gz: 65f9fa08d1b4a99e973d33eb5a7c0ddab85a98b5d6a675f8a2f5cfe381e664aa
 SHA512:
-  metadata.gz: d5b67fdc030ad2ed3dd4d9da47d7f12068b71f14a1ae390f65c411bc2e835ac2da1648fccb554cbda4a3a56bdf6b16fc63080620453e75481590622cfd883247
-  data.tar.gz: 97fa38ef8db895d3a8aa05d2e4e521bb98d28924b389429994f6e98bceb7bfa7ad74920676c17b8b124bf288700e80078909ea20a84c3c0b24efd0cf7b0632cb
+  metadata.gz: cd78580f6d0fd2a417f15f425602131d4e3150fc8a0f35a1d2f63f5868e29cae8a9f59244a2fdc5d642441051b8528bcd2f8596aaf4bc11414437e6d4dc4cd14
+  data.tar.gz: ad61b8616e5029ff0ee748b48ef78928dc862bfbb0ba763610ad9f89f7e865975d816318ea388d30d774f1088894b9c2a3d42487b9a3c01c795c45cd5b37f2d7

data/.vimproject

@@ -8,6 +8,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   alias
   find
   glob
+  log
   doc
   update
   template
@@ -16,7 +17,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    task
    list
    info
+   write_info
    install
+   trace
+   prov
+  }
+  batch=batch{
+   list
+   clean
   }
   resource=resource{
    produce
@@ -72,6 +80,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      unzip.rb
      reorder.rb
      sort.rb
+     melt.rb
     }
     parser.rb
     dumper.rb
@@ -80,9 +89,8 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     persist=persist{
      adapter.rb
      serialize.rb
-     sdbm.rb
-     tkrzw.rb
      tokyocabinet.rb
+     tkrzw.rb
      fix_width_table.rb
     }
     index.rb
@@ -122,6 +130,10 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    test_tsv.rb
    test_work_queue.rb
    test_workflow.rb
+   work_queue=work_queue{
+    test_socket.rb
+    test_worker.rb
+   }
    workflow=workflow{
     test_definition.rb
     test_documentation.rb
@@ -140,10 +152,42 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      test_dependencies.rb
      test_inputs.rb
     }
+    deployment=deployment{
+     test_orchestrator.rb
+    }
    }
-   indiferent_hash=indiferent_hash{
-    test_case_insensitive.rb
-    test_options.rb
+   tsv=tsv{
+    test_attach.rb
+    test_change_id.rb
+    test_dumper.rb
+    test_index.rb
+    test_open.rb
+    test_parser.rb
+    test_persist.rb
+    test_stream.rb
+    test_transformer.rb
+    test_traverse.rb
+    test_util.rb
+    util=util{
+     test_filter.rb
+     test_melt.rb
+     test_process.rb
+     test_reorder.rb
+     test_select.rb
+     test_sort.rb
+     test_unzip.rb
+    }
+    persist=persist{
+     test_adapter.rb
+     test_fix_width_table.rb
+     test_tokyocabinet.rb
+    }
+   }
+   offsite=offsite{
+    test_ssh.rb
+    test_step.rb
+    test_sync.rb
+    test_task.rb
    }
   }
  }
@@ -524,6 +568,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
       test_filter.rb
       test_index.rb
       test_manipulate.rb
+      test_marshal.rb
       test_matrix.rb
       test_parallel.rb
       test_parser.rb

data/VERSION

@@ -1 +1 @@
-10.1.0
+10.3.0

data/lib/scout/tsv/persist/tkrzw.rb

@@ -0,0 +1,84 @@
+require 'tkrzw'
+require_relative 'adapter'
+
+module ScoutTKRZW
+  attr_accessor :persistence_path, :persistence_class, :open_options
+
+  def self.open(path, write = true, persistence_class = 'tkh', options = {})
+    open_options = IndiferentHash.add_defaults options, truncate: true, num_buckets: 100, dbm: "HashDBM", sync_hard: true, encoding: "UTF-8"
+  
+    path = path.find if Path === path
+
+    dir = File.dirname(File.expand_path(path))
+    Open.mkdir(dir) unless File.exist?(dir)
+
+    database = Persist::CONNECTIONS[[persistence_class, path]*":"] ||= Tkrzw::DBM.new
+
+    database.close if database.open?
+
+    database.open(path, write, open_options)
+
+    database.extend ScoutTKRZW
+    database.persistence_path ||= path
+    database.open_options = open_options
+
+    #Persist::CONNECTIONS[[persistence_class, path]*":"] = database
+
+    database
+  end
+
+  def close
+    @closed = true
+    @writable = false
+    super
+  end
+
+  def read(force = false)
+    return if open? && ! writable? && ! force
+    self.close if open?
+    if !self.open(@persistence_path, false, @open_options)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = false
+    @closed = false
+
+    self
+  end
+
+  def write(force = true)
+    return if open? && writable? && ! force
+    self.close if self.open?
+
+    if !self.open(@persistence_path, true, @open_options)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = true
+    @closed = false
+
+    self
+  end
+
+  def keys
+    search("contain", "")
+  end
+end
+
+Persist.save_drivers[:tkh] = proc do |file, content|
+  data = ScoutTKRZW.open(file, true, "tkh")
+  content.annotate(data)
+  data.extend TSVAdapter
+  data.merge!(content)
+  data.close
+  data.read
+  data
+end
+
+Persist.load_drivers[:tkh] = proc do |file| 
+  data = ScoutTKRZW.open(file, false, "tkh")
+  data.extend TSVAdapter unless TSVAdapter === data
+  data
+end

data/lib/scout/tsv/persist.rb

@@ -1,6 +1,15 @@
 require 'scout/persist'
 require_relative 'persist/adapter'
-require_relative 'persist/tokyocabinet'
+
+begin
+  require_relative 'persist/tokyocabinet'
+rescue
+end
+
+begin
+  require_relative 'persist/tkrzw'
+rescue
+end
 
 Persist.save_drivers[:tsv] = proc do |file,content| 
   stream = if IO === content

data/lib/scout/tsv/transformer.rb

@@ -2,15 +2,16 @@ module TSV
   class Transformer
     attr_accessor :unnamed, :parser, :dumper
 
-    def initialize(parser, dumper = nil, unnamed: false)
+    def initialize(parser, dumper = nil, unnamed: nil)
       if TSV::Parser === parser
         @parser = parser
       elsif TSV === parser
         @parser = parser
+        @unnamed = parser.unnamed
       else
         @parser = TSV::Parser.new parser
       end
-      @unnamed = unnamed
+      @unnamed = unnamed unless unnamed.nil?
       if dumper.nil?
         @dumper = TSV::Dumper.new(@parser)
         @dumper.sep = "\t"
@@ -70,7 +71,7 @@ module TSV
       @dumper.init if @dumper.respond_to?(:init) && ! @dumper.initialized
       Log.debug "Transform #{Log.fingerprint @parser} into #{Log.fingerprint @dumper}"
       Open.traverse(@parser, *args, **kwargs) do |k,v|
-        NamedArray.setup(v, @parser.fields, k) unless @unnamed
+        NamedArray.setup(v, @parser.fields, k) unless @unnamed || @parser.fields.nil?
         block.call k, v
       end
     end
@@ -112,16 +113,18 @@ module TSV
 
   def to_list
     res = self.annotate({})
-    transformer = Transformer.new self, res
-    transformer.type = :list
-    transformer.traverse do |k,v|
-      case self.type
-      when :single
-        [k, [v]]
-      when :double
-        [k, v.collect{|v| v.first }]
-      when :flat
-        [k, v.slice(0,1)]
+    self.with_unnamed do
+      transformer = Transformer.new self, res
+      transformer.type = :list
+      transformer.traverse do |k,v|
+        case self.type
+        when :single
+          [k, [v]]
+        when :double
+          [k, v.collect{|v| v.first }]
+        when :flat
+          [k, v.slice(0,1)]
+        end
       end
     end
     res
@@ -131,6 +134,7 @@ module TSV
     res = self.annotate({})
     transformer = Transformer.new self, res
     transformer.type = :single
+    transformer.unnamed = true
     transformer.traverse do |k,v|
       v = v.first while Array === v
       [k, v]

data/lib/scout/tsv/util/melt.rb

@@ -0,0 +1,13 @@
+module TSV
+  def melt_columns(value_field, column_field)
+    target = TSV.setup({}, :key_field => "ID", :fields => [key_field, value_field, column_field], :type => :list, :cast => cast)
+    each do |k,values|
+      i = 0
+      values.zip(fields).each do |v,f|
+        target["#{k}:#{i}"] = [k,v,f]
+        i+=1
+      end
+    end
+    target
+  end
+end

data/lib/scout/tsv/util.rb

@@ -7,6 +7,7 @@ require_relative 'util/select'
 require_relative 'util/reorder'
 require_relative 'util/unzip'
 require_relative 'util/sort'
+require_relative 'util/melt'
 module TSV
   def self.identify_field(key_field, fields, name, strict: nil)
     return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))

data/lib/scout/tsv.rb

@@ -34,11 +34,23 @@ module TSV
   end
 
   def self.open(file, options = {})
-    persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
-    type = type.to_sym if type
+    grep, invert_grep = IndiferentHash.process_options options, :grep, :invert_grep, :persist => false
+
+    persist_options = IndiferentHash.pull_keys options, :persist
+    persist_options = IndiferentHash.add_defaults persist_options, :prefix => "TSV", :type => :HDB
+
     file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
-    Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
-      data = filename ? ScoutCabinet.open(filename, true, type) : nil
+    Persist.persist(file, persist_options[:type], persist_options.merge(:other_options => options)) do |filename|
+      if filename
+        data = case persist_options[:type]
+               when :HDB, :BDB
+                 ScoutCabinet.open(filename, true, persist_options[:type])
+               when :tkh, :tkt, :tks
+                 ScoutTKRZW.open(filename, true, persist_options[:type])
+               end
+      else
+        data = nil
+      end
       options[:data] = data if data
       options[:filename] = file
 

data/lib/scout/workflow/definition.rb

@@ -84,8 +84,10 @@ module Workflow
     annotate_next_task :deps, [workflow, task, options, block, args]
   end
 
-  def input(*args)
-    annotate_next_task(:inputs, args)
+  def input(name, type = nil, *rest)
+    name = name.to_sym
+    type = type.to_sym if type
+    annotate_next_task(:inputs, [name, type] + rest)
   end
 
   def desc(description)
@@ -102,8 +104,10 @@ module Workflow
 
   def task(name_and_type, &block)
     name, type = name_and_type.collect.first
+    type = type.to_sym if String === type
+    name = name.to_sym if String === name
     @tasks ||= IndiferentHash.setup({})
-    block = self.method(name) if block.nil?
+    block = lambda &self.method(name) if block.nil?
     begin
       @annotate_next_task ||= {}
       @annotate_next_task[:extension] ||=  
@@ -132,8 +136,10 @@ module Workflow
   def task_alias(name, workflow, oname, *rest, &block)
     dep(workflow, oname, *rest, &block) 
     extension :dep_task unless @extension
-    returns workflow.tasks[oname].returns if @returns.nil?
-    type = workflow.tasks[oname].type 
+    task_proc = workflow.tasks[oname]
+    raise "Task #{oname} not found" if task_proc.nil?
+    returns task_proc.returns if @returns.nil?
+    type = task_proc.type 
     task name => type do
       raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
       Step.wait_for_jobs dependencies.select{|d| d.streaming? }

data/lib/scout/workflow/step/dependencies.rb

@@ -5,7 +5,7 @@ class Step
       next if seen.include? dep.path
       next if connected && dep.done? && dep.updated?
       direct_deps << dep
-    end
+    end if dependencies
     seen.concat direct_deps.collect{|d| d.path }
     direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
   end

data/lib/scout/workflow/step/file.rb

@@ -12,9 +12,10 @@ class Step
                    end
   end
 
-  def file(file)
+  def file(file = nil)
     dir = files_dir
     Path.setup(dir) unless Path === dir
+    return dir if file.nil?
     dir[file]
   end
 

data/lib/scout/workflow/step/provenance.rb

@@ -68,7 +68,7 @@ class Step
       $inputs.each do |input|
         value = job_inputs[input]
         next if  value.nil?
-        value_str = Misc.fingerprint(value)
+        value_str = Log.fingerprint(value)
         str << "\t#{Log.color :magenta, input}=#{value_str}"
       end
     end
@@ -78,7 +78,7 @@ class Step
         IndiferentHash.setup(info)
         value = info[field]
         next if value.nil?
-        value_str = Misc.fingerprint(value)
+        value_str = Log.fingerprint(value)
         str << "\t#{Log.color :magenta, field}=#{value_str}"
       end
     end

data/lib/scout/workflow/step/status.rb

@@ -27,10 +27,10 @@ class Step
     @result = nil
     @info = nil
     @info_load_time = nil
-    Open.rm path if Open.exist?(path)
-    Open.rm tmp_path if Open.exist?(tmp_path)
-    Open.rm info_file if Open.exist?(info_file)
-    Open.rm_rf files_dir if Open.exist?(files_dir)
+    Open.rm path if Open.exist_or_link?(path)
+    Open.rm tmp_path if Open.exist_or_link?(tmp_path)
+    Open.rm info_file if Open.exist_or_link?(info_file)
+    Open.rm_rf files_dir if Open.exist_or_link?(files_dir)
     self
   end
 

data/lib/scout/workflow/step.rb

@@ -76,10 +76,14 @@ class Step
 
   def task_name
     @task_name ||= @task.name if @task.respond_to?(:name)
+    @task_name ||= info[:task_name] if Open.exist?(info_file)
+    @task_name ||= path.split("/")[-2]
   end
 
   def workflow
-    @task.workflow if @task
+    @workflow ||= @task.workflow if Task === @task
+    @workflow ||= info[:workflow] if Open.exist?(info_file)
+    @workflow ||= path.split("/")[-3]
   end
 
   def exec
@@ -130,6 +134,7 @@ class Step
         input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
         merge_info :status => :start, :start => Time.now,
           :pid => Process.pid, :pid_hostname => Misc.hostname, 
+          :task_name => task_name, :workflow => workflow.to_s,
           :inputs => MetaExtension.purge(inputs), :input_names => input_names, :type => type,
           :dependencies => dependencies.collect{|d| d.path }
 
@@ -232,11 +237,14 @@ class Step
     while @result && streaming? && stream = self.stream
       threads << Open.consume_stream(stream, true)
     end
+
+    threads.compact!
+
     threads.each do |t| 
       begin
         t.join 
       rescue Exception
-        threads.each{|t| t.raise(Aborted); t.join }
+        threads.compact.each{|t| t.raise(Aborted); t.join }
         raise $!
       end
     end
@@ -252,6 +260,7 @@ class Step
     while ! present?
       sleep 0.1
     end
+    self
   end
 
   def terminated?

data/lib/scout/workflow/task/dependencies.rb

@@ -19,7 +19,7 @@ module Task
       step_inputs.each do |k,v|
         if Symbol === v
           input_dep = dependencies.select{|d| d.task_name == v }.first
-          resolved_inputs[k] = input_dep || step_inputs[v] || k
+          resolved_inputs[k] = input_dep || provided_inputs[v] || step_inputs[v] || k
         else
           resolved_inputs[k] = v
         end
@@ -79,7 +79,7 @@ module Task
         when Step
           dep = res
           dependencies << dep
-          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
+          dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options)
           non_default_inputs.concat(dep_non_default_inputs)
         when Hash
           step_options = block_options.merge(res)

data/lib/scout/workflow/task/inputs.rb

@@ -3,7 +3,7 @@ module Task
   def self.format_input(value, type, options = {})
     return value if IO === value || StringIO === value || Step === value
 
-    if String === value && ! [:path, :file, :folder, :binary].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
+    if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options &&  (options[:noload] || options[:stream] || options[:nofile]))
       if Open.exists?(value) && ! Open.directory?(value)
         Persist.load(value, type) 
       else
@@ -85,8 +85,12 @@ module Task
       input_file = File.join(directory, name.to_s)
 
       if Path.is_filename?(value) 
-        relative_file = save_file_input(value, directory)
-        Open.write(input_file + ".as_file", relative_file)
+        if type == :path
+          Open.write(input_file + ".as_path", value)
+        else
+          relative_file = save_file_input(value, directory)
+          Open.write(input_file + ".as_file", relative_file)
+        end
       elsif Step === value
         Open.write(input_file + ".as_step", value.short_path)
       elsif type == :file

data/lib/scout/workflow/usage.rb

@@ -52,8 +52,9 @@ module Task
         inputs = task.inputs.reject{|name, _| seen.include? name }
         inputs = task.inputs.reject{|name, _| options.include? name }
         next unless inputs.any?
-        task.inputs.select{|name, _| inputs.include? name }.each do |name,_,_,_,options|
-          selects << [i, options[:select_options]] if options[:select_options]
+        input_names = inputs.collect{|name,_| name }
+        task.inputs.select{|name,_| input_names.include? name }.each do |name,_,_,_,options|
+          selects << [name, options[:select_options]] if options && options[:select_options]
         end
 
         dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
@@ -291,6 +292,7 @@ module Workflow
       prov_tree = prov_tree(dep_tree)
       if prov_tree && ! prov_tree.empty? && prov_tree.split("\n").length > 2
 
+        str.puts
         str.puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
         str.puts
         prov_tree.split("\n").each do |line|

data/lib/scout/workflow/util.rb

@@ -7,5 +7,10 @@ module Workflow
     mod.instance_eval(&block)
     mod
   end
+
+  def find_in_dependencies(name, dependencies)
+    name = name.to_sym
+    dependencies.select{|dep| dep.task_name.to_sym == name }
+  end
 end
 

data/scout-gear.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.1.0 ruby lib
+# stub: scout-gear 10.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.1.0"
+  s.version = "10.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Miguel Vazquez".freeze]
-  s.date = "2023-07-07"
+  s.date = "2023-08-01"
   s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
   s.email = "mikisvaz@gmail.com".freeze
   s.executables = ["scout".freeze]
@@ -51,12 +51,14 @@ Gem::Specification.new do |s|
     "lib/scout/tsv/persist/adapter.rb",
     "lib/scout/tsv/persist/fix_width_table.rb",
     "lib/scout/tsv/persist/serialize.rb",
+    "lib/scout/tsv/persist/tkrzw.rb",
     "lib/scout/tsv/persist/tokyocabinet.rb",
     "lib/scout/tsv/stream.rb",
     "lib/scout/tsv/transformer.rb",
     "lib/scout/tsv/traverse.rb",
     "lib/scout/tsv/util.rb",
     "lib/scout/tsv/util/filter.rb",
+    "lib/scout/tsv/util/melt.rb",
     "lib/scout/tsv/util/process.rb",
     "lib/scout/tsv/util/reorder.rb",
     "lib/scout/tsv/util/select.rb",
@@ -89,9 +91,12 @@ Gem::Specification.new do |s|
     "lib/workflow-scout.rb",
     "scout-gear.gemspec",
     "scout_commands/alias",
+    "scout_commands/batch/clean",
+    "scout_commands/batch/list",
     "scout_commands/doc",
     "scout_commands/find",
     "scout_commands/glob",
+    "scout_commands/log",
     "scout_commands/offsite",
     "scout_commands/rbbt",
     "scout_commands/resource/produce",
@@ -100,7 +105,10 @@ Gem::Specification.new do |s|
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
     "scout_commands/workflow/list",
+    "scout_commands/workflow/prov",
     "scout_commands/workflow/task",
+    "scout_commands/workflow/trace",
+    "scout_commands/workflow/write_info",
     "share/color/color_names",
     "share/color/diverging_colors.hex",
     "share/software/install_helpers",
@@ -117,6 +125,7 @@ Gem::Specification.new do |s|
     "test/scout/test_workflow.rb",
     "test/scout/tsv/persist/test_adapter.rb",
     "test/scout/tsv/persist/test_fix_width_table.rb",
+    "test/scout/tsv/persist/test_tkrzw.rb",
     "test/scout/tsv/persist/test_tokyocabinet.rb",
     "test/scout/tsv/test_attach.rb",
     "test/scout/tsv/test_change_id.rb",
@@ -130,6 +139,7 @@ Gem::Specification.new do |s|
     "test/scout/tsv/test_traverse.rb",
     "test/scout/tsv/test_util.rb",
     "test/scout/tsv/util/test_filter.rb",
+    "test/scout/tsv/util/test_melt.rb",
     "test/scout/tsv/util/test_process.rb",
     "test/scout/tsv/util/test_reorder.rb",
     "test/scout/tsv/util/test_select.rb",