scout-gear 10.8.4 → 10.9.0

data/scout_commands/entity

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Execute an entity property
+
+$ #{$0} [<options>] <entity_type> <property> <entity> [<entity_options>] [<property_parameters>]*
+
+Property options are an array of strings that get passed to the property, the
+remaining ARGV elements; property parameters may be optional
+
+-h--help Print this help
+-W--workflows* Workflows to load
+EOF
+
+type, property, entity = ARGV
+
+if options[:help] and type.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+raise MissingParameterException, :type if type.nil?
+
+workflows = IndiferentHash.process_options options, :workflows, workflows: 'local'
+
+workflows = workflows.split(",")
+
+workflows.each do |workflow|
+  workflow = workflow.strip
+  wf = case workflow
+       when 'local'
+         [:lib, :current].each do |location|
+           Workflow.require_workflow Scout.root['workflow.rb'].find(location) if Scout.root['workflow.rb'].find(location).exists?
+         end
+       when 'lib'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:lib) if Scout.root['workflow.rb'].find(:lib).exists?
+       when 'current'
+         Workflow.require_workflow Scout.root['workflow.rb'].find(:current) if Scout.root['workflow.rb'].find(:current).exists?
+       else
+         Workflow.require_workflow workflow
+       end
+
+  begin
+    require "#{wf.to_s}/entity/#{Misc.snake_case(type)}"
+  rescue Exception
+  end
+
+end
+
+begin
+  require "entity/#{Misc.snake_case(type)}"
+rescue Exception
+end
+entity_module = Kernel.const_get type
+
+entity_module.annotations.uniq.each do |attr|
+  SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute" 
+end
+
+if options[:help] and property.nil?
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  properties = entity_module.properties
+
+  puts Log.color :title, 'Properties: '
+  puts properties.keys * "\n"
+
+  exit 0
+end
+
+SOPT.synopsys[' [<entity_options>]'] = ""
+
+raise MissingParameterException, :property if property.nil?
+
+
+raise ParameterException, "Property no available" unless entity_module.instance_methods.include? property.to_sym
+
+if options[:help]
+  properties = entity_module.properties[property.to_sym]
+
+  property_str = properties.collect do |type, name|
+    if type != :opt
+      "[<#{name} (#{type})>]"
+    else
+      "[<#{name}>]"
+    end
+  end * " "
+
+  SOPT.synopsys['[<property_parameters>]*'] = property_str
+  
+  if defined? scout_usage
+    scout_usage
+  else
+    puts SOPT.doc
+  end
+
+  exit 0
+end
+
+raise MissingParameterException, :entity if entity.nil?
+
+entity = Entity.prepare_entity(entity, type, options)
+
+entity_params = ARGV[3..-1]
+
+entity_params = entity_params.collect do |param|
+  if Path.located?(param) && Path.can_read?(param)
+    Open.read(param)
+  else
+    case param
+    when 'true'
+      true
+    when 'false'
+      true
+    when /^-?\d+$/
+      param.to_i
+    when /^-?\d+\.\d+$/
+      param.to_f
+    else
+      param
+    end
+  end
+end
+
+res = entity.send(property, *entity_params)
+
+case res
+when nil
+when Array
+  puts res * "\n"
+when String
+  puts res unless res.empty?
+when Symbol, Numeric
+  puts res.to_s
+when Step
+  puts res.path
+else
+  puts res.to_s
+end

data/scout_commands/system/status

@@ -0,0 +1,238 @@
+#!/usr/bin/env ruby
+
+require 'scout'
+require 'scout/monitor'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Report the status of the system
+
+$ rbbt system status <workflow> <task>
+
+Specify workflow '.' and no task to examine the jobs of the current directory (usefull for web-server cache).
+
+-h--help Print this help
+-q--quick Quick check
+-a--all Print all jobs, not only uncompleted
+-i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
+EOF
+scout_usage and exit 0 if options[:help]
+
+workflow, task = ARGV
+
+
+workflow = workflow.split "," if workflow
+task = task.split "," if task
+
+all = options.delete :all
+
+inputs = (options[:inputs] || "").split(",")
+info_fields = (options[:info_fields] || "").split(",")
+
+def pid_msg(pid)
+  color = if pid and Misc.pid_alive? pid
+            :green
+          else
+            :red
+          end
+  if pid.nil?
+    ""
+  else
+    Log.color(color, pid)
+  end
+end
+
+
+#def status_msg(status)
+#  color = case status.to_sym
+#          when :error, :aborted, :missing, :dead, :broken
+#            :red
+#          when :streaming, :started
+#            :cyan
+#          when :done
+#            :green
+#          when :noinfo, :notfound
+#            :blue
+#          when :dependencies, :waiting, :setup
+#            :yellow
+#          else
+#            if status.to_s.index ">"
+#              :cyan
+#            else
+#              :cyan
+#            end
+#          end
+#  Log.color(color, status.to_s)
+#end
+
+def input_msg(file, inputs)
+
+  str = ""
+  job_inputs = Step.new(file).recursive_inputs.to_hash
+  IndiferentHash.setup(job_inputs)
+
+  inputs.each do |input|
+    value = job_inputs[input]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, input}=#{value_str}"
+  end
+  str
+end
+
+def info_msg(info, info_fields)
+
+  str = ""
+  info_fields.each do |field|
+    value = info[field]
+    next if  value.nil?
+    value_str = Misc.fingerprint(value)
+    str << "\t#{Log.color :magenta, field}=#{value_str}"
+  end
+  str
+end
+
+
+puts Log.color(:magenta, "# System report")
+puts
+sort_files = Proc.new do |a,b|
+  fa,ia = a
+  fb,ib = b
+
+  dira = fa.split(/[:_][^\/>]*$/).first
+  dirb = fb.split(/[:_][^\/>]*$/).first
+
+  case dira <=> dirb
+  when -1
+    -1
+  when 1
+    1
+  else
+    ia[:ctime] <=> ib[:ctime]
+  end
+end
+
+
+persists = Scout.persist_info
+if persists.any?
+  puts Log.color(:magenta, "Persist:")
+  persists.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+sensiblewrites = Scout.sensiblewrite_info
+if sensiblewrites.any?
+  puts Log.color(:magenta, "Writing:")
+  sensiblewrites.sort_by{|f,i| i[:ctime] }.each do |file,info|
+    elapsed = info[:elapsed]
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}")
+  end
+  puts
+end
+
+locks = Scout.lock_info
+if locks.any?
+  puts Log.color(:magenta, "Locks:")
+  locks.sort(&sort_files).each do |file,info|
+    elapsed, pid, ppid = info.values_at :elapsed, :pid, :ppid
+    puts "  " << file + Log.color(:blue, " -- time: #{Misc.format_seconds elapsed}; ppid: #{ppid}; pid: #{pid_msg pid}")
+  end
+  puts
+end
+
+exit 0 if workflow.nil?
+workflow = nil if workflow == ["all"]
+
+puts Log.color(:magenta, "# Workflows")
+
+if workflow === ['.']
+  jobs = Scout.job_info ["all"], task, ['.']
+else
+  jobs = Scout.job_info workflow, task
+end
+
+workflows = {}
+
+TSV.traverse jobs, :_bar => "Checking job status" do |file,info|
+  next unless all || ! info[:done] || ! File.exist?(file)
+  workflow = info[:workflow]
+  task = info[:task]
+  workflows[workflow] ||= {}
+  workflows[workflow][task] ||= {}
+  workflows[workflow][task][file] ||= info
+end
+
+workflows.sort.each do |workflow,tasks|
+  tasks.sort.each do |task,jobs|
+    puts "* " << Log.color(:magenta, workflow) << "#" << Log.color(:yellow, task) << ": " <<  Log.color(:blue, jobs.length.to_s)
+
+    files_txt = jobs.collect do |file, i|
+      str = file.dup
+      if options[:quick] and i[:done]
+        status = 'done'
+        str << " #{ Step.prov_status_msg status }"
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          info = begin
+                   Open.open(i[:info_file]) do |f|
+                     Step.load_info(f)
+                   end
+                 rescue
+                   Log.exception $!
+                   {:status => :noinfo}
+                 end
+          IndiferentHash.setup(info)
+          str << info_msg(info, info_fields)
+        end
+      else
+        info = begin
+                 Open.open(i[:info_file]) do |f|
+                   Step.load_info(f)
+                 end
+               rescue
+                 {:status => :noinfo}
+               end
+        IndiferentHash.setup(info)
+
+        pid = info[:pid]
+        status = info[:status]
+        status = :missing if status == :done and not (Open.exist?(file) && ! Open.broken_link?(file))
+        status = :broken if Open.broken_link?(file)
+        status = status.to_s
+        if status != "done" and pid and not Misc.pid_alive?(pid)
+          if File.exist? file
+            status << Log.color(:red, " (out of sync)") 
+          else
+            status << Log.color(:red, " (dead)") 
+          end
+        end
+        str << " #{ Step.prov_status_msg status }"
+        str << " (dirty)" if status == 'done' && Step.new(file).dirty?
+
+        if inputs and inputs.any? 
+          str << input_msg(file, inputs)
+        end
+
+        if info_fields and info_fields.any?
+          str << info_msg(info, info_fields)
+        end
+      end
+      str << "; #{pid_msg pid}" unless status == "done"
+      str
+    end
+
+    files_txt.each do |f|
+      next if f.nil?
+      puts "  " << f
+    end
+  end
+end

data/scout_commands/workflow/info

@@ -11,6 +11,8 @@ Show info from job
 $ #{$0} [<options>] <step_path>
 
 -h--help Print this help
+-i--inputs Pretty print the inputs
+-ri--recursive_inputs Pretty print the inputs (recursively)
 EOF
 if options[:help]
   if defined? scout_usage
@@ -25,17 +27,28 @@ path = ARGV.first
 raise MissingParameterException.new :step_path if path.nil?
 step = Step.load(path.dup)
 
-step.info.each do |k,v|
-  case v
-  when nil
+if options[:inputs]
+  names, values = step.info.values_at :input_names, :inputs
+  names.zip(values).each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end
+elsif options[:recursive_inputs]
+  step.recursive_inputs.each do |name,value|
+    puts [Log.color(:title, name), Log.fingerprint(value)] * " = "
+  end
+else
+  step.info.each do |k,v|
+    case v
+    when nil
+      next
+    when Exception
+      puts Log.color(:title, "Exception")
+      Log.exception v
+    else
+      puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
+    end
+  rescue
     next
-  when Exception
-    puts Log.color(:title, "Exception")
-    Log.exception v
-  else
-    puts [Log.color(:title, k), Log.fingerprint(v)] * " = "
   end
-rescue
-  next
 end
 

data/scout_commands/workflow/install

@@ -65,7 +65,7 @@ Misc.in_dir(workflow_dir) do
       Log.info "Installing: " + workflow
 
       if base_repo.nil?
-        repo_base_url = Rbbt.etc.workflow_repo.exists? ? Rbbt.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
+        repo_base_url = Scout.etc.workflow_repo.exists? ? Scout.etc.workflow_repo.read.strip : 'https://github.com/Rbbt-Workflows/'
       else
         repo_base_url = base_repo
       end

data/test/scout/entity/test_property.rb

@@ -335,7 +335,7 @@ class TestEntityProperty < Test::Unit::TestCase
 
   def test_all_properties
     assert ReversableString.setup("TEST").all_properties.include?(:reverse_text_ary)
-    assert_equal ReversableString.setup("TEST").all_properties, ReversableString.properties
+    assert_equal ReversableString.setup("TEST").all_properties.sort, ReversableString.properties.keys.sort
   end
 
   def test_times

data/test/scout/knowledge_base/test_registry.rb

@@ -31,5 +31,24 @@ Isa,IV
       end
     end
   end
+
+  def test_registry_identifiers_target
+    identifier =<<-EOF
+#Alias,Initials
+Clei,CC
+Miki,MV
+Guille,GC
+Isa,IV
+    EOF
+    TmpFile.with_dir do |dir|
+      TmpFile.with_file(identifier) do |identifier_file|
+        identifiers = TSV.open(identifier_file, sep: ",", type: :single)
+        brothers = datafile_test(:person).brothers
+        kb = KnowledgeBase.new dir
+        kb.register :brothers, brothers, identifiers: identifiers
+        assert_include kb.get_index(:brothers, source: "=>Initials", target: "=>Initials"), "CC~GC"
+      end
+    end
+  end
 end
 

data/test/scout/test_work_queue.rb

@@ -142,7 +142,7 @@ class TestWorkQueue < Test::Unit::TestCase
 
       res = []
       q.process do |out|
-        raise ScoutException 
+        raise ScoutException
         res << out
       end
 

data/test/scout/work_queue/test_worker.rb

@@ -156,7 +156,7 @@ class TestQueueWorker < Test::Unit::TestCase
     input = WorkQueue::Socket.new
     output = WorkQueue::Socket.new
 
-    workers = 10.times.collect{ WorkQueue::Worker.new }
+    workers = 5.times.collect{ WorkQueue::Worker.new }
     workers.each do |w|
       w.process(input, output) do |obj|
         raise ScoutException
@@ -164,18 +164,19 @@ class TestQueueWorker < Test::Unit::TestCase
       end
     end
 
-    read = Thread.new do 
+    Open.purge_pipes(input.swrite, output.sread)
+    read = Thread.new do
       Thread.current.report_on_exception = false
-      begin
-        while obj = output.read
-          if DoneProcessing === obj
-            pid = obj.pid
-            @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
-            break if workers.empty?
-          end
-          raise obj if Exception === obj
+      while obj = output.read
+        if DoneProcessing === obj
+          pid = obj.pid
+          @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
+          break if workers.empty?
         end
+        raise obj if Exception === obj
       end
+    ensure
+      output.close_read
     end
 
     write = Thread.new do
@@ -187,6 +188,7 @@ class TestQueueWorker < Test::Unit::TestCase
         input.write DoneProcessing.new
       end
       input.close_write
+    rescue
     end
 
     write.join

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scout-gear
 version: !ruby/object:Gem::Version
-  version: 10.8.4
+  version: 10.9.0
 platform: ruby
 authors:
 - Miguel Vazquez
@@ -101,18 +101,26 @@ executables:
 extensions: []
 extra_rdoc_files:
 - LICENSE.txt
+- README.md
 - README.rdoc
 files:
 - ".document"
 - ".gitmodules"
 - ".vimproject"
 - LICENSE.txt
+- README.md
 - README.rdoc
 - Rakefile
 - VERSION
 - bin/scout
-- doc/lib/scout/path.md
-- doc/lib/scout/workflow/task.md
+- doc/Association.md
+- doc/Entity.md
+- doc/KnowledgeBase.md
+- doc/Persist.md
+- doc/Semaphore.md
+- doc/TSV.md
+- doc/WorkQueue.md
+- doc/Workflow.md
 - lib/scout-gear.rb
 - lib/scout.rb
 - lib/scout/association.rb
@@ -213,7 +221,9 @@ files:
 - scout_commands/alias
 - scout_commands/batch/clean
 - scout_commands/batch/list
+- scout_commands/cat
 - scout_commands/doc
+- scout_commands/entity
 - scout_commands/find
 - scout_commands/glob
 - scout_commands/kb/config
@@ -228,6 +238,7 @@ files:
 - scout_commands/resource/produce
 - scout_commands/resource/sync
 - scout_commands/system/clean
+- scout_commands/system/status
 - scout_commands/template
 - scout_commands/update
 - scout_commands/workflow/cmd
@@ -350,7 +361,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.7.0.dev
 specification_version: 4
 summary: basic gear for scouts
 test_files: []

data/doc/lib/scout/path.md

@@ -1,35 +0,0 @@
-Path
-===
-
-```ruby
-:current => '{PWD}/{TOPLEVEL}/{SUBPATH}',
-:user    => '{HOME}/.{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:global  => '/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:usr     => '/usr/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:local   => '/usr/local/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:fast    => '/fast/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:cache   => '/cache/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:bulk    => '/bulk/{TOPLEVEL}/{PKGDIR}/{SUBPATH}',
-:lib     => '{LIBDIR}/{TOPLEVEL}/{SUBPATH}',
-:scout_gear => File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}"),
-:tmp     => '/tmp/{PKGDIR}/{TOPLEVEL}/{SUBPATH}',
-:default => :user
-
-@@basic_map_order ||= %w(current workflow user local global lib fast cache bulk)
-
-```
-
-
-
-# tags
-{PKGDIR}
-{LIBDIR}
-{RESOURCE}
-{HOME}
-{PWD}
-{TOPLEVEL}
-{SUBPATH}
-{BASENAME}
-{PATH}
-{MAPNAME}
-{REMOVE}

data/doc/lib/scout/workflow/task.md

@@ -1,13 +0,0 @@
-Task
-====
-
-```ruby
-    wf = Workflow.annonymous_workflow "TaskInputs" do
-      input :input1, :integer
-      task :step1 => :integer do |i| i end
-
-      dep :step1
-      input :input2, :integer, "Integer", 3
-      task :step2 => :integer do |i| i * step(:step1).load end
-    end
-```