scout-gear 10.8.4 → 10.10.1

data/lib/scout/knowledge_base/list.rb

@@ -12,7 +12,7 @@ class KnowledgeBase
       if entity_type.to_s == "simple"
         path = dir.lists[entity_type.to_s][id]
       else
-        path = dir.lists[entity_type.to_s][id + ".tsv"]
+        path = dir.lists[entity_type.to_s][id].find_with_extension("tsv")
       end
     else
       path = dir.lists.glob("*/#{id}").first
@@ -33,6 +33,7 @@ class KnowledgeBase
     Open.lock path do
       begin
         if AnnotatedArray === list
+          path = path.set_extension('tsv')
           Open.write(path, Annotation.tsv(list, :all).to_s)
         else
           Open.write(path, list * "\n")
@@ -60,7 +61,11 @@ class KnowledgeBase
         list.extend AnnotatedArray
         list
       else
-        path.list
+        list = path.list
+        if entity_type
+          Entity.prepare_entity(list, entity_type)
+        end
+        list
       end
     rescue
       Log.exception $!

data/lib/scout/knowledge_base/registry.rb

@@ -84,7 +84,7 @@ class KnowledgeBase
           key = name.to_s + "_" + fp
         end
 
-        Persist.memory("Index:" << [key, dir] * "@") do
+        Persist.memory("Index:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir
@@ -148,7 +148,7 @@ class KnowledgeBase
         options[:namespace] ||= self.namespace unless self.namespace.nil?
 
         key += '.database'
-        Persist.memory("Database:" << [key, dir] * "@") do
+        Persist.memory("Database:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir
@@ -175,14 +175,13 @@ class KnowledgeBase
 
           database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
                        Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      else
                        options = IndiferentHash.add_defaults options, registered_options if registered_options
-                       undirected = IndiferentHash.process_options options, :undirected
                        raise "Repo #{ name } not found and not registered" if file.nil?
                        Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
                        file = file.call if Proc === file
-                       Association.database(file, **options.merge(persist_options: persist_options))
+                       Association.database(file, **options.merge(persist_options: persist_options).except(:undirected))
                      end
 
           database.namespace = self.namespace if self.namespace

data/lib/scout/knowledge_base.rb

@@ -59,8 +59,26 @@ class KnowledgeBase
   end
 
   def self.load(dir)
-    dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
-    kb = KnowledgeBase.new dir
+    kb = case dir
+         when Path
+           KnowledgeBase.new dir
+         when Symbol
+           dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
+           kb = KnowledgeBase.new dir
+         when Workflow
+           raise if dir.knowledge_base.nil?
+           kb = dir.knowledge_base
+         when String
+           if Workflow.list.include? dir
+             workflow = Workflow.require_workflow dir
+             kb = workflow.knowledge_base
+           elsif dir =~ /^\w+$/
+             load(dir.to_sym)
+           else
+             kb = KnowledgeBase.new dir
+           end
+         end
+
     kb.load
     kb
   end

data/lib/scout/monitor.rb

@@ -2,14 +2,18 @@ require 'scout'
 
 module Scout
 
-  LOCK_DIRS = Scout.tmp.tsv_open_locks.find_all + Scout.tmp.persist_locks.find_all + Scout.tmp.sensiblewrite_locks.find_all +
-    Scout.tmp.produce_locks.find_all + Scout.tmp.step_info_locks.find_all
-
   SENSIBLE_WRITE_DIRS = Open.sensible_write_dir.find_all
 
-  PERSIST_DIRS    = Scout.share.find_all  + Scout.var.cache.persistence.find_all
+  LOCK_DIRS = Path.setup('tmp/tsv_open_locks').find_all +
+    Path.setup('tmp/tsv_locks').find_all +
+    Path.setup('tmp/persist_locks').find_all +
+    Path.setup('tmp/sensible_write_locks').find_all +
+    Path.setup('tmp/produce_locks').find_all +
+    Path.setup('tmp/step_info_locks').find_all
+
+  PERSIST_DIRS    = Path.setup('share').find_all  + Path.setup('var/cache/persistence').find_all
 
-  JOB_DIRS = Scout.var.jobs.find_all
+  JOB_DIRS = Path.setup('var/jobs').find_all
 
   MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
 
@@ -68,7 +72,7 @@ module Scout
           lock_info[f][:ppid] = info[:ppid]
         end
       rescue Exception
-        Log.exception $!
+        Log.warn $!.message
       end
     end
     lock_info

data/lib/scout/persist/engine/packed_index.rb

@@ -9,7 +9,7 @@ class PackedIndex
   }
 
   def self.process_mask(mask)
-    str = ""
+    str = "".dup
     size = 0
     mask.each do |e|
       if ELEMS.include? e
@@ -50,7 +50,7 @@ class PackedIndex
       @mask = @stream.read(mask_length)
       @offset = @mask.length + 8
     end
-    @nil_string = "NIL" << ("-" * (@item_size - 3))
+    @nil_string = "NIL" + ("-" * (@item_size - 3))
   end
 
   def file

data/lib/scout/persist/engine/sharder.rb

@@ -37,7 +37,7 @@ class Sharder
       databases[shard]
     else
       database = databases[shard] ||= begin
-                                        path = File.join(persistence_path, 'shard-' << shard.to_s)
+                                        path = File.join(persistence_path, 'shard-' + shard.to_s)
                                         (writable or File.exist?(path)) ? Persist.open_database(path, (File.exist?(path) ? false : writable), :clean, db_type, @persist_options) : nil
                                       end
       Log.warn "Database #{ path } missing" if database.nil?

data/lib/scout/persist/tsv.rb

@@ -63,6 +63,7 @@ module Persist
 
         yield(data)
         data.save_annotation_hash if Annotation.is_annotated?(data)
+        data.read if data.respond_to?(:read)
         data
       end
     end

data/lib/scout/semaphore.rb

@@ -75,7 +75,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
+        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end

data/lib/scout/tsv/dumper.rb

@@ -16,7 +16,7 @@ module TSV
       if String === preamble
         preamble_str = preamble
       elsif preamble && options.values.compact.any?
-        preamble_str = "#: " << IndiferentHash.hash2string(options.merge(serializer: nil))
+        preamble_str = "#: " + IndiferentHash.hash2string(options.merge(serializer: nil))
       else
         preamble_str = nil
       end
@@ -81,7 +81,7 @@ module TSV
       header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
       @mutex.synchronize do
         @initialized = true
-        @sin << header << "\n" if header and ! header.empty?
+        @sin << header + "\n" if header and ! header.empty?
       end
     end
 
@@ -134,7 +134,7 @@ module TSV
     end
 
     def fingerprint
-      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Dumper:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/open.rb

@@ -61,6 +61,7 @@ module Open
       end
 
       if into.respond_to?(:close)
+        obj = obj.find if Path === obj
         into_thread = Thread.new do 
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Traverse into"

data/lib/scout/tsv/parser.rb

@@ -447,7 +447,7 @@ module TSV
     end
 
     def fingerprint
-      "Parser:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Parser:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/transformer.rb

@@ -3,6 +3,7 @@ module TSV
     attr_accessor :unnamed, :parser, :dumper, :namespace
 
     def initialize(parser, dumper = nil, unnamed: nil, namespace: nil)
+
       if TSV::Parser === parser
         @parser = parser
       elsif TSV === parser

data/lib/scout/tsv/util.rb

@@ -165,11 +165,11 @@ Example:
   end
 
   def fingerprint
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
   end
 
   def digest_str
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
   end
 
   def inspect

data/lib/scout/work_queue/socket.rb

@@ -9,7 +9,7 @@ class WorkQueue
 
       @serializer = serializer || Marshal
 
-      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @key = "/" + rand(1000000000).to_s << '.' << Process.pid.to_s;
       @write_sem = @key + '.in'
       @read_sem = @key + '.out'
       Log.debug "Creating socket semaphores: #{@key}"

data/lib/scout/work_queue/worker.rb

@@ -27,9 +27,9 @@ class WorkQueue
       run do
         begin
           if output
-            Open.purge_pipes(output.swrite)
+            Open.purge_pipes(input.sread, output.swrite)
           else
-            Open.purge_pipes
+            Open.purge_pipes(input.sread)
           end
 
           while obj = input.read
@@ -43,9 +43,11 @@ class WorkQueue
         rescue DoneProcessing
         rescue Interrupt
         rescue Exception
-          output.write WorkerException.new($!, Process.pid)
-          exit -1
-        ensure
+          begin
+            output.write WorkerException.new($!, Process.pid)
+          ensure
+            exit -1
+          end
         end
         exit 0
       end

data/lib/scout/workflow/definition.rb

@@ -227,4 +227,15 @@ module Workflow
   alias export_asynchronous export
   alias export_exec export
   alias export_stream export
+
+  def include_workflow(workflow)
+    workflow.documentation
+    self.asynchronous_exports += workflow.asynchronous_exports
+    self.synchronous_exports += workflow.synchronous_exports
+    self.exec_exports += workflow.exec_exports
+    self.stream_exports += workflow.stream_exports
+    self.tasks.merge! workflow.tasks
+    self.tasks.each{|_,t| t.workflow = workflow }
+    self.helpers.merge! workflow.helpers
+  end
 end

data/lib/scout/workflow/deployment/local.rb

@@ -0,0 +1,288 @@
+require_relative 'orchestrator/batches'
+class Workflow::LocalExecutor
+  class NoWork < Exception; end
+
+  def self.process(*args)
+    self.new.process(*args)
+  end
+
+  def self.produce(jobs, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    orchestrator = self.new produce_timer.to_f, cpus: produce_cpus.to_i
+    begin
+      orchestrator.process(rules, jobs)
+    rescue self::NoWork
+    end
+  end
+
+  def self.produce_dependencies(jobs, tasks, rules = {}, produce_cpus: Etc.nprocessors, produce_timer: 1)
+    jobs = [jobs] unless Array === jobs
+    tasks = tasks.collect{|task| (String === task) ? task.to_sym : task }
+
+    produce_list = []
+    jobs.each do |job|
+      next if job.done? || job.running?
+      job.rec_dependencies.each do |dep|
+        task_name = dep.task_name.to_sym
+        task_name = task_name.to_sym if String === task_name
+        produce_list << dep if tasks.include?(task_name) ||
+          tasks.include?(job.task_name.to_s) ||
+          tasks.include?(job.full_task_name)
+      end
+    end
+
+    produce(produce_list, rules, produce_cpus: produce_cpus, produce_timer: produce_timer)
+  end
+
+  attr_accessor :available_resources, :resources_requested, :resources_used, :timer
+
+  def initialize(timer = 5, available_resources = nil)
+    available_resources  = {:cpus => Etc.nprocessors } if available_resources.nil?
+    @timer               = timer
+    @available_resources = IndiferentHash.setup(available_resources)
+    @resources_requested = IndiferentHash.setup({})
+    @resources_used      = IndiferentHash.setup({})
+  end
+
+  def process_batches(batches)
+    failed_jobs = []
+
+    while batches.reject{|b| Workflow::Orchestrator.done_batch?(b) }.any?
+
+      candidates = Workflow::LocalExecutor.candidates(batches)
+      top_level_jobs = candidates.collect{|batch| batch[:top_level] }
+
+      raise NoWork, "No candidates and no running jobs #{Log.fingerprint batches}" if resources_used.empty? && top_level_jobs.empty?
+
+      candidates.each do |batch|
+        begin
+
+          job = batch[:top_level]
+
+          case
+          when (job.error? || job.aborted?)
+            begin
+              if job.recoverable_error?
+                if failed_jobs.include?(job)
+                  Log.warn "Failed twice #{job.path} with recoverable error"
+                  next
+                else
+                  failed_jobs << job
+                  job.clean
+                  raise TryAgain
+                end
+              else
+                Log.warn "Non-recoverable error in #{job.path}"
+                next
+              end
+            ensure
+              Log.warn "Releases resources from failed job: #{job.path}"
+              release_resources(job)
+            end
+          when job.done?
+            Log.debug "Orchestrator done #{job.path}"
+            release_resources(job)
+            clear_batch(batches, batch)
+            erase_job_dependencies(job, batches)
+          when job.running?
+            next
+
+          else
+            check_resources(batch) do
+              run_batch(batch)
+            end
+          end
+        rescue TryAgain
+          retry
+        end
+      end
+
+      batches.each do |batch|
+        job = batch[:top_level]
+        if job.done? || job.aborted? || job.error?
+          job.join if job.done?
+          clear_batch(batches, batch)
+          release_resources(job)
+          erase_job_dependencies(job, batches)
+        end
+      end
+
+      sleep timer
+    end
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      begin
+        job.join
+      rescue
+        Log.warn "Job #{job.short_path} ended with exception #{$!.class.to_s}: #{$!.message}"
+      end
+    }
+
+    batches.each{|batch|
+      job = batch[:top_level]
+      erase_job_dependencies(job, batches) if job.done?
+    }
+  end
+
+  def process(rules, jobs = nil)
+    jobs, rules = rules, {} if jobs.nil?
+    jobs = [jobs] if Step === jobs
+
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    batches.each do |batch|
+      rules = IndiferentHash.setup batch[:rules]
+      rules.delete :erase if jobs.include?(batch[:top_level])
+      resources = Workflow::Orchestrator.normalize_resources_from_rules(rules)
+      resources = IndiferentHash.add_defaults resources, rules[:default_resources] if rules[:default_resources]
+      batch[:resources] = resources
+      batch[:rules] = rules
+    end
+
+    process_batches(batches)
+  end
+
+  def release_resources(job)
+    if resources_used[job]
+      Log.debug "Orchestrator releasing resouces from #{job.path}"
+      resources_used[job].each do |resource,value| 
+        next if resource == 'size'
+        resources_requested[resource] -= value.to_i
+      end
+      resources_used.delete job
+    end
+  end
+
+  def check_resources(batch)
+    resources = batch[:resources]
+    job = batch[:top_level]
+
+    limit_resources = resources.select do |resource,value|
+      value && available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource]
+    end.collect do |resource,v|
+      resource
+    end
+
+    if limit_resources.any?
+      Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}"
+    else
+
+      resources_used[job] = resources
+      resources.each do |resource,value|
+        resources_requested[resource] ||= 0
+        resources_requested[resource] += value.to_i
+      end
+      Log.low "Orchestrator producing #{job.path} with resources #{resources}"
+
+      return yield
+    end
+  end
+
+  def run_batch(batch)
+    job, job_rules = batch.values_at :top_level, :rules
+
+    rules = batch[:rules] 
+    deploy = rules[:deploy] if rules
+    case deploy
+    when nil, 'local', :local, :serial, 'serial'
+      Scout::Config.with_config do
+        job_rules[:config_keys].split(/,\s*/).each do |config|
+          Scout::Config.process_config config
+        end if job_rules && job_rules[:config_keys]
+
+        log = job_rules[:log] if job_rules
+        log = Log.severity if log.nil?
+        Log.with_severity log do
+          job.fork
+        end
+      end
+    when 'batch', 'sched', 'slurm', 'pbs', 'lsf'
+      job.init_info
+      Workflow::Scheduler.process_batches([batch])
+      job.join
+    else
+      require 'scout/offsite'
+      if deploy.end_with?('-batch')
+        server = deploy.sub('-batch','')
+        OffsiteStep.setup(job, server: server, batch: true)
+      else
+        OffsiteStep.setup(job, server: deploy)
+      end
+
+      job.produce
+      job.join
+    end
+  end
+
+  def erase_job_dependencies(job, batches)
+    all_jobs = batches.collect{|b| b[:jobs] }.flatten
+    top_level_jobs = batches.collect{|b| b[:top_level] }
+
+    job.dependencies.each do |dep|
+      batch =  batches.select{|b| b[:jobs].include? dep}.first
+      next unless batch
+      rules = batch[:rules]
+      next unless rules[:erase].to_s == 'true'
+
+      dep_path = dep.path
+      parents = all_jobs.select do |parent|
+        parent.rec_dependencies.include?(dep)
+      end
+
+      next if parents.select{|parent| ! parent.done? }.any?
+
+      parents.each do |parent|
+        Log.high "Erasing #{dep.path} from #{parent.path}"
+        parent.archive_deps
+        parent.copy_linked_files_dir
+        parent.dependencies = parent.dependencies - [dep]
+      end
+
+      dep.clean
+    end
+  end
+
+  def clear_batch(batches, batch)
+    job = batch[:top_level]
+
+    parents = batches.select do |b|
+      b[:deps].include? batch
+    end
+
+    parents.each{|b| b[:deps].delete batch }
+  end
+
+  #{{{ HELPER
+
+  def self.purge_duplicates(batches)
+    seen = Set.new
+    batches.select do |batch|
+      path = batch[:top_level].path
+      if seen.include? path
+        false
+      else
+        seen << path
+        true
+      end
+    end
+  end
+
+  def self.sort_candidates(batches)
+    seen = Set.new
+    batches.sort_by do |batch|
+      - batch[:resources].values.compact.select{|e| Numeric === e }.inject(0.0){|acc,e| acc += e}
+    end
+  end
+
+  def self.candidates(batches)
+
+    leaf_nodes = batches.select{|b| b[:deps].empty? }
+
+    leaf_nodes.reject!{|b| Workflow::Orchestrator.done_batch?(b) }
+
+    leaf_nodes = purge_duplicates leaf_nodes
+    leaf_nodes = sort_candidates leaf_nodes
+
+    leaf_nodes
+  end
+end