scout-gear 10.8.3 → 10.9.0

data/lib/scout/knowledge_base/list.rb

@@ -12,7 +12,7 @@ class KnowledgeBase
       if entity_type.to_s == "simple"
         path = dir.lists[entity_type.to_s][id]
       else
-        path = dir.lists[entity_type.to_s][id + ".tsv"]
+        path = dir.lists[entity_type.to_s][id].find_with_extension("tsv")
       end
     else
       path = dir.lists.glob("*/#{id}").first
@@ -33,6 +33,7 @@ class KnowledgeBase
     Open.lock path do
       begin
         if AnnotatedArray === list
+          path = path.set_extension('tsv')
           Open.write(path, Annotation.tsv(list, :all).to_s)
         else
           Open.write(path, list * "\n")
@@ -60,7 +61,11 @@ class KnowledgeBase
         list.extend AnnotatedArray
         list
       else
-        path.list
+        list = path.list
+        if entity_type
+          Entity.prepare_entity(list, entity_type)
+        end
+        list
       end
     rescue
       Log.exception $!

data/lib/scout/knowledge_base/registry.rb

@@ -84,7 +84,7 @@ class KnowledgeBase
           key = name.to_s + "_" + fp
         end
 
-        Persist.memory("Index:" << [key, dir] * "@") do
+        Persist.memory("Index:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir
@@ -148,7 +148,7 @@ class KnowledgeBase
         options[:namespace] ||= self.namespace unless self.namespace.nil?
 
         key += '.database'
-        Persist.memory("Database:" << [key, dir] * "@") do
+        Persist.memory("Database:" + [key, dir] * "@") do
           options = options.dup
 
           persist_dir = dir

data/lib/scout/knowledge_base.rb

@@ -59,8 +59,26 @@ class KnowledgeBase
   end
 
   def self.load(dir)
-    dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
-    kb = KnowledgeBase.new dir
+    kb = case dir
+         when Path
+           KnowledgeBase.new dir
+         when Symbol
+           dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
+           kb = KnowledgeBase.new dir
+         when Workflow
+           raise if dir.knowledge_base.nil?
+           kb = dir.knowledge_base
+         when String
+           if Workflow.list.include? dir
+             workflow = Workflow.require_workflow dir
+             kb = workflow.knowledge_base
+           elsif dir =~ /^\w+$/
+             load(dir.to_sym)
+           else
+             kb = KnowledgeBase.new dir
+           end
+         end
+
     kb.load
     kb
   end

data/lib/scout/monitor.rb

@@ -0,0 +1,300 @@
+require 'scout'
+
+module Scout
+
+  SENSIBLE_WRITE_DIRS = Open.sensible_write_dir.find_all
+
+  LOCK_DIRS = Path.setup('tmp/tsv_open_locks').find_all +
+    Path.setup('tmp/tsv_locks').find_all +
+    Path.setup('tmp/persist_locks').find_all +
+    Path.setup('tmp/sensible_write_locks').find_all +
+    Path.setup('tmp/produce_locks').find_all +
+    Path.setup('tmp/step_info_locks').find_all
+
+  PERSIST_DIRS    = Path.setup('share').find_all  + Path.setup('var/cache/persistence').find_all
+
+  JOB_DIRS = Path.setup('var/jobs').find_all
+
+  MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
+
+  def self.dump_memory(file, obj = nil)
+    Log.info "Dumping #{obj} objects into #{ file }"
+    Thread.new do
+      while true
+        Open.write(file) do |f|
+          MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
+            GC.start
+            ObjectSpace.each_object(obj) do |o|
+              f.puts "---"
+              f.puts(String === o ? o : o.inspect)
+            end
+          end
+        end
+        FileUtils.cp file, file + '.save'
+        sleep 3
+      end
+    end
+  end
+
+  def self.file_time(file)
+    info = {}
+    begin
+      info[:ctime] = File.ctime file
+      info[:atime] = File.atime file
+      info[:elapsed] = Time.now - info[:ctime]
+    rescue Exception
+    end
+    info[:ctime] = Time.now - 999
+    info
+  end
+
+  #{{{ LOCKS
+
+  def self.locks(dirs = LOCK_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -name "*.lock" 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.lock_info(dirs = LOCK_DIRS)
+    lock_info = {}
+    locks(dirs).each do |f|
+      lock_info[f] = {}
+      begin
+        lock_info[f].merge!(file_time(f))
+        if File.size(f) > 0
+          info = Open.open(f) do |s|
+            Open.yaml(s)
+          end
+          IndiferentHash.setup(info)
+          lock_info[f][:pid] = info[:pid]
+          lock_info[f][:ppid] = info[:ppid]
+        end
+      rescue Exception
+        Log.warn $!.message
+      end
+    end
+    lock_info
+  end
+
+  #{{{ SENSIBLE WRITES
+
+  def self.sensiblewrites(dirs = SENSIBLE_WRITE_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -not -name "*.lock" -not -type d 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.sensiblewrite_info(dirs = SENSIBLE_WRITE_DIRS)
+    info = {}
+    sensiblewrites(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+
+  def self.persists(dirs = PERSIST_DIRS)
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+      `find -L "#{ dir }" -name "*.persist" 2>/dev/null`.split "\n"
+    end.compact.flatten
+  end
+
+  def self.persist_info(dirs = PERSIST_DIRS)
+    info = {}
+    persists(dirs).each do |f|
+      begin
+        i = file_time(f)
+        info[f] = i
+      rescue
+        Log.exception $!
+      end
+    end
+    info
+  end
+
+  # PERSISTS
+
+  def self.job_info(workflows = nil, tasks = nil, dirs = JOB_DIRS)
+    require 'rbbt/workflow/step'
+
+    workflows = [workflows] if workflows and not Array === workflows
+    workflows = workflows.collect{|w| w.to_s} if workflows
+
+    tasks = [tasks] if tasks and not Array === tasks
+    tasks = tasks.collect{|w| w.to_s} if tasks
+
+    jobs = {}
+    seen = Set.new
+    _files = Set.new
+    dirs.collect do |dir|
+      next unless Open.exists? dir
+
+      task_dir_workflows = {}
+      tasks_dirs = if dir == '.'
+                    ["."]
+                   else
+                     #workflowdirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                     #                 repo_dir, sub_path = dir_sub_path
+                     #                 Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                     #               else
+                     #                 dir.glob("*")
+                     #               end
+
+                     workflowdirs = dir.glob("*")
+
+                     workflowdirs.collect do |workflowdir|
+                       workflow = File.basename(workflowdir)
+                       next if workflows and not workflows.include? workflow
+
+                       #task_dirs = if (dir_sub_path = Open.find_repo_dir(workflowdir))
+                       #              repo_dir, sub_path = dir_sub_path
+                       #              Open.list_repo_files(*dir_sub_path).collect{|f| f.split("/").first}.uniq.collect{|f| File.join(repo_dir, f)}.uniq
+                       #            else
+                       #              workflowdir.glob("*")
+                       #            end
+
+                       task_dirs = workflowdir.glob("*")
+
+                       task_dirs.each do |tasks_dir|
+                         task_dir_workflows[tasks_dir] = workflow
+                       end
+                     end.compact.flatten
+                   end
+
+      tasks_dirs.collect do |taskdir|
+        task = File.basename(taskdir)
+        next if tasks and not tasks.include? task
+
+
+        #files = if (dir_sub_path = Open.find_repo_dir(taskdir))
+        #          repo_dir, sub_path = dir_sub_path
+        #          Open.list_repo_files(*dir_sub_path).reject do |f|
+        #            f.include?("/.info/") ||
+        #              f.include?(".files/") ||
+        #              f.include?(".pid/") ||
+        #              File.directory?(f)
+        #          end.collect do |f|
+        #            File.join(repo_dir, f)
+        #          end
+        #        else
+        #          #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+        #          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+        #          CMD.cmd(cmd, :pipe => true).read.split("\n")
+        #        end
+
+        files = begin
+                  cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/.info/*\" -prune \\) -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.md5' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
+
+                  CMD.cmd(cmd, :pipe => true).read.split("\n")
+                end
+
+        files = files.sort_by{|f| Open.mtime(f) || Time.now}
+        workflow = task_dir_workflows[taskdir]
+        TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
+          _files << file
+          if m = file.match(/(.*)\.(info|pid|files)$/)
+            file = m[1]
+          end
+          next if seen.include? file
+          seen << file
+
+          name = file[taskdir.length+1..-1]
+          info_file = file + '.info'
+
+          info = {}
+
+          info[:workflow] = workflow
+          info[:task] = task
+          info[:name] = name
+
+          if Open.exists? file
+            info = info.merge(file_time(file))
+            info[:done] = true
+            info[:info_file] = Open.exist?(info_file) ? info_file : nil
+          else
+            info = info.merge({:info_file => info_file, :done => false})
+          end
+
+          [file, info]
+        end
+
+      end.compact.flatten
+    end.compact.flatten
+    jobs
+  end
+
+  # REST
+
+  def self.__jobs(dirs = JOB_DIRS)
+    job_files = {}
+    dirs.each do |dir|
+      workflow_dirs = dir.glob("*").each do |wdir|
+        workflow = File.basename(wdir)
+        job_files[workflow] = {}
+        task_dirs = wdir.glob('*')
+        task_dirs.each do |tdir|
+          task = File.basename(tdir)
+          job_files[workflow][task] = tdir.glob('*')
+        end
+      end
+    end
+    jobs = {}
+    job_files.each do |workflow,task_jobs|
+      jobs[workflow] ||= {}
+      task_jobs.each do |task, files|
+        jobs[workflow][task] ||= {}
+        files.each do |f|
+          next if f =~ /\.lock$/
+          job = f.sub(/\.(info|files)/,'')
+
+          jobs[workflow][task][job] ||= {}
+          if jobs[workflow][task][job][:status].nil?
+            status = nil
+            status = :done if Open.exists? job
+            if status.nil? and f=~/\.info/
+              info = begin
+                       Step::INFO_SERIALIZER.load(Open.read(f, :mode => 'rb'))
+                     rescue
+                       {}
+                     end
+              status = info[:status]
+              pid = info[:pid]
+            end
+
+            jobs[workflow][task][job][:pid] = pid if pid
+            jobs[workflow][task][job][:status] = status if status
+          end
+        end
+      end
+    end
+    jobs
+  end
+
+  def self.load_lock(lock)
+    begin
+      info = Misc.insist 3 do
+        Open.yaml(lock)
+      end
+      info.values_at "pid", "ppid", "time"
+    rescue Exception
+      time = begin
+               File.atime(lock)
+             rescue Exception
+               Time.now
+             end
+      [nil, nil, time]
+    end
+  end
+
+end

data/lib/scout/persist/engine/packed_index.rb

@@ -9,7 +9,7 @@ class PackedIndex
   }
 
   def self.process_mask(mask)
-    str = ""
+    str = "".dup
     size = 0
     mask.each do |e|
       if ELEMS.include? e
@@ -50,7 +50,7 @@ class PackedIndex
       @mask = @stream.read(mask_length)
       @offset = @mask.length + 8
     end
-    @nil_string = "NIL" << ("-" * (@item_size - 3))
+    @nil_string = "NIL" + ("-" * (@item_size - 3))
   end
 
   def file

data/lib/scout/persist/engine/sharder.rb

@@ -37,7 +37,7 @@ class Sharder
       databases[shard]
     else
       database = databases[shard] ||= begin
-                                        path = File.join(persistence_path, 'shard-' << shard.to_s)
+                                        path = File.join(persistence_path, 'shard-' + shard.to_s)
                                         (writable or File.exist?(path)) ? Persist.open_database(path, (File.exist?(path) ? false : writable), :clean, db_type, @persist_options) : nil
                                       end
       Log.warn "Database #{ path } missing" if database.nil?

data/lib/scout/persist/tsv.rb

@@ -63,6 +63,7 @@ module Persist
 
         yield(data)
         data.save_annotation_hash if Annotation.is_annotated?(data)
+        data.read if data.respond_to?(:read)
         data
       end
     end

data/lib/scout/semaphore.rb

@@ -75,7 +75,7 @@ if continue
 
     def self.with_semaphore(size, file = nil)
       if file.nil?
-        file = "/scout-" << Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
+        file = "/scout-" + Misc.digest(rand(100000000000).to_s)[0..10] if file.nil?
       else
         file = file.gsub('/', '_') if file
       end

data/lib/scout/tsv/dumper.rb

@@ -16,7 +16,7 @@ module TSV
       if String === preamble
         preamble_str = preamble
       elsif preamble && options.values.compact.any?
-        preamble_str = "#: " << IndiferentHash.hash2string(options.merge(serializer: nil))
+        preamble_str = "#: " + IndiferentHash.hash2string(options.merge(serializer: nil))
       else
         preamble_str = nil
       end
@@ -81,7 +81,7 @@ module TSV
       header = Dumper.header(@options.merge(type: @type, sep: @sep, preamble: preamble))
       @mutex.synchronize do
         @initialized = true
-        @sin << header << "\n" if header and ! header.empty?
+        @sin << header + "\n" if header and ! header.empty?
       end
     end
 
@@ -134,7 +134,7 @@ module TSV
     end
 
     def fingerprint
-      "Dumper:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Dumper:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/open.rb

@@ -61,6 +61,7 @@ module Open
       end
 
       if into.respond_to?(:close)
+        obj = obj.find if Path === obj
         into_thread = Thread.new do 
           Thread.current.report_on_exception = false
           Thread.current["name"] = "Traverse into"

data/lib/scout/tsv/parser.rb

@@ -447,7 +447,7 @@ module TSV
     end
 
     def fingerprint
-      "Parser:{"<< Log.fingerprint(self.all_fields|| []) << "}"
+      "Parser:{" + Log.fingerprint(self.all_fields|| []) << "}"
     end
 
     def digest_str

data/lib/scout/tsv/transformer.rb

@@ -3,6 +3,7 @@ module TSV
     attr_accessor :unnamed, :parser, :dumper, :namespace
 
     def initialize(parser, dumper = nil, unnamed: nil, namespace: nil)
+
       if TSV::Parser === parser
         @parser = parser
       elsif TSV === parser

data/lib/scout/tsv/util.rb

@@ -165,11 +165,11 @@ Example:
   end
 
   def fingerprint
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
   end
 
   def digest_str
-    "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
+    "TSV:{" + Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << ";" << Log.fingerprint(self.values) << "}"
   end
 
   def inspect

data/lib/scout/work_queue/socket.rb

@@ -9,7 +9,7 @@ class WorkQueue
 
       @serializer = serializer || Marshal
 
-      @key = "/" << rand(1000000000).to_s << '.' << Process.pid.to_s;
+      @key = "/" + rand(1000000000).to_s << '.' << Process.pid.to_s;
       @write_sem = @key + '.in'
       @read_sem = @key + '.out'
       Log.debug "Creating socket semaphores: #{@key}"

data/lib/scout/work_queue/worker.rb

@@ -27,9 +27,9 @@ class WorkQueue
       run do
         begin
           if output
-            Open.purge_pipes(output.swrite)
+            Open.purge_pipes(input.sread, output.swrite)
           else
-            Open.purge_pipes
+            Open.purge_pipes(input.sread)
           end
 
           while obj = input.read
@@ -43,9 +43,11 @@ class WorkQueue
         rescue DoneProcessing
         rescue Interrupt
         rescue Exception
-          output.write WorkerException.new($!, Process.pid)
-          exit -1
-        ensure
+          begin
+            output.write WorkerException.new($!, Process.pid)
+          ensure
+            exit -1
+          end
         end
         exit 0
       end

data/lib/scout/workflow/documentation.rb

@@ -2,7 +2,7 @@ module Workflow
   attr_accessor :title, :description
 
   def self.doc_parse_first_line(str)
-    if str.match(/^([^\n]*)\n\n(.*)/sm)
+    if str.match(/^([^\n]*)\n\n(.*)/smu)
       str.replace $2
       $1
     else

data/lib/scout/workflow/entity.rb

@@ -71,10 +71,31 @@ module EntityWorkflow
     end
 
     property_name = task_name.to_s.sub(/^(#{entity_name}_list|#{entity_name}|list)_/, '')
+    property_job_name = property_name + '_job'
+
+    property property_job_name => property_type do |*args|
+      job(task_name, *args)
+    end
+
     property property_name => property_type do |*args|
-      job = job(task_name, *args)
+      job = self.send(property_job_name)
+
+      job.join if job.running?
+
+      if job.error?
+        if job.recoverable_error?
+          job.clean
+        else
+          raise job.exception
+        end
+      end
+
+      job.run unless job.done?
+
+      job.load
       Array === job ? job.collect(&:run) : job.run
     end
+
   end
 
   def entity_task(task_name, *args, &block)

data/lib/scout/workflow/step/config.rb

@@ -8,10 +8,10 @@ class Step
     new_tokens = []
     if workflow
       workflow_name = workflow.name
-      new_tokens << ("workflow:" << workflow_name)
-      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+      new_tokens << ("workflow:" + workflow_name)
+      new_tokens << ("task:" + workflow_name << "#" << task_name.to_s)
     end
-    new_tokens << ("task:" << task_name.to_s)
+    new_tokens << ("task:" + task_name.to_s)
 
     Scout::Config.get(key, tokens + new_tokens, options)
   end

data/lib/scout/workflow/step/file.rb

@@ -13,6 +13,10 @@ class Step
                    end
   end
 
+  def files_dir=(dir)
+    @files_dir = dir
+  end
+
   def file(file = nil)
     dir = files_dir
     Path.setup(dir) unless Path === dir

data/lib/scout/workflow/step/info.rb

@@ -120,7 +120,7 @@ class Step
       if info.include?(key)
         case info[key]
         when Array
-          info[key].concat Array === value ? value : [value]
+          info[key].concat(Array === value ? value : [value])
         when Hash
           info[key].merge! value
         else
@@ -201,7 +201,13 @@ class Step
   end
 
   def exception
-    info[:exception]
+    return nil unless info[:exception]
+    begin
+      Marshal.load(Base64.decode64(info[:exception]))
+    rescue
+      Log.exception $!
+      nil
+    end
   end
 
   # Marshal Step

data/lib/scout/workflow/step.rb

@@ -16,7 +16,7 @@ require_relative 'step/archive'
 class Step 
 
   attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
-  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context = nil, &task)
+  def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
     @path = path
     @inputs = inputs
     @dependencies = dependencies
@@ -112,17 +112,22 @@ class Step
 
   def exec
 
-    if inputs 
+    if inputs
       if Task === task
         types = task.inputs.collect{|name,type| type }
-        new_inputs = inputs.zip(types).collect{|input,info|  
+        new_inputs = inputs.zip(types).collect{|input,info|
           type, desc, default, options = info
           next input unless Step === input
           input.join if input.streaming?
           Task.format_input(input.join.path, type, options)
         }
       else
-        new_inputs = inputs.collect{|input|  
+        if Hash === inputs
+          new_inputs = inputs.values
+        else
+          new_inputs = inputs
+        end
+        new_inputs = new_inputs.collect{|input|
           Step === input ? input.load : input
         }
       end
@@ -218,7 +223,7 @@ class Step
 
       @result
     rescue Exception => e
-      merge_info :status => :error, :exception => e, :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
+      merge_info :status => :error, :exception => Base64.encode64(Marshal.dump(e)), :end => Time.now, :backtrace => e.backtrace, :message => "#{e.class}: #{e.message}"
       begin
         abort_dependencies
       ensure

data/lib/scout/workflow/task/inputs.rb

@@ -169,7 +169,7 @@ module Task
   end
 
   def load_inputs(directory)
-    if Open.exists?(directory) && ! Open.directory?(directory)
+    if Open.exists?(directory) && ! Open.directory?(directory) && ! Open.size(directory) == 0
       TmpFile.with_file do |tmp_directory|
         Misc.in_dir tmp_directory do
           CMD.cmd("tar xvfz '#{directory}'")