scout-gear 7.1.0 → 7.2.0

data/lib/scout/tsv/persist/tokyocabinet.rb

@@ -0,0 +1,113 @@
+require 'tokyocabinet'
+require_relative 'adapter'
+
+module ScoutCabinet
+  attr_accessor :persistence_path, :persistence_class
+
+  CONNECTIONS = {}
+  def self.open(path, write, tokyocabinet_class = TokyoCabinet::HDB)
+    if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
+      big = true
+      tokyocabinet_class = tokyocabinet_class.split(":").first
+    else
+      big = false
+    end
+
+    dir = File.dirname(File.expand_path(path))
+    Open.mkdir(dir) unless File.exist?(dir)
+
+    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
+    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+
+    database = CONNECTIONS[path] ||= tokyocabinet_class.new
+
+    if big and not Open.exists?(path)
+      database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+    end
+
+    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+    database.close 
+
+    if !database.open(path, flags)
+      ecode = database.ecode
+      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+    end
+
+    database.extend ScoutCabinet
+    database.persistence_path ||= path
+    database.persistence_class = tokyocabinet_class
+
+    database.open(path, tokyocabinet_class::OREADER)
+
+    CONNECTIONS[path] = database
+
+    database
+  end
+
+  def close
+    @closed = true
+    @writable = false
+    super
+  end
+
+  def read(force = false)
+    return if ! write? && ! closed && ! force
+    self.close
+    if !self.open(@persistence_path, persistence_class::OREADER)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = false
+    @closed = false
+
+    self
+  end
+
+  def write(force = true)
+    return if write? && ! closed && ! force
+    self.close
+
+    if !self.open(@persistence_path, persistence_class::OWRITER)
+      ecode = self.ecode
+      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    end
+
+    @writable = true
+    @closed = false
+
+    self
+  end
+
+  #def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
+  #  raise
+  #  write = true unless File.exist? path
+
+  #  FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
+
+  #  database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
+
+  #  unless serializer == :clean
+  #    TSV.setup database
+  #    database.write_and_read do
+  #      database.serializer = serializer
+  #    end if serializer && database.serializer != serializer
+  #  end
+
+  #  database
+  #end
+end
+
+Persist.save_drivers[:HDB] = proc do |file, content|
+  data = ScoutCabinet.open(file, true, "HDB")
+  content.annotate(data)
+  data.extend TSVAdapter
+  data.merge!(content)
+  data
+end
+
+Persist.load_drivers[:HDB] = proc do |file| 
+  data = ScoutCabinet.open(file, false, "HDB")
+  data.extend TSVAdapter unless TSVAdapter === data
+  data
+end

data/lib/scout/tsv/persist.rb

@@ -0,0 +1,15 @@
+require_relative 'persist/adapter'
+require_relative 'persist/tokyocabinet'
+
+Persist.save_drivers[:tsv] = proc do |file,content| 
+  stream = if IO === content
+             content
+           elsif content.respond_to?(:get_stream)
+             content.get_stream
+           elsif content.respond_to?(:stream)
+             content.stream
+           end
+  Open.sensible_write(file, stream)
+end
+
+Persist.load_drivers[:tsv] = proc do |file| TSV.open file end

data/lib/scout/tsv/traverse.rb

@@ -0,0 +1,48 @@
+require_relative 'parser'
+module TSV
+  def self.traverse_add(into, res)
+    case into
+    when TSV::Dumper
+      into.add *res
+    when TSV, Hash
+      key, value = res
+      into[key] = value
+    end
+  end
+
+  def self.traverse(obj, into: nil, cpus: nil, bar: nil, **options, &block)
+    case obj
+    when TSV
+      self.traverse(obj.stream, into: into, cpus: cpus, bar: bar, **options, &block)
+    when String
+      f = Open.open(obj)
+      self.traverse(f, into: into, cpus: cpus, bar: bar, **options, &block)
+    when Step
+      self.traverse(obj.get_stream, into: into, cpus: cpus, bar: bar, **options, &block)
+    when IO
+      if into
+        into_thread = Thread.new do 
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "Traverse into"
+          TSV.parse obj, **options do |k,v|
+            begin
+              res = block.call k, v
+              traverse_add into, res
+            rescue
+              into.abort $!
+            end
+            nil
+          end
+          into.close if into.respond_to?(:close)
+        end
+        Thread.pass until into_thread
+        into
+      else
+        TSV.parse obj, **options do |k,v|
+          block.call k, v
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/scout/tsv/util.rb

@@ -0,0 +1,24 @@
+#require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
+#Log.warn "USING OLD RBBT CODE: #{__FILE__}"
+module TSV
+  #[:each, :collect, :map].each do |method|
+  #  define_method(method) do |*args,&block|
+  #    super(*args) do |k,v|
+  #      NamedArray.setup(v, @fields) unless @unnamed
+  #      block.call k, v
+  #    end
+  #  end
+  #end
+
+  #[:select, :reject].each do |method|
+  #  define_method(method) do |*args,&block|
+  #    res = super(*args) do |k,v|
+  #      NamedArray.setup(v, @fields) unless @unnamed
+  #      block.call k, v
+  #    end
+  #    self.annotate(res)
+  #    res
+  #  end
+  #end
+
+end

data/lib/scout/tsv.rb

@@ -1,14 +1,27 @@
 require_relative 'meta_extension'
+require_relative 'tsv/util'
 require_relative 'tsv/parser'
+require_relative 'tsv/dumper'
+require_relative 'tsv/persist'
+require_relative 'tsv/index'
+require_relative 'tsv/path'
+require_relative 'tsv/traverse'
 
 module TSV
   extend MetaExtension
-  extension_attr :key_field, :fields
+  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed
 
   def self.open(file, options = {})
-    Open.open(file) do |f|
-      TSV.parse(f,**options)
+    persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
+    Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
+      data = filename ? ScoutCabinet.open(filename, true, type) : nil
+      options[:data] = data if data
+      options[:filename] = file
+      Open.open(file) do |f|
+        TSV.parse(f, **options)
+      end
     end
   end
+
 end
 

data/lib/scout/work_queue/worker.rb

@@ -21,7 +21,7 @@ class WorkQueue
               raise obj 
             end
             res = block.call obj
-            output.write res unless output.nil? || ignore_ouput || res == :ignore 
+            output.write res unless ignore_ouput || res == :ignore 
           end
         rescue DoneProcessing
         rescue Interrupt
@@ -34,14 +34,14 @@ class WorkQueue
 
     def abort
       begin
-        Log.log "Aborting worker #{@pid}"
+        Log.debug "Aborting worker #{@pid}"
         Process.kill "INT", @pid 
       rescue Errno::ECHILD
       end
     end
 
     def join
-      Log.log "Joining worker #{@pid}"
+      Log.debug "Joining worker #{@pid}"
       Process.waitpid @pid
     end
 

data/lib/scout/work_queue.rb

@@ -36,8 +36,12 @@ class WorkQueue
 
   def remove_worker(pid)
     @worker_mutex.synchronize do
-      @workers.delete_if{|w| w.pid == pid }
-      @removed_workers << pid
+      worker = @workers.index{|w| w.pid == pid}
+      if worker
+        Log.debug "Removed worker #{pid}"
+        @workers.delete_at(worker)
+        @removed_workers << pid
+      end
     end
   end
 
@@ -50,11 +54,13 @@ class WorkQueue
         while true
           obj = @output.read
           if DoneProcessing === obj
+
             done = @worker_mutex.synchronize do
               Log.low "Worker #{obj.pid} done"
               @done_workers << obj.pid
-              @done_workers.length == @removed_workers.length + @workers.length
+              @closed && @done_workers.length == @removed_workers.length + @workers.length
             end
+
             break if done
           elsif Exception === obj
             raise obj
@@ -65,9 +71,13 @@ class WorkQueue
       rescue DoneProcessing
       rescue Aborted
       rescue WorkerException
-        Log.error "Exception in worker #{obj.pid} #{Log.fingerprint obj.exception}"
+        Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
         self.abort
-        raise obj.exception
+        raise obj.worker_exception
+      rescue
+        Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
+        self.abort
+        raise $!
       end
     end
 
@@ -84,11 +94,12 @@ class WorkQueue
         while true
           pid = Process.wait
           remove_worker(pid)
-          break if workers.empty?
+          break if @worker_mutex.synchronize{ @workers.empty? }
         end
       end
     end
 
+    Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
     Thread.pass until @waiter["name"]
   end
 
@@ -97,10 +108,14 @@ class WorkQueue
   end
 
   def abort
-    workers.each{|w| w.abort }
+    Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
+    @worker_mutex.synchronize do
+      @workers.each{|w| w.abort }
+    end
   end
 
   def close
+    @closed = true
     @worker_mutex.synchronize{ @workers.length }.times do
       @input.write DoneProcessing.new()
     end

data/lib/scout/workflow/definition.rb

@@ -2,7 +2,7 @@ require_relative '../meta_extension'
 
 module Workflow
   extend MetaExtension
-  extension_attr :name, :tasks
+  extension_attr :name, :tasks, :helpers
 
   class << self
     attr_accessor :directory
@@ -17,6 +17,32 @@ module Workflow
     @name ||= self.to_s
   end
 
+  def helpers
+    @helpers ||= {}
+  end
+
+  def helper(name, *args, &block)
+    if block_given?
+      helpers[name] = block
+    else
+      raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
+      helpers[name].call(*args)
+    end
+  end
+
+  def step_module
+    @_m ||= begin
+              m = Module.new
+
+              helpers.each do |name,block|
+                m.send(:define_method, name, &block)
+              end
+
+              m
+            end
+    @_m
+  end
+
   attr_accessor :directory
   def directory
     @directory ||= Workflow.directory[name]
@@ -60,19 +86,82 @@ module Workflow
     annotate_next_task(:inputs, args)
   end
 
+  def desc(description)
+    annotate_next_task_single(:description, description)
+  end
+
+  def returns(type)
+    annotate_next_task_single(:returns, type)
+  end
+
+  def extension(extension)
+    annotate_next_task_single(:extension, extension)
+  end
+
   def task(name_and_type, &block)
     name, type = name_and_type.collect.first
     @tasks ||= IndiferentHash.setup({})
     begin
       @annotate_next_task ||= {}
-      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
+      task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
       @tasks[name] = task
     ensure
       @annotate_next_task = {}
     end
   end
 
-  def desc(description)
-    annotate_next_task_single(:description, description)
+  def task_alias(name, workflow, oname, *rest, &block)
+    dep(workflow, oname, *rest, &block) 
+    extension :dep_task unless @extension
+    returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns 
+    task name => nil do
+      raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
+      Step.wait_for_jobs dependencies.select{|d| d.streaming? }
+      dep = dependencies.last
+      dep.join
+      raise dep.get_exception if dep.error?
+      raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
+      set_info :result_type, dep.info[:result_type]
+      forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
+      if forget
+        remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
+
+        self.archive_deps
+        self.copy_files_dir
+        self.dependencies = self.dependencies - [dep]
+        Open.rm_rf self.files_dir if Open.exist? self.files_dir
+        FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
+
+        if dep.overriden || ! Workflow.job_path?(dep.path)
+          Open.link dep.path, self.tmp_path
+        else
+          Open.ln_h dep.path, self.tmp_path
+
+          case remove.to_s
+          when 'true'
+            dep.clean
+          when 'recursive'
+            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+              next if d.overriden
+              d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
+            end
+            dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
+          end 
+        end
+      else
+        if Open.exists?(dep.files_dir)
+          Open.rm_rf self.files_dir 
+          Open.link dep.files_dir, self.files_dir
+        end
+        if defined?(RemoteStep) && RemoteStep === dep
+          Open.write(self.tmp_path, Open.read(dep.path))
+        else
+          Open.link dep.path, self.path
+        end
+      end
+      nil
+    end
   end
+
+  alias dep_task task_alias
 end

data/lib/scout/workflow/step/config.rb

@@ -0,0 +1,18 @@
+require_relative '../../config'
+
+class Step
+  def config(key, *tokens)
+    options = tokens.pop if Hash === tokens.last
+    options ||= {}
+
+    new_tokens = []
+    if workflow
+      workflow_name = workflow.to_s
+      new_tokens << ("workflow:" << workflow_name)
+      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+    end
+    new_tokens << ("task:" << task_name.to_s)
+
+    Scout::Config.get(key, tokens + new_tokens, options)
+  end
+end

data/lib/scout/workflow/step/dependencies.rb

@@ -0,0 +1,40 @@
+class Step
+  def recursive_inputs
+    dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
+      acc.concat(dep.inputs) if dep.inputs
+      acc
+    end
+  end
+
+  def input_dependencies
+    return [] unless inputs
+    inputs.select do |d|
+      Step === d
+    end
+  end
+
+  def prepare_dependencies
+    inverse_dep = {}
+    dependencies.each{|dep| 
+      next if dep.done?
+      if dep.dependencies
+        dep.dependencies.each do |d|
+          inverse_dep[d] ||= []
+          inverse_dep[d] << dep 
+        end
+      end
+      input_dependencies.each do |d|
+        inverse_dep[d] ||= []
+        inverse_dep[d] << dep 
+      end
+    }
+    inverse_dep.each do |dep,list|
+      dep.tee_copies = list.length
+    end
+  end
+
+  def run_dependencies
+    dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
+  end
+
+end

data/lib/scout/workflow/step/file.rb

@@ -0,0 +1,15 @@
+class Step
+  def files_dir
+    @files_dir ||= begin
+                     dir = @path + ".files"
+                     @path.annotate(dir) if Path === @path
+                     dir
+                   end
+  end
+
+  def file(file)
+    dir = files_dir
+    Path.setup(dir) unless Path === dir
+    dir[file]
+  end
+end

data/lib/scout/workflow/step/info.rb

@@ -1,5 +1,5 @@
 class Step
-  SERIALIZER = :json
+  SERIALIZER = :marshal
   def info_file
     @info_file ||= begin
                      info_file = @path + ".info"
@@ -20,7 +20,11 @@ class Step
   end
 
   def info
-    outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
+    outdated = begin
+                 @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
+               rescue
+                 true
+               end
 
     if @info.nil? || outdated
       load_info
@@ -33,6 +37,19 @@ class Step
     info = self.info
     new_info.each do |key,value|
       report_status new_info[:status], new_info[:message] if key == :status
+      if Exception === value
+        begin
+          Marshal.dump(value)
+        rescue TypeError
+          if ScoutException === value
+            new = ScoutException.new value.message
+          else
+            new = Exception.new value.message
+          end
+          new.set_backtrace(value.backtrace)
+          value = new
+        end
+      end
       if info.include?(key)
         case info[key]
         when Array
@@ -62,7 +79,6 @@ class Step
   end
 
   def log(status, message = nil)
-    report_status status, message
     if message
       merge_info :status => status, :messages => [message]
     else
@@ -71,7 +87,18 @@ class Step
   end
 
   def status
-    info[:status]
+    info[:status].tap{|s| s.nil? ? s : s.to_sym }
+  end
+
+  def error?
+    status == :error
   end
 
+  def aborted?
+    status == :aborted
+  end
+
+  def running?
+    ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
+  end
 end