scout-gear 10.7.2 → 10.7.4

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -1,105 +1,112 @@
-require 'tokyocabinet'
-
-module ScoutCabinet
-  attr_accessor :persistence_path, :persistence_class
-
-  def self.open(path, write = true, tokyocabinet_class = TokyoCabinet::HDB)
-    path = path.find if Path === path
-    if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
-      big = true
-      tokyocabinet_class = tokyocabinet_class.split(":").first
-    else
-      big = false
-    end
+begin
+  require 'tokyocabinet'
+  continue = true
+rescue Exception
+  Log.warn "The Tokyocabinet gem could not be loaded: TSV persistence may not work"
+  continue = false
+end
 
-    dir = File.dirname(File.expand_path(path))
-    Open.mkdir(dir) unless File.exist?(dir)
+if continue
+  module ScoutCabinet
+    attr_accessor :persistence_path, :persistence_class
 
-    tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
-    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
-    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+    def self.open(path, write = true, tokyocabinet_class = TokyoCabinet::HDB)
+      path = path.find if Path === path
+      if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
+        big = true
+        tokyocabinet_class = tokyocabinet_class.split(":").first
+      else
+        big = false
+      end
 
-    # Hack - Ignore warning: undefining the allocator of T_DATA class
-    # TokyoCabinet::HDB_data
-    database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
+      dir = File.dirname(File.expand_path(path))
+      Open.mkdir(dir) unless File.exist?(dir)
 
-    if big and not Open.exists?(path)
-      database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
-    end
+      tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
+      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
+      tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-    database.close 
+      # Hack - Ignore warning: undefining the allocator of T_DATA class
+      # TokyoCabinet::HDB_data
+      database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
-    if !database.open(path, flags)
-      ecode = database.ecode
-      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
-    end
+      if big and not Open.exists?(path)
+        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+      end
 
-    database.extend ScoutCabinet
-    database.persistence_path ||= path
-    database.persistence_class = tokyocabinet_class
+      flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+      database.close 
 
-    database.open(path, tokyocabinet_class::OREADER)
+      if !database.open(path, flags)
+        ecode = database.ecode
+        raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+      end
 
-    database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
+      database.extend ScoutCabinet
+      database.persistence_path ||= path
+      database.persistence_class = tokyocabinet_class
 
-    Persist::CONNECTIONS[path] = database
+      database.open(path, tokyocabinet_class::OREADER)
 
-    database
-  end
+      database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
 
-  def close
-    @closed = true
-    @writable = false
-    super
-  end
+      Persist::CONNECTIONS[path] = database
 
-  def read(force = false)
-    return if ! @writable && ! @closed && ! force
-    self.close
-    if !self.open(@persistence_path, persistence_class::OREADER)
-      ecode = self.ecode
-      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      database
     end
 
-    @writable = false
-    @closed = false
-
-    self
-  end
+    def close
+      @closed = true
+      @writable = false
+      super
+    end
 
-  def write?
-    @writable
-  end
+    def read(force = false)
+      return if ! @writable && ! @closed && ! force
+      self.close
+      if !self.open(@persistence_path, persistence_class::OREADER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
 
-  def closed?
-    @closed
-  end
+      @writable = false
+      @closed = false
 
+      self
+    end
 
-  def write(force = true)
-    return if write? && ! closed? && ! force
-    self.close
+    def write?
+      @writable
+    end
 
-    if !self.open(@persistence_path, persistence_class::OWRITER)
-      ecode = self.ecode
-      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    def closed?
+      @closed
     end
 
-    @writable = true
-    @closed = false
 
-    self
-  end
+    def write(force = true)
+      return if write? && ! closed? && ! force
+      self.close
 
-  def write_and_read
-    begin
-      write
-      yield
-    ensure
-      read
+      if !self.open(@persistence_path, persistence_class::OWRITER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
+
+      @writable = true
+      @closed = false
+
+      self
+    end
+
+    def write_and_read
+      begin
+        write
+        yield
+      ensure
+        read
+      end
     end
-  end
 
   def write_and_close
     begin
@@ -139,4 +146,5 @@ module ScoutCabinet
   end
 
   alias load_stream importtsv
+  end
 end

data/lib/scout/persist/tsv/adapter/base.rb

@@ -23,6 +23,7 @@ module TSVAdapter
   end
 
   def save_annotation_hash
+    self.close
     self.with_write do
       self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
     end
@@ -165,15 +166,6 @@ module TSVAdapter
     end
   end
 
-  def with_write(*args, &block)
-    if @write
-      yield
-    elsif @closed
-      write_and_close &block
-    else
-      write_and_read &block
-    end
-  end
 
   def close(*args)
     begin
@@ -250,12 +242,11 @@ module TSVAdapter
 
     lock do
       write(true) if closed? || ! write?
-      res = begin
-              yield
-            ensure
-              close
-            end
-      res
+      begin
+        yield
+      ensure
+        close
+      end
     end
   end
 
@@ -272,18 +263,13 @@ module TSVAdapter
       return yield
     else
       if self.read?
-        self.write_and_read do
-          return yield
-        end
+        self.write_and_read(&block)
       else
-        self.write_and_close do
-          return yield
-        end
+        self.write_and_close(&block)
       end
     end
   end
 
-
   def read_and_close
     if read? || write?
       begin

data/lib/scout/tsv/annotation.rb

@@ -48,12 +48,12 @@ module Annotation
 
     fields = fields.flatten.compact.uniq
 
-    annotations = if Annotation.is_annotated?(objs) 
-                    objs.annotations 
+    annotations = if Annotation.is_annotated?(objs)
+                    objs.annotation_hash.keys
                   elsif (Array === objs && objs.any?)
                     first = objs.compact.first
                     if Annotation.is_annotated?(first)
-                      objs.compact.first.annotations
+                      objs.compact.first.annotation_hash.keys
                     else
                       raise "Objects didn't have annotations"
                     end
@@ -158,7 +158,7 @@ module Annotation
         Annotation.load_tsv_values(id, values, tsv.fields)
       end
 
-      case tsv.key_field 
+      case tsv.key_field
       when "List"
         annotated_objects.first
       else

data/lib/scout/tsv/index.rb

@@ -66,8 +66,6 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      tsv_file = TSV.open(tsv_file, **data_options) if ! TSV === tsv_file
-
       log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
       Log.low log_msg
       bar = log_msg if TrueClass === bar

data/lib/scout/tsv/parser.rb

@@ -1,5 +1,15 @@
 require 'scout/named_array'
 module TSV
+  def self.acceptable_parser_options(func = nil)
+    if func.nil?
+      TSV.method(:parse_line).parameters.collect{|a| a.last } +
+        TSV.method(:parse_stream).parameters.collect{|a| a.last } +
+        TSV.method(:parse).parameters.collect{|a| a.last } - [:line, :block]
+    else
+      TSV.method(func).parameters.collect{|a| a.last }
+    end.uniq
+  end
+
   def self.cast_value(value, cast)
     if Array === value
       value.collect{|e| cast_value(e, cast) }
@@ -7,7 +17,7 @@ module TSV
       if Proc === cast
         cast.call value
       else
-        if value.nil? || value.empty?
+        if value.nil? || value == ""
           nil
         else
           value.send(cast)

data/lib/scout/tsv/stream.rb

@@ -1,5 +1,5 @@
 module TSV
-  def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
+  def self.paste_streams(streams, type: nil, sort: nil, sort_cmd_args: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
     sep = "\t" if sep.nil?
 
     streams = streams.collect do |stream|
@@ -20,7 +20,7 @@ module TSV
     num_streams = streams.length
 
     streams = streams.collect do |stream|
-      Open.sort_stream(stream, memory: sort_memory)
+      Open.sort_stream(stream, memory: sort_memory, cmd_args: sort_cmd_args)
     end if sort
 
     begin
@@ -193,7 +193,7 @@ module TSV
       dumper.close
 
       streams.each do |stream|
-        stream.close if stream.respond_to?(:close) && ! stream.closed?
+        stream.close if stream.respond_to?(:close)
         stream.join if stream.respond_to?(:join)
       end
       end

data/lib/scout/tsv/transformer.rb

@@ -175,5 +175,17 @@ module TSV
     end
     res
   end
+
+  def head(max=10)
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    i = 0
+    transformer.traverse do |k,v|
+      i += 1
+      break if i > max
+      [k, v]
+    end
+    res
+  end
 end
 

data/lib/scout/tsv/util/process.rb

@@ -30,9 +30,9 @@ module TSV
       when type == :flat
         self[key] = new_values
       else
-        if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
+        if ! values[field_pos].frozen? && ! NamedArray === values && ((String === values[field_pos] && String === new_values) ||
           (Array === values[field_pos] && Array === new_values))
-           values[field_pos].replace new_values
+          values[field_pos].replace new_values
         else
           values[field_pos] = new_values
         end

data/lib/scout/tsv.rb

@@ -133,6 +133,8 @@ module TSV
 
       tsv.unnamed = unnamed unless unnamed.nil?
 
+      tsv.entity_options = entity_options
+
       tsv
     end
   end

data/lib/scout/workflow/definition.rb

@@ -13,8 +13,12 @@ module Workflow
 
   end
 
+  def to_s
+    @name || super
+  end
+
   def name
-    @name ||= self.to_s
+    @name || to_s
   end
 
   def helpers
@@ -192,7 +196,7 @@ module Workflow
           when 'true'
             dep.clean
           when 'recursive'
-            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
               next if d.overriden
               d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
             end

data/lib/scout/workflow/deployment/trace.rb

@@ -164,7 +164,7 @@ module Workflow
   def self.trace(seed_jobs, options = {})
     jobs = []
     seed_jobs.each do |step|
-      jobs += step.rec_dependencies + [step]
+      jobs += step.rec_dependencies.to_a + [step]
       step.info[:archived_info].each do |path,ainfo|
         next unless Hash === ainfo
         archived_step = Step.new path

data/lib/scout/workflow/step/dependencies.rb

@@ -1,5 +1,5 @@
 class Step
-  def rec_dependencies(connected = false, seen = [])
+  def rec_dependencies(connected = false, seen = Set.new)
     @rec_dependencies = {}
     @rec_dependencies[connected] ||= begin
                             direct_deps = []
@@ -8,11 +8,8 @@ class Step
                               next if connected && dep.done? && dep.updated?
                               direct_deps << dep
                             end if dependencies
-                            seen.concat direct_deps.collect{|d| d.path }
-                            seen.uniq!
-                            direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
-                            direct_deps.uniq!
-                            direct_deps
+                            seen += direct_deps.collect{|d| d.path }
+                            direct_deps.inject(Set.new(direct_deps)){|acc,d| acc += d.rec_dependencies(connected, seen) }
                           end
   end
 

data/lib/scout/workflow/step/info.rb

@@ -1,5 +1,6 @@
+require 'scout/config'
 class Step
-  SERIALIZER = :marshal
+  SERIALIZER = Scout::Config.get(:serializer, :step_info, :info, :step, env: "SCOUT_SERIALIZER", default: :json)
   def info_file
     return nil if @path.nil?
     @info_file ||= begin
@@ -13,7 +14,11 @@ class Step
     info = begin
              Persist.load(info_file, SERIALIZER) || {}
            rescue
-             {status: :noinfo}
+             begin
+               Persist.load(info_file, :marshal) || {}
+             rescue
+               {status: :noinfo}
+             end
            end
     IndiferentHash.setup(info)
   end
@@ -50,6 +55,15 @@ class Step
     @info
   end
 
+  def pid
+    info[:pid]
+  end
+
+  def pid=(pid)
+    set_info :pid, pid
+  end
+
+
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
@@ -121,7 +135,7 @@ class Step
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
-      message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
+      message = Log.fingerprint(message.split("\n").first).sub(/^'/,'').sub(/'$/,'')
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end

data/lib/scout/workflow/task/info.rb

@@ -0,0 +1,99 @@
+require_relative 'inputs'
+module Workflow
+
+  def rec_inputs(task_name)
+    tasks[task_name].recursive_inputs.collect{|name, _| name }
+  end
+
+  def rec_input_types(task_name)
+    tasks[task_name].recursive_inputs.inject({}) do |acc,l|
+      name, type, desc, default, options = l
+      acc.merge!(name => type) unless acc.include?(name)
+      acc
+    end
+  end
+
+
+  def rec_input_descriptions(task_name)
+    tasks[task_name].recursive_inputs.inject({}) do |acc,l|
+      name, type, desc, default, options = l
+      acc.merge!(name => desc)  unless desc.nil? || acc.include?(name)
+      acc
+    end
+  end
+
+  def rec_input_defaults(task_name)
+    tasks[task_name].recursive_inputs.inject({}) do |acc,l|
+      name, type, desc, default, options = l
+      acc.merge!(name => default)  unless default.nil? || acc.include?(name)
+      acc
+    end
+  end
+
+  def rec_input_options(task_name)
+    tasks[task_name].recursive_inputs.inject({}) do |acc,l|
+      name, type, desc, default, options = l
+      acc.merge!(name => options) unless options.nil? unless acc.include?(name)
+      acc
+    end
+  end
+
+
+  def rec_input_use(task_name)
+    input_use = {}
+    task = self.tasks[task_name]
+    task.inputs.each do |name,_| 
+      input_use[name] ||= {} 
+      input_use[name][self] ||= []
+      input_use[name][self] << task_name
+    end
+
+    task.deps.inject(input_use) do |acc,p|
+      workflow, task_name = p
+      next if task_name.nil?
+      workflow.rec_input_use(task_name).each do |name,uses|
+        acc[name] ||= {}
+        uses.each do |workflow, task_names|
+          acc[name][workflow] ||= []
+          acc[name][workflow].concat(task_names)
+        end
+      end
+      acc
+    end if task.deps
+
+    input_use
+  end
+  def task_info(name)
+    name = name.to_sym
+    task = tasks[name]
+    raise "No '#{name}' task in '#{self.name}' Workflow" if task.nil?
+    id = File.join(self.name, name.to_s)
+    @task_info ||= {}
+    @task_info[id] ||= begin 
+                         description = task.description
+                         returns = task.returns
+
+                         inputs = rec_inputs(name).uniq
+                         input_types = rec_input_types(name)
+                         input_descriptions = rec_input_descriptions(name)
+                         input_use = rec_input_use(name)
+                         input_defaults = rec_input_defaults(name)
+                         input_options = rec_input_options(name)
+                         extension = task.extension
+
+                         dependencies = tasks[name].deps
+                         { :id => id,
+                           :description => description,
+                           :inputs => inputs,
+                           :input_types => input_types,
+                           :input_descriptions => input_descriptions,
+                           :input_defaults => input_defaults,
+                           :input_options => input_options,
+                           :input_use => input_use,
+                           :returns => returns,
+                           :dependencies => dependencies,
+                           :extension => extension
+                         }
+                       end
+  end
+end

data/lib/scout/workflow/task.rb

@@ -3,6 +3,7 @@ require 'scout/named_array'
 require_relative 'step'
 require_relative 'task/inputs'
 require_relative 'task/dependencies'
+require_relative 'task/info'
 
 module Task
   extend Annotation