scout-gear 10.7.3 → 10.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 469b6e128a39e5612de5698005de8f4859c015bf3d65a5490a58f2f9f0312a39
-  data.tar.gz: 0a76edd86052c35af31f6de227c0bc43c71bb6b64fdc757e1179bf88a4ce8701
+  metadata.gz: 60a8c5a91d83e96aaa721154fa4f7fdd63d2699c691b8c99962c6ba5561365eb
+  data.tar.gz: 4b2303f3db25c1316286e8c6edbd8d76be66ba8b851626393f0c0b5ea34aac1d
 SHA512:
-  metadata.gz: c8753c031fda58c461afdefa3412a0d7996c6d5139a30653b1e6fcfaeba7fdf31b4d300bb2cb07060e7eb12d63891fc7a157cec3964c59d5cc68584ffc8823c8
-  data.tar.gz: 05cbb772edf0384f2889ad1be2cc597425d071da8c488364a8c72b23e6b7098aea0089247c180da09165f3eb7d899de494dca9ce2a00f285dcfaa564c8dc3e68
+  metadata.gz: 0eb9761f9cc9acab8842372de6bbe9f69413bc9d6b2c1b05ebae6c51e40ca292fbce9edce27d1c5d26a7e298273e700881bdd39f8bac3441b1e9af604c443eb0
+  data.tar.gz: cb46e73a548bca55f41e088167eff7494691a6fdd61ed17f7a06f6a1a0257679441b7d62a56e67a376943e4e0a96acd39d3a4ee5888550c1721c5d79e4f7e6a4

data/.gitmodules

@@ -1,4 +0,0 @@
-[submodule "modules/rbbt-util"]
-	path = modules/rbbt-util
-	url = https://github.com/mikisvaz/rbbt-util.git
-	branch = scout

data/.vimproject

@@ -8,6 +8,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
    workflow=workflow{
     exceptions.rb
+    export.rb
     definition.rb
     documentation.rb
     usage.rb
@@ -31,12 +32,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     task=task{
      dependencies.rb
      inputs.rb
+     info.rb
     }
     deployment.rb
     deployment=deployment{
      trace.rb
      orchestrator.rb
     }
+    entity.rb
    }
    work_queue.rb
    work_queue=work_queue{
@@ -121,12 +124,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     enrichment.rb
     list.rb
    }
-   offsite.rb
-   offsite=offsite{
-    ssh.rb
-    sync.rb
-    step.rb
-   }
    semaphore.rb
   }
   scout-gear.rb
@@ -143,7 +140,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   doc
   update
   template
-  offsite
   kb=kb{
    config
    entities
@@ -185,7 +181,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   scout=scout{
    test_association.rb
    test_entity.rb
-   test_offsite.rb
    test_semaphore.rb
    test_tsv.rb
    test_workflow.rb
@@ -247,12 +242,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      test_tokyocabinet.rb
     }
    }
-   offsite=offsite{
-    test_ssh.rb
-    test_step.rb
-    test_sync.rb
-    test_task.rb
-   }
   }
  }
  share=share{

data/Rakefile

@@ -12,7 +12,7 @@ Juwelier::Tasks.new do |gem|
   gem.homepage = "http://github.com/mikisvaz/scout-gear"
   gem.license = "MIT"
   gem.summary = %Q{basic gear for scouts}
-  gem.description = %Q{Temporary files, logs, path, resources, persistence, workflows, TSV, etc.}
+  gem.description = %Q{Scout gear: workflow, TSVs, persistence, entities, associations, and knowledge_bases.}
   gem.email = "mikisvaz@gmail.com"
   gem.authors = ["Miguel Vazquez"]
 
@@ -26,10 +26,7 @@ Juwelier::Tasks.new do |gem|
   gem.add_runtime_dependency 'RubyInline'
   #gem.add_runtime_dependency 'tokyocabinet'
 
-  gem.add_development_dependency "rdoc", "~> 3.12"
-  gem.add_development_dependency "bundler", "~> 1.0"
   gem.add_development_dependency "juwelier", "~> 2.1.0"
-  gem.add_development_dependency "simplecov", ">= 0"
 end
 Juwelier::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-10.7.3
+10.7.5

data/bin/scout

@@ -52,6 +52,16 @@ if dev_dir
   end
 end
 
+requires = nil
+if _i = ARGV.index("--require")
+  requires = ARGV[_i+1].split(",")
+  ARGV.delete_at _i + 1
+  ARGV.delete_at _i
+  requires.each do |p|
+    require p
+  end
+end
+
 require 'scout-gear'
 
 require 'scout/simple_opt'
@@ -97,8 +107,11 @@ if config_keys = options.delete(:config_keys)
   end
 end
 
-$scout_command_dir = Scout.bin.scout
-$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
+#$scout_command_dir = Scout.bin.scout
+#$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
+
+$scout_command_dir = Scout.scout_commands
+#$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
 SOPT.description =<<EOF
 This command controls many aspects of the Scout framework, from configuration tasks to running applications. 

data/lib/scout/association/index.rb

@@ -141,7 +141,7 @@ module Association
                        new.extend Index
                        new
                      else
-                       FileUtils.mkdir_p File.dirname(reverse_filename) unless File.exist?(File.dirname(reverse_filename))
+                       Open.mkdir File.dirname(reverse_filename) unless Open.exist?(File.dirname(reverse_filename))
 
                        new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
 

data/lib/scout/association.rb

@@ -11,7 +11,9 @@ module Association
     target = kwargs.delete :target if kwargs.include?(:target)
 
     if Path.is_filename?(obj)
-      options = TSV.parse_options(obj).merge(kwargs)
+      tsv_header_options = TSV.parse_options(obj)
+      tsv_header_options = tsv_header_options.slice(TSV.acceptable_parser_options)
+      options = tsv_header_options.merge(kwargs)
     else
       options = kwargs.dup
     end
@@ -116,11 +118,12 @@ module Association
     persist_options = IndiferentHash.pull_keys kwargs, :persist
 
     database_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
-      prefix: "Association::Index", serializer: :list,
+      prefix: "Association::Index", serializer: :double,
       other_options: kwargs  
 
     Persist.tsv(file, kwargs, engine: "BDB", persist_options: database_persist_options) do |data|
       tsv = open(file, *args, **kwargs)
+      data.serializer =  TSVAdapter.serializer_module(tsv.type) if data.respond_to?(:serializer)
       if TSV::Transformer === tsv 
         tsv.tsv(merge: true, data: data) 
       elsif data.respond_to?(:persistence_path)

data/lib/scout/entity/identifiers.rb

@@ -64,13 +64,16 @@ module Entity
       Persist.memory("Entity index #{identity_type}: #{format} (from #{source || "All"})", :persist => true, :format => format, :source => source) do
         source ||= self.respond_to?(:format)? self.format : nil
 
+        saved_exception = nil
         begin
           index = TSV.translation_index(identifier_files, source, format, :persist => true)
           raise "No index from #{ Misc.fingerprint source } to #{ Misc.fingerprint format }: #{Misc.fingerprint identifier_files}" if index.nil?
           index.unnamed = true
           index
         rescue
-          raise $! if source.nil?
+          raise saved_exception || $! if source.nil?
+          Log.debug "Retrying identifier index without specifying source"
+          saved_exception = $!
           source = nil
           retry
         end
@@ -93,7 +96,7 @@ module Entity
 
     name = default if name.nil?
 
-    self.send(:include, Entity::Identified) unless Entity::Identified === self
+    self.send(:include, Entity::Identified) unless self.include?(Entity::Identified)
 
     self.format = all_fields
     @formats ||= []

data/lib/scout/entity/property.rb

@@ -62,7 +62,6 @@ module Entity
 
       properties.push name
 
-
       entity_class = self
       if type == :multiple
         self.define_method(real_method) do |*args,**kwargs|
@@ -89,7 +88,7 @@ module Entity
 
             new_responses = missing.instance_exec(*args, **kwargs, &block)
 
-            missing.each do |item,i|
+            missing.each do |item|
               responses[item] = Entity::Property.persist(name, item, type, options) do
                 Array === new_responses ? new_responses[item.container_index] : new_responses[item]
               end
@@ -149,7 +148,7 @@ module Entity
 
     def persist(name, type = :marshal, options = {})
       options = IndiferentHash.add_defaults options, persist: true,
-        dir: File.join(Entity.entity_property_cache, self.to_s, name.to_s)
+        dir: Entity.entity_property_cache[self.to_s][name.to_s]
       @persisted_methods ||= {}
       @persisted_methods[name] = [type, options]
     end

data/lib/scout/knowledge_base/entity.rb

@@ -63,7 +63,7 @@ class KnowledgeBase
     entities.collect{|entity| Entity.formats[entity] }.uniq
   end
 
-  def identifier_files(name)
+  def database_identifier_files(name)
     get_database(name).identifier_files.dup + self.identifier_files
   end
 
@@ -73,7 +73,11 @@ class KnowledgeBase
 
   def source_index(name)
     Persist.memory("Source index #{name}: KB directory #{dir}") do
-      identifier_files = identifier_files(name)
+      if @identifier_files && @identifier_files.any?
+        identifier_files = @identifier_files
+      else
+        identifier_files = database_identifier_files(name)
+      end
       identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
       identifier_files.uniq!
       identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
@@ -97,7 +101,12 @@ class KnowledgeBase
 
   def identify_source(name, entity)
     return :all if entity == :all
-    index = begin source_index(name) rescue nil end
+    index = begin 
+              source_index(name) 
+            rescue 
+              Log.exception $!
+              nil
+            end
     return entity if index.nil?
     if Array === entity
       entity.collect{|e| index[e] || e }

data/lib/scout/knowledge_base/registry.rb

@@ -43,9 +43,11 @@ class KnowledgeBase
   end
 
   def undirected(name)
-    description(name)[2]
+    description(name).length == 3
   end
 
+  alias undirected? undirected
+
   def get_index(name, options = {})
     name = name.to_s
     options[:namespace] ||= self.namespace unless self.namespace.nil?

data/lib/scout/persist/engine/tokyocabinet.rb

@@ -1,105 +1,112 @@
-require 'tokyocabinet'
-
-module ScoutCabinet
-  attr_accessor :persistence_path, :persistence_class
-
-  def self.open(path, write = true, tokyocabinet_class = TokyoCabinet::HDB)
-    path = path.find if Path === path
-    if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
-      big = true
-      tokyocabinet_class = tokyocabinet_class.split(":").first
-    else
-      big = false
-    end
+begin
+  require 'tokyocabinet'
+  continue = true
+rescue Exception
+  Log.warn "The Tokyocabinet gem could not be loaded: TSV persistence may not work"
+  continue = false
+end
 
-    dir = File.dirname(File.expand_path(path))
-    Open.mkdir(dir) unless File.exist?(dir)
+if continue
+  module ScoutCabinet
+    attr_accessor :persistence_path, :persistence_class
 
-    tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
-    tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
-    tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
+    def self.open(path, write = true, tokyocabinet_class = TokyoCabinet::HDB)
+      path = path.find if Path === path
+      if String === tokyocabinet_class && tokyocabinet_class.include?(":big")
+        big = true
+        tokyocabinet_class = tokyocabinet_class.split(":").first
+      else
+        big = false
+      end
 
-    # Hack - Ignore warning: undefining the allocator of T_DATA class
-    # TokyoCabinet::HDB_data
-    database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
+      dir = File.dirname(File.expand_path(path))
+      Open.mkdir(dir) unless File.exist?(dir)
 
-    if big and not Open.exists?(path)
-      database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
-    end
+      tokyocabinet_class = tokyocabinet_class.to_s if Symbol === tokyocabinet_class
+      tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
+      tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
 
-    flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
-    database.close 
+      # Hack - Ignore warning: undefining the allocator of T_DATA class
+      # TokyoCabinet::HDB_data
+      database = Log.ignore_stderr do Persist::CONNECTIONS[path] ||= tokyocabinet_class.new end
 
-    if !database.open(path, flags)
-      ecode = database.ecode
-      raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
-    end
+      if big and not Open.exists?(path)
+        database.tune(nil, nil, nil, tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE) 
+      end
 
-    database.extend ScoutCabinet
-    database.persistence_path ||= path
-    database.persistence_class = tokyocabinet_class
+      flags = (write ? tokyocabinet_class::OWRITER | tokyocabinet_class::OCREAT : tokyocabinet_class::OREADER)
+      database.close 
 
-    database.open(path, tokyocabinet_class::OREADER)
+      if !database.open(path, flags)
+        ecode = database.ecode
+        raise "Open error: #{database.errmsg(ecode)}. Trying to open file #{path}"
+      end
 
-    database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
+      database.extend ScoutCabinet
+      database.persistence_path ||= path
+      database.persistence_class = tokyocabinet_class
 
-    Persist::CONNECTIONS[path] = database
+      database.open(path, tokyocabinet_class::OREADER)
 
-    database
-  end
+      database.define_singleton_method(:fingerprint){ "#{self.persistence_class}:#{self.persistence_path}" }
 
-  def close
-    @closed = true
-    @writable = false
-    super
-  end
+      Persist::CONNECTIONS[path] = database
 
-  def read(force = false)
-    return if ! @writable && ! @closed && ! force
-    self.close
-    if !self.open(@persistence_path, persistence_class::OREADER)
-      ecode = self.ecode
-      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      database
     end
 
-    @writable = false
-    @closed = false
-
-    self
-  end
+    def close
+      @closed = true
+      @writable = false
+      super
+    end
 
-  def write?
-    @writable
-  end
+    def read(force = false)
+      return if ! @writable && ! @closed && ! force
+      self.close
+      if !self.open(@persistence_path, persistence_class::OREADER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
 
-  def closed?
-    @closed
-  end
+      @writable = false
+      @closed = false
 
+      self
+    end
 
-  def write(force = true)
-    return if write? && ! closed? && ! force
-    self.close
+    def write?
+      @writable
+    end
 
-    if !self.open(@persistence_path, persistence_class::OWRITER)
-      ecode = self.ecode
-      raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+    def closed?
+      @closed
     end
 
-    @writable = true
-    @closed = false
 
-    self
-  end
+    def write(force = true)
+      return if write? && ! closed? && ! force
+      self.close
 
-  def write_and_read
-    begin
-      write
-      yield
-    ensure
-      read
+      if !self.open(@persistence_path, persistence_class::OWRITER)
+        ecode = self.ecode
+        raise "Open error: #{self.errmsg(ecode)}. Trying to open file #{@persistence_path}"
+      end
+
+      @writable = true
+      @closed = false
+
+      self
+    end
+
+    def write_and_read
+      begin
+        write
+        yield
+      ensure
+        read
+      end
     end
-  end
 
   def write_and_close
     begin
@@ -139,4 +146,5 @@ module ScoutCabinet
   end
 
   alias load_stream importtsv
+  end
 end

data/lib/scout/persist/tsv/adapter/base.rb

@@ -23,6 +23,7 @@ module TSVAdapter
   end
 
   def save_annotation_hash
+    self.close
     self.with_write do
       self.orig_set(ANNOTATION_ATTR_HASH_KEY, ANNOTATION_ATTR_HASH_SERIALIZER.dump(self.annotation_hash))
     end
@@ -165,15 +166,6 @@ module TSVAdapter
     end
   end
 
-  def with_write(*args, &block)
-    if @write
-      yield
-    elsif @closed
-      write_and_close &block
-    else
-      write_and_read &block
-    end
-  end
 
   def close(*args)
     begin
@@ -250,12 +242,11 @@ module TSVAdapter
 
     lock do
       write(true) if closed? || ! write?
-      res = begin
-              yield
-            ensure
-              close
-            end
-      res
+      begin
+        yield
+      ensure
+        close
+      end
     end
   end
 
@@ -272,18 +263,13 @@ module TSVAdapter
       return yield
     else
       if self.read?
-        self.write_and_read do
-          return yield
-        end
+        self.write_and_read(&block)
       else
-        self.write_and_close do
-          return yield
-        end
+        self.write_and_close(&block)
       end
     end
   end
 
-
   def read_and_close
     if read? || write?
       begin

data/lib/scout/tsv/change_id/translate.rb

@@ -40,7 +40,7 @@ module TSV
           target_file = target_files.select{|file| fields = file_fields[file]; (fields & middle_fields).any? }.collect{|file,f| file }.first
           [source_file, middle_file, target_file]
         else
-          raise "Could not traverse identifier path from #{Log.fingerprint source} to #{Log.fingerprint target} in #{Log.fingerprint file_fields}"
+          raise "Could not traverse identifier path from #{Log.fingerprint source} to #{Log.fingerprint target}. #{file_fields.empty? ? "No identifier files" : Log.fingerprint(file_fields)}"
         end
       end
     end
@@ -55,7 +55,13 @@ module TSV
     files = [files] unless Array === files
 
     files.each do |file|
-      next if Path === file && ! Open.exist?(file)
+      #next if Path === file && ! Open.exist?(file)
+      begin
+        file = file.produce if Path === file
+      rescue
+        Log.warn $!.message
+        next
+      end
       file = file.find if Path === file
       file_fields[file] = all_fields(file)
     end

data/lib/scout/tsv/open.rb

@@ -151,6 +151,8 @@ module Open
                   res = block.call(line)
                   callback.call res if callback
                 end
+                obj.close
+                obj.join if obj.respond_to? :join
               else
                 Log.low "Traverse stream with parser #{Log.fingerprint obj}"
                 parser = options[:sep].nil? ? TSV::Parser.new(obj) : TSV::Parser.new(obj, sep: options[:sep])

data/lib/scout/tsv/parser.rb

@@ -1,5 +1,15 @@
 require 'scout/named_array'
 module TSV
+  def self.acceptable_parser_options(func = nil)
+    if func.nil?
+      TSV.method(:parse_line).parameters.collect{|a| a.last } +
+        TSV.method(:parse_stream).parameters.collect{|a| a.last } +
+        TSV.method(:parse).parameters.collect{|a| a.last } - [:line, :block]
+    else
+      TSV.method(func).parameters.collect{|a| a.last }
+    end.uniq
+  end
+
   def self.cast_value(value, cast)
     if Array === value
       value.collect{|e| cast_value(e, cast) }

data/lib/scout/tsv/transformer.rb

@@ -175,5 +175,17 @@ module TSV
     end
     res
   end
+
+  def head(max=10)
+    res = self.annotate({})
+    transformer = Transformer.new self, res
+    i = 0
+    transformer.traverse do |k,v|
+      i += 1
+      break if i > max
+      [k, v]
+    end
+    res
+  end
 end
 

data/lib/scout/tsv/util/process.rb

@@ -30,9 +30,9 @@ module TSV
       when type == :flat
         self[key] = new_values
       else
-        if ! values[field_pos].frozen? && ((String === values[field_pos] && String === new_values) ||
+        if ! values[field_pos].frozen? && ! NamedArray === values && ((String === values[field_pos] && String === new_values) ||
           (Array === values[field_pos] && Array === new_values))
-           values[field_pos].replace new_values
+          values[field_pos].replace new_values
         else
           values[field_pos] = new_values
         end

data/lib/scout/workflow/definition.rb

@@ -13,8 +13,12 @@ module Workflow
 
   end
 
+  def to_s
+    @name || super
+  end
+
   def name
-    @name ||= self.to_s
+    @name || to_s
   end
 
   def helpers
@@ -192,7 +196,7 @@ module Workflow
           when 'true'
             dep.clean
           when 'recursive'
-            (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
+            (dep.dependencies.to_a + dep.rec_dependencies.to_a).uniq.each do |d|
               next if d.overriden
               d.clean unless Scout::Config.get(:remove_dep, "task:#{d.task_signature}", "task:#{d.task_name}", "workflow:#{d.workflow.name}", :default => true).to_s == 'false'
             end

data/lib/scout/workflow/deployment/trace.rb

@@ -164,7 +164,7 @@ module Workflow
   def self.trace(seed_jobs, options = {})
     jobs = []
     seed_jobs.each do |step|
-      jobs += step.rec_dependencies + [step]
+      jobs += step.rec_dependencies.to_a + [step]
       step.info[:archived_info].each do |path,ainfo|
         next unless Hash === ainfo
         archived_step = Step.new path