scout-gear 10.11.4 → 10.11.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4ce69534bfbce4c385cfc9c23b280c6bb935a9f7228afbf7e585c3f093a5a18
-  data.tar.gz: 98dee8673df93baf527d75df6fb9358b9ec1ec54a37e60c38f0bcb34920fb124
+  metadata.gz: 46332850d0c2cee351bade4c0b7a1d6a46fedebc7de433cc1f0c94d8c3813063
+  data.tar.gz: 1a129853fc38015166f689e3ceb23723c44233fb902977155c5c9c617cf39cba
 SHA512:
-  metadata.gz: 81b897cea0e0d4153fd1d97cae601a17dc75c4c36ebb0c6fe0699a4440d3b7baddbb501fbdb48c409f89386379fcc153a67b7b53b71f95d460366c6993637986
-  data.tar.gz: d0cb23b7fa0b9b04e5719cab7ee365231105fb16e32f62aed37c22f886054de9538922b07cc4495371eb1356fbc9b98707ebfb23635dee847564623b53c50bce
+  metadata.gz: d1994a0a9d77b55b35853946a0887875a3208a6ad5c5d53cc73bc9dffb8ed0d8d77692d03c2685383dbc8cc8613df2e9f977ed5b592cc02448a4a0d1172579af
+  data.tar.gz: 93acf2601660be878b80eb22ee1a91259ee19bc15902d41f3e790695cb192495ae3331a209bec7a4cc9a8bfa8d3de68c17d4307ecdcc3d917854eb9752a7818c

data/.vimproject

@@ -2,6 +2,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
  Rakefile
  README.md
  chats=chats filter="*"{
+  job_chains
+
+
+  intro
+  dev
 
   semaphore
 
@@ -76,8 +81,6 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      queue.rb
 
      trace.rb
-
-     orchestrator.rb
     }
     entity.rb
    }
@@ -193,6 +196,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   }
   workflow=workflow{
    task
+   example
    list
    info
    write_info
@@ -207,6 +211,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    tail
    list
    clean
+   cmd
   }
   resource=resource{
    produce
@@ -304,4 +309,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    workflow.rb
   }
  }
+ doc=doc filter="*"{
+  Association.md
+  Entity.md
+  KnowledgeBase.md
+  Persist.md
+  Semaphore.md
+  TSV.md
+  WorkQueue.md
+  Workflow.md
+ }
 }

data/VERSION

@@ -1 +1 @@
-10.11.4
+10.11.7

data/bin/scout

@@ -21,7 +21,7 @@ if _i = ARGV.index("--dev")
   ARGV.delete dev_dir
 end
 
-if dev_dir.nil? 
+if dev_dir.nil?
   _s = nil
   ARGV.each_with_index do |s,i|
     if s.match(/^--dev(?:=(.*))?/)
@@ -95,7 +95,7 @@ elsif Log.severity.nil?
 
   global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
   if ENV["SCOUT_LOG"]
-    Log.severity = ENV["SCOUT_LOG"].to_i 
+    Log.severity = ENV["SCOUT_LOG"].to_i
   else
     global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
     Log.severity = global_severity.to_i if global_severity
@@ -119,10 +119,10 @@ $scout_command_dir = Scout.scout_commands
 #$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
 SOPT.description =<<EOF
-This command controls many aspects of the Scout framework, from configuration tasks to running applications. 
+This command controls many aspects of the Scout framework, from configuration tasks to running applications.
 
-Commands are implemented in separate files under the Scout path '#{$scout_command_dir}'. 
-Known locations are: #{([$scout_command_dir] + $scout_command_dir.find_all) * ", " }. 
+Commands are implemented in separate files under the Scout path '#{$scout_command_dir}'.
+Known locations are: #{([$scout_command_dir] + $scout_command_dir.find_all) * ", " }.
 You can place your own commads at #{$scout_command_dir.find(:user)}.
 EOF
 
@@ -158,11 +158,11 @@ def scout_usage(prev = nil)
     puts Log.color :magenta, "## COMMANDS"
     puts
     puts Log.color :magenta, "Command:"
-    puts 
+    puts
     puts "  #{File.basename($0)} #{prev * " "}"
-    puts 
+    puts
     puts Log.color :magenta, "Subcommands:"
-    puts 
+    puts
     prev_dir = prev_dir(prev)
     commands(prev).each do |command|
       directory = File.directory? prev_dir[command].find
@@ -177,7 +177,7 @@ def scout_usage(prev = nil)
   true
 end
 
-alias usage scout_usage 
+alias usage scout_usage
 
 def print_error(error, backtrace = nil)
   puts Log.color :magenta, "## ERROR"
@@ -209,7 +209,7 @@ def cmd_alias
     ARGV.replace tokenize_cmd_params(aliases[ARGV[0]]) + ARGV[1..-1]
   end
 end
-    
+
 dir = $scout_command_dir
 $previous_commands = []
 

data/lib/scout/association/fields.rb

@@ -24,14 +24,14 @@ module Association
 
   def self.normalize_specs(spec, all_fields = nil)
     return nil if spec.nil?
-    field, header, format = parse_field_specification spec 
+    field, header, format = parse_field_specification spec
 
     specs = if all_fields.nil? or all_fields.include? field
                [field, header, format]
              else
                if all_fields.nil?
                  begin
-                   identify_entity_format field, all_fields 
+                   identify_entity_format field, all_fields
                  rescue
                    [field, header, format]
                  end
@@ -69,30 +69,30 @@ module Association
     end
 
     if source_specs[0].nil? and target_specs[0].nil?
-      source_specs[0] = key_field 
+      source_specs[0] = key_field
       target_specs[0] = fields[0]
-    elsif source_specs[0].nil? 
+    elsif source_specs[0].nil?
       if target_specs[0] == :key or target_specs[0] == key_field
         source_specs[0] = fields[0]
       else
         source_specs[0] = key_field
       end
-    elsif target_specs[0].nil? 
-      if source_specs[0] == fields.first 
+    elsif target_specs[0].nil?
+      if source_specs[0] == fields.first
         target_specs[0] = key_field
       else
-        target_specs[0] = fields.first 
+        target_specs[0] = fields.first
       end
     end
 
     # If format is specified, then perhaps we need to change the
-    if target_specs[2].nil? 
+    if target_specs[2].nil?
       target_type = Entity.formats[target_specs[1] || target_specs[0]]
       target_specs[2] = format[target_type.to_s] if format
       target_specs[2] = nil if target_specs[2] == target_specs[0] or target_specs[2] == target_specs[1]
     end
 
-    if source_specs[2].nil? 
+    if source_specs[2].nil?
       source_type = Entity.formats[source_specs[1] || source_specs[0]]
       source_specs[2] = format[source_type.to_s] if format
       source_specs[2] = nil if source_specs[2] == source_specs[0] or source_specs[2] == source_specs[1]
@@ -105,7 +105,7 @@ module Association
     return nil if default_format.nil? or default_format.empty?
     default_format.each do |type, format|
       entity_type = Entity.formats[field] || format
-      return format if entity_type.to_s === type 
+      return format if entity_type.to_s === type
     end
     return nil
   end
@@ -118,7 +118,7 @@ module Association
 
     #source_pos = all_fields.index source_field
     #target_pos = all_fields.index target_field
-   
+
     source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
     target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
 
@@ -133,13 +133,13 @@ module Association
     info_fields.delete_at NamedArray.identify_name(info_fields, target_field) if NamedArray.identify_name(info_fields, target_field)
     info_fields.unshift target_field
 
-    field_headers = [target_header] 
+    field_headers = [target_header]
     info_fields[1..-1].each do |field|
       header = case field
-               when String 
+               when String
                  field
                when Numeric
-                 all_fields[field] 
+                 all_fields[field]
                when :key
                  all_fields.first
                end
@@ -147,7 +147,7 @@ module Association
       field_headers << header
     end
 
-    field_pos = info_fields.collect do |f| 
+    field_pos = info_fields.collect do |f|
       p = TSV.identify_field all_fields.first, all_fields[1..-1], f
       p == :key ? 0 : p + 1
     end

data/lib/scout/association/index.rb

@@ -7,8 +7,8 @@ module Association
     target = kwargs.delete :target if kwargs.include?(:target)
 
     persist_options = IndiferentHash.pull_keys kwargs, :persist
-    index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
-      prefix: "Association::Index", 
+    index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true,
+      prefix: "Association::Index",
       other_options: kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format)
 
     index = Persist.tsv(file, kwargs, engine: "BDB", persist_options: index_persist_options) do |data|
@@ -103,7 +103,7 @@ module Association
         end
       end
 
-      matches = source.uniq.inject([]){|acc,e| 
+      matches = source.uniq.inject([]){|acc,e|
         if block_given?
           acc.concat(match(e))
         else
@@ -115,7 +115,7 @@ module Association
 
       target_matches = {}
 
-      matches.each{|code| 
+      matches.each{|code|
         s,sep,t = code.partition "~"
         next if undirected and t > s and source.include? t
         target_matches[t] ||= []
@@ -217,7 +217,7 @@ module Association
 
       tsv = if value_pos
               AssociationItem.incidence self.keys, key_field do |key|
-                if block_given? 
+                if block_given?
                   yield self[key][value_pos]
                 else
                   self[key][value_pos]
@@ -226,7 +226,7 @@ module Association
             elsif block_given?
               AssociationItem.incidence self.keys, key_field, &block
             else
-              AssociationItem.incidence self.keys, key_field 
+              AssociationItem.incidence self.keys, key_field
             end
     end
   end

data/lib/scout/association/item.rb

@@ -9,11 +9,11 @@ module AssociationItem
 
   property :name => :single do
     [source_entity, target_entity].collect{|e| e.respond_to?(:name)? e.name || e : e } * "~"
-  end 
+  end
 
   property :full_name => :single do
     database ? [database, name] * ":" : name
-  end 
+  end
 
   property :invert => :both do
     if Array === self
@@ -73,7 +73,7 @@ module AssociationItem
   end
 
   property :undirected => :both do
-    knowledge_base.undirected(database)
+    knowledge_base.undirected(database) if knowledge_base
   end
 
   property :target_entity => :array2single do
@@ -124,7 +124,7 @@ module AssociationItem
         if info_fields.empty?
           tsv[p] = [source, target]
         else
-          tsv[p] = [source, target].concat v 
+          tsv[p] = [source, target].concat v
         end
       end
     end
@@ -142,6 +142,15 @@ module AssociationItem
     keys
   end
 
+  property :incidence => :array do |*args,**kwargs,&block|
+    AssociationItem.incidence(self,*args,**kwargs,&block)
+  end
+
+  property :adjacency => :array do |*args,**kwargs,&block|
+    AssociationItem.adjacency(self,*args,**kwargs,&block)
+  end
+
+
   def self.incidence(pairs, key_field = nil, &block)
     matrix = {}
     targets = []
@@ -159,7 +168,7 @@ module AssociationItem
         matches[s][t] = value unless value.nil? or (mv = matches[s][t] and value > mv)
       else
         matches[s] ||= Hash.new{false}
-        matches[s][t] ||= true 
+        matches[s][t] ||= true
       end
     end
 
@@ -173,15 +182,16 @@ module AssociationItem
     defined?(TSV)? TSV.setup(matrix, :key_field => (key_field || "Source") , :fields => targets, :type => :list) : matrix
   end
 
+
   def self.adjacency(pairs, key_field = nil, &block)
-    incidence = incidence(pairs, key_field, &block)
+    incidence = pairs.respond_to?(:incidence) ? pairs.incidence(key_field, &block) : incidence(pairs, key_field, &block)
 
     targets = incidence.fields
     adjacency = TSV.setup({}, :key_field => incidence.key_field, :fields => ["Target"], :type => :double)
     TSV.traverse incidence, :into => adjacency, :unnamed => true do |k,values|
       target_values = targets.zip(values).reject{|t,v| v.nil? }.collect{|t,v| [t,v]}
       next if target_values.empty?
-      [k, Misc.zip_fields(target_values)]
+      [k, NamedArray.zip_fields(target_values)]
     end
   end
 
@@ -210,7 +220,7 @@ module AssociationItem
           else
             orig = item.info[key]
             orig = orig.split(";;") if String orig
-            _select_match orig, value 
+            _select_match orig, value
           end
         end
       end

data/lib/scout/association.rb

@@ -130,15 +130,15 @@ module Association
   def self.database(file, *args, **kwargs)
     persist_options = IndiferentHash.pull_keys kwargs, :persist
 
-    database_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
+    database_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true,
       prefix: "Association::Index", serializer: :double, update: true,
-      other_options: kwargs  
+      other_options: kwargs
 
     Persist.tsv(file, kwargs, engine: "BDB", persist_options: database_persist_options) do |data|
       tsv = open(file, *args, **kwargs)
       data.serializer =  TSVAdapter.serializer_module(tsv.type) if data.respond_to?(:serializer)
-      if TSV::Transformer === tsv 
-        tsv.tsv(merge: true, data: data) 
+      if TSV::Transformer === tsv
+        tsv.tsv(merge: true, data: data)
       elsif data.respond_to?(:persistence_path)
         data.merge!(tsv)
         tsv.annotate(data)

data/lib/scout/entity/identifiers.rb

@@ -1,7 +1,7 @@
 module Entity
   def self.identifier_files(field)
     entity_type = Entity.formats[Entity.formats.find(field)]
-    return [] unless entity_type and entity_type.include? Entity::Identified 
+    return [] unless entity_type and entity_type.include? Entity::Identified
     entity_type.identifier_files
   end
 
@@ -20,9 +20,9 @@ module Entity
 
         target_format = case target_format
                         when :name
-                          identity_type.name_format 
+                          identity_type.name_format
                         when :default
-                          identity_type.default_format 
+                          identity_type.default_format
                         else
                           target_format
                         end
@@ -45,14 +45,14 @@ module Entity
       end
     end
 
-    def identifier_files 
+    def identifier_files
       files = identity_type.identifier_files.dup
       return [] if files.nil?
       files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotation_hash.include? :namespace and self.namespace
       if files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }.any?
         Log.warn "Rejecting some identifier files for lack of 'namespace': " + files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } * ", "
       end
-      files.reject!{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } 
+      files.reject!{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }
       files
     end
 

data/lib/scout/entity/property.rb

@@ -29,7 +29,7 @@ module Entity
     def self.single_method(name)
       ("_single_" + name.to_s).to_sym
     end
-     
+
     def self.array_method(name)
       ("_ary_" + name.to_s).to_sym
     end
@@ -76,7 +76,7 @@ module Entity
           self.each do |item|
             begin
               responses[item] = Entity::Property.persist(name, item, type, options.merge(other: {args: args, kwargs: kwargs})) do
-                raise MultipleEntityProperty 
+                raise MultipleEntityProperty
               end
             rescue MultipleEntityProperty
               missing << item

data/lib/scout/entity.rb

@@ -34,7 +34,7 @@ module Entity
       mod = Entity === field ? field : Entity.formats[field]
 
       entity = entity.dup
-      entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity) 
+      entity = (entity.frozen? and not entity.nil?) ? entity.dup : ((Array === entity and dup_array) ? entity.collect{|e| e.nil? ? e : e.dup} : entity)
 
       entity = mod.setup(entity, params)
 

data/lib/scout/knowledge_base/description.rb

@@ -19,7 +19,7 @@ class KnowledgeBase
 
   def self.parse_knowledge_base_doc(doc)
     description, db_description = doc_parse_up_to doc, /^#/, true
-    databases = doc_parse_chunks db_description, /^# (.*)/ 
+    databases = doc_parse_chunks db_description, /^# (.*)/
     IndiferentHash.setup({:description => description.strip, :databases => databases})
   end
 
@@ -32,11 +32,11 @@ class KnowledgeBase
       ""
     end
   end
-  
+
   def database_description_file(name)
     dir[name.to_s + '.md']
   end
-  
+
   def knowledge_base_description_file(name)
     file = dir['README.md']
     return file if file.exists?
@@ -46,11 +46,11 @@ class KnowledgeBase
     source_readme = file.dirname['README.md'] if file
     return source_readme if source_readme  && source_readme.exists?
   end
-  
+
   def description(name)
     return registered_options(name)[:description] if registered_options(name)[:description]
-    
-    return database_description_file(name).read if database_description_file(name).exist? 
+
+    return database_description_file(name).read if database_description_file(name).exist?
 
     if knowledge_base_description_file(name)
       KnowledgeBase.parse_knowledge_base_doc(knowledge_base_description_file(name).read)[:databases][name.to_s.downcase]
@@ -79,21 +79,21 @@ class KnowledgeBase
                        []
                      end
 
-    if source_type 
+    if source_type
       full_description << "Source: #{source_type} - #{source(name)}"
     else
       full_description << "Source: #{source(name)}"
     end
     #full_description.last << ". Accepted formats: #{source_formats*", "}" if source_formats.any?
 
-    if target_type 
+    if target_type
       full_description << "Target: #{target_type} - #{target(name)}"
     else
       full_description << "Target: #{target(name)}"
     end
     #full_description.last << ". Accepted formats: #{target_formats*", "}" if target_formats.any?
-    
-    if undirected?(name) 
+
+    if undirected?(name)
       full_description << "Undirected database, source and target can be reversed."
     end
 

data/lib/scout/knowledge_base/entity.rb

@@ -24,7 +24,7 @@ class KnowledgeBase
     options[:format] = @format[type] if Hash === @format && @format.include?(type)
     namespace = self.namespace
     namespace = db_namespace(database_name) if namespace.nil? and database_name
-    if database_name  
+    if database_name
       database = get_database(database_name)
       if database.entity_options and (database.entity_options[type] or database.entity_options[Entity.formats[type.to_s].to_s])
         options = options.merge(database.entity_options[type] || database.entity_options[Entity.formats[type.to_s].to_s])
@@ -94,7 +94,7 @@ class KnowledgeBase
       TSV.translation_index identifier_files, nil, source(name), :persist => true
     end
   end
-  
+
   def target_index(name)
     Persist.memory("Target index #{name}: KB directory #{dir}") do
       if @identifier_files && @identifier_files.any?
@@ -120,9 +120,9 @@ class KnowledgeBase
 
   def identify_source(name, entity)
     return :all if entity == :all
-    index = begin 
-              source_index(name) 
-            rescue 
+    index = begin
+              source_index(name)
+            rescue
               Log.exception $!
               nil
             end
@@ -133,7 +133,7 @@ class KnowledgeBase
       index[entity] || entity
     end
   end
-  
+
   def identify_target(name, entity)
     return :all if entity == :all
     index = begin target_index(name) rescue nil end

data/lib/scout/knowledge_base/list.rb

@@ -47,7 +47,7 @@ class KnowledgeBase
 
   def load_list(id, entity_type = nil)
     if entity_type
-      path = list_file(id, entity_type) 
+      path = list_file(id, entity_type)
       path = list_file(id) unless path.exists?
     else
       path = list_file(id)

data/lib/scout/knowledge_base/query.rb

@@ -19,7 +19,7 @@ class KnowledgeBase
                when :all
                  {:target => :all, :source => :all}
                when AnnotatedArray
-                 format = entities.format if entities.respond_to? :format 
+                 format = entities.format if entities.respond_to? :format
                  format ||= entities.base_entity.to_s
                  {format => entities.purge}
                when Hash
@@ -31,7 +31,7 @@ class KnowledgeBase
     identify, identify_source, identify_target = entities.merge(options || {}).values_at :identify, :identify_source, :identify_target
 
     source, target = select_entities(name, entities, options)
-    
+
     source = identify_source(name, source) if identify_source
     target = identify_target(name, target) if identify_target
 
@@ -46,7 +46,7 @@ class KnowledgeBase
 
     setup(name, matches)
 
-    matches = matches.select(&block) if block_given? 
+    matches = matches.select(&block) if block_given?
 
     matches
   end
@@ -89,7 +89,7 @@ class KnowledgeBase
   def neighbours(name, entity)
     hash = _neighbours(name, entity)
     IndiferentHash.setup(hash)
-    setup(name, hash[:children]) if hash[:children] 
+    setup(name, hash[:children]) if hash[:children]
     setup(name, hash[:parents], true) if hash[:parents]
     hash
   end

data/lib/scout/knowledge_base/registry.rb

@@ -23,7 +23,7 @@ class KnowledgeBase
     return [] unless @registry
     (@registry.keys + present_databases).uniq
   end
-  
+
   def database_file(name)
     if @registry[name].nil?
       nil
@@ -72,8 +72,8 @@ class KnowledgeBase
     name = name.to_s
     options[:namespace] ||= self.namespace unless self.namespace.nil?
     @indices ||= IndiferentHash.setup({})
-    @indices[[name, options]] ||= 
-      begin 
+    @indices[[name, options]] ||=
+      begin
         if options.empty?
           key = name.to_s
         elsif options[:key]
@@ -131,11 +131,11 @@ class KnowledgeBase
   def get_database(name, options = {})
     options = options.dup
     if self.namespace == options[:namespace]
-      options.delete(:namespace) 
+      options.delete(:namespace)
     end
     @databases ||= IndiferentHash.setup({})
-    @databases[[name, options]] ||= 
-      begin 
+    @databases[[name, options]] ||=
+      begin
         fp = Log.fingerprint([name,options])
 
         if options.empty?