scout-gear 10.7.2 → 10.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1182999631b54e94f842c90017e1f97dde904ddafe47129cd80b51097714a1c4
-  data.tar.gz: ec736a5762fc2e58450a7955dffffe61eb73584f0bdc53bf3cd368e194ae7cbf
+  metadata.gz: 92262edf62404f5935bf749d3ab71d2ce39ffc7292a1b3d860fd5dc712501fcb
+  data.tar.gz: 47f69a80c0a135d3a9b3b42558e7b91e766b7781ff2360cd8344ef9c0f7ac8b9
 SHA512:
-  metadata.gz: 66a5c6ab90c2aa047cfbd000731eaec3913994880adddc9ac2daac9802ee2ebef14a054c1e292200624a6411ac2244d5f915de8d3ed9113d1237edc3795ab027
-  data.tar.gz: 27a0e87ac1360420bffc87b34b01eb6f21631fdf3f6ef95983fb46152f6e0e390200518dd2bebcc95c880e74d6f6f8a98a76577be2fbad0aea7d1fbe9c0720b1
+  metadata.gz: 3d532c6a4ac2bd3d0b5584761531b0364f381c317a9864b85b22c7a23bd61dbf2b31d1655f5fc7085c893c843f1b3f36a1b23fffae4c4b9dc2253352f3f5febf
+  data.tar.gz: 50e36c694239f0d74528b9156435b8769d16ae172dd87029ee741c0002b8ef3606f5088fc2c8f881d413b78f1f00f66134f06fc14129da3015648546130809c6

data/.vimproject

@@ -31,6 +31,7 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     task=task{
      dependencies.rb
      inputs.rb
+     info.rb
     }
     deployment.rb
     deployment=deployment{
@@ -38,38 +39,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      orchestrator.rb
     }
    }
-   semaphore.rb
    work_queue.rb
    work_queue=work_queue{
     socket.rb
     worker.rb
    }
-
-   persist=persist{
-    engine.rb
-    engine=engine{
-     tokyocabinet.rb
-     fix_width_table.rb
-     tkrzw.rb
-     packed_index.rb
-     sharder.rb
-    }
-    tsv.rb
-    tsv=tsv{
-     adapter.rb
-     serialize.rb
-     adapter=adapter{
-      base.rb
-
-      fix_width_table.rb
-      packed_index.rb
-      tkrzw.rb
-      tokyocabinet.rb
-      sharder.rb
-     }
-    }
-   }
-
    tsv.rb
    tsv=tsv{
     util.rb
@@ -101,6 +75,30 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     open.rb
     csv.rb
    }
+   persist=persist{
+    engine.rb
+    engine=engine{
+     tokyocabinet.rb
+     fix_width_table.rb
+     tkrzw.rb
+     packed_index.rb
+     sharder.rb
+    }
+    tsv.rb
+    tsv=tsv{
+     adapter.rb
+     serialize.rb
+     adapter=adapter{
+      base.rb
+
+      fix_width_table.rb
+      packed_index.rb
+      tkrzw.rb
+      tokyocabinet.rb
+      sharder.rb
+     }
+    }
+   }
    entity.rb
    entity=entity{
     property.rb
@@ -111,17 +109,26 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
    }
    association.rb
    association=association{
-    fields.rb
     index.rb
+    fields.rb
     item.rb
    }
-
+   knowledge_base.rb
+   knowledge_base=knowledge_base{
+    registry.rb
+    entity.rb
+    query.rb
+    traverse.rb
+    enrichment.rb
+    list.rb
+   }
    offsite.rb
    offsite=offsite{
     ssh.rb
     sync.rb
     step.rb
    }
+   semaphore.rb
   }
   scout-gear.rb
   workflow-scout.rb
@@ -138,10 +145,14 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   update
   template
   offsite
-  db=db{
+  kb=kb{
+   config
+   entities
    register
-   query
    show
+   query
+   traverse
+   list
   }
   workflow=workflow{
    task
@@ -164,8 +175,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   test_helper.rb
   test_scout-gear.rb
   test_scout.rb
-  data=data{
+  data=data filter="*"{
    person=person{
+    brothers
+    identifiers
+    marriages
+    parents
    }
   }
   scout=scout{

data/VERSION

@@ -1 +1 @@
-10.7.2
+10.7.4

data/bin/scout

@@ -52,6 +52,16 @@ if dev_dir
   end
 end
 
+requires = nil
+if _i = ARGV.index("--require")
+  requires = ARGV[_i+1].split(",")
+  ARGV.delete_at _i + 1
+  ARGV.delete_at _i
+  requires.each do |p|
+    require p
+  end
+end
+
 require 'scout-gear'
 
 require 'scout/simple_opt'
@@ -97,8 +107,11 @@ if config_keys = options.delete(:config_keys)
   end
 end
 
-$scout_command_dir = Scout.bin.scout
-$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
+#$scout_command_dir = Scout.bin.scout
+#$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
+
+$scout_command_dir = Scout.scout_commands
+#$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
 
 SOPT.description =<<EOF
 This command controls many aspects of the Scout framework, from configuration tasks to running applications. 

data/lib/scout/association/index.rb

@@ -2,6 +2,10 @@ require 'scout/annotation'
 module Association
 
   def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+    IndiferentHash.setup(kwargs)
+    source = kwargs.delete :source if kwargs.include?(:source)
+    target = kwargs.delete :target if kwargs.include?(:target)
+
     persist_options = IndiferentHash.pull_keys kwargs, :persist
     index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
       prefix: "Association::Index", 

data/lib/scout/association/item.rb

@@ -102,7 +102,7 @@ module AssociationItem
   property :info => :array2single do
     fields = self.info_fields
 
-    return [{}] * self.length if fields.nil? or fields.empty?
+    next [{}] * self.length if fields.nil? or fields.empty?
 
     value = self.value
     value.collect{|v|

data/lib/scout/association.rb

@@ -6,10 +6,14 @@ require_relative 'association/item'
 
 module Association
   def self.open(obj, source: nil, target: nil, fields: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
-
+    IndiferentHash.setup(kwargs)
+    source = kwargs.delete :source if kwargs.include?(:source)
+    target = kwargs.delete :target if kwargs.include?(:target)
 
     if Path.is_filename?(obj)
-      options = TSV.parse_options(obj).merge(kwargs)
+      tsv_header_options = TSV.parse_options(obj)
+      tsv_header_options = tsv_header_options.slice(TSV.acceptable_parser_options)
+      options = tsv_header_options.merge(kwargs)
     else
       options = kwargs.dup
     end
@@ -29,9 +33,12 @@ module Association
 
     type, identifiers = IndiferentHash.process_options options, :type, :identifiers
 
-    if source_format
+    if source_format || target_format
       translation_files = [TSV.identifier_files(obj), Entity.identifier_files(source_format), identifiers].flatten.compact
-      translation_files.collect!{|f| Path.is_filename?(f, false) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
+      translation_files.collect!{|f| (Path.is_filename?(f, false) && options[:namespace]) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
+    end
+
+    if source_format
       source_index = begin
                        TSV.translation_index(translation_files, source_header, source_format)
                      rescue
@@ -40,8 +47,6 @@ module Association
     end
 
     if target_format
-      translation_files = [TSV.identifier_files(obj), Entity.identifier_files(target_format), identifiers].flatten.compact
-      translation_files.collect!{|f| Path.is_filename?(f, false) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
       target_index = begin
                        TSV.translation_index(translation_files, field_headers.first, target_format)
                      rescue
@@ -82,7 +87,7 @@ module Association
 
     if source_index.nil? && target_index.nil?
       if TSV === obj
-        IndiferentHash.pull_keys kwargs, :persist
+        IndiferentHash.pull_keys options, :persist
         type = options[:type] || obj.type
         res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **options.merge(type: type, merge: true)
       else
@@ -109,8 +114,25 @@ module Association
     transformer
   end
 
-  def self.database(*args, **kwargs)
-    tsv = open(*args, **kwargs)
-    TSV::Transformer === tsv ? tsv.tsv(merge: true) : tsv
+  def self.database(file, *args, **kwargs)
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+
+    database_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
+      prefix: "Association::Index", serializer: :double,
+      other_options: kwargs  
+
+    Persist.tsv(file, kwargs, engine: "BDB", persist_options: database_persist_options) do |data|
+      tsv = open(file, *args, **kwargs)
+      data.serializer =  TSVAdapter.serializer_module(tsv.type) if data.respond_to?(:serializer)
+      if TSV::Transformer === tsv 
+        tsv.tsv(merge: true, data: data) 
+      elsif data.respond_to?(:persistence_path)
+        data.merge!(tsv)
+        tsv.annotate(data)
+        data
+      else
+        tsv
+      end
+    end
   end
 end

data/lib/scout/entity/identifiers.rb

@@ -1,6 +1,6 @@
 module Entity
   def self.identifier_files(field)
-    entity_type = Entity.formats[field]
+    entity_type = Entity.formats[Entity.formats.find(field)]
     return [] unless entity_type and entity_type.include? Entity::Identified 
     entity_type.identifier_files
   end
@@ -48,7 +48,7 @@ module Entity
     def identifier_files 
       files = identity_type.identifier_files.dup
       return [] if files.nil?
-      files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotations.include? :namespace and self.namespace
+      files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotation_hash.include? :namespace and self.namespace
       if files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }.any?
         Log.warn "Rejecting some identifier files for lack of 'namespace': " << files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } * ", "
       end

data/lib/scout/entity/property.rb

@@ -1,3 +1,4 @@
+require 'scout/exceptions'
 module Entity
   class << self
     attr_accessor :entity_property_cache
@@ -61,7 +62,6 @@ module Entity
 
       properties.push name
 
-
       entity_class = self
       if type == :multiple
         self.define_method(real_method) do |*args,**kwargs|
@@ -88,7 +88,7 @@ module Entity
 
             new_responses = missing.instance_exec(*args, **kwargs, &block)
 
-            missing.each do |item,i|
+            missing.each do |item|
               responses[item] = Entity::Property.persist(name, item, type, options) do
                 Array === new_responses ? new_responses[item.container_index] : new_responses[item]
               end

data/lib/scout/knowledge_base/enrichment.rb

@@ -0,0 +1,9 @@
+require 'rbbt/knowledge_base/registry'
+class KnowledgeBase
+  def enrichment(name, entities, options = {})
+    require 'rbbt/statistics/hypergeometric'
+    database = get_database(name, options)
+    entities = identify_source name, entities
+    database.enrichment entities, database.fields.first, :persist => false
+  end
+end

data/lib/scout/knowledge_base/entity.rb

@@ -0,0 +1,152 @@
+require_relative '../entity'
+
+class KnowledgeBase
+
+  def select_entities(name, entities, options = {})
+    index = get_index(name, options)
+
+    source_field = index.source_field
+    target_field = index.target_field
+
+    source_type = source_type name
+    target_type = target_type name
+
+    source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] || entities[:both]
+    target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s] || entities[:both]
+
+    [source_entities, target_entities]
+  end
+
+  def entity_options_for(type, database_name = nil)
+    entity_options = self.entity_options
+    IndiferentHash.setup entity_options if entity_options and not IndiferentHash === entity_options
+    options = entity_options[type.to_s] || entity_options[Entity.formats[type].to_s] || {}
+    options[:format] = @format[type] if Hash === @format && @format.include?(type)
+    namespace = self.namespace
+    namespace = db_namespace(database_name) if namespace.nil? and database_name
+    if database_name  
+      database = get_database(database_name)
+      if database.entity_options and (database.entity_options[type] or database.entity_options[Entity.formats[type.to_s].to_s])
+        options = options.merge(database.entity_options[type] || database.entity_options[Entity.formats[type.to_s].to_s])
+      end
+    end
+    options
+  end
+
+  def annotate(entities, type, database = nil)
+    format = @format[type] || type
+    entity_options = entity_options_for(type, database)
+    Entity.prepare_entity(entities, format, entity_options)
+  end
+
+  def translate(entities, type)
+    if format = @format[type] and (entities.respond_to? :format and format != entities.format)
+      entities.to format
+    else
+      entities
+    end
+  end
+
+  def source_type(name)
+    Entity.formats[Entity.formats.find(source(name))]
+  end
+
+  def target_type(name)
+    Entity.formats[Entity.formats.find(target(name))]
+  end
+
+  def entities
+    all_databases.inject([]){|acc,name| acc << source(name); acc << target(name)}.uniq
+  end
+
+  def entity_types
+    entities.collect{|entity| Entity.formats[entity] }.uniq
+  end
+
+  def database_identifier_files(name)
+    get_database(name).identifier_files.dup + self.identifier_files
+  end
+
+  def db_namespace(name)
+    get_database(name).namespace
+  end
+
+  def source_index(name)
+    Persist.memory("Source index #{name}: KB directory #{dir}") do
+      if @identifier_files && @identifier_files.any?
+        identifier_files = @identifier_files
+      else
+        identifier_files = database_identifier_files(name)
+      end
+      identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if not namespace and db_namespace(name)
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, nil, source(name), :persist => true
+    end
+  end
+  
+  def target_index(name)
+    Persist.memory("Target index #{name}: KB directory #{dir}") do
+      identifier_files = identifier_files(name)
+      identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if namespace.nil? and db_namespace(name)
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, nil, target(name), :persist => true
+    end
+  end
+
+  def identify_source(name, entity)
+    return :all if entity == :all
+    index = begin 
+              source_index(name) 
+            rescue 
+              Log.exception $!
+              nil
+            end
+    return entity if index.nil?
+    if Array === entity
+      entity.collect{|e| index[e] || e }
+    else
+      index[entity] || entity
+    end
+  end
+  
+  def identify_target(name, entity)
+    return :all if entity == :all
+    index = begin target_index(name) rescue nil end
+    return entity if index.nil?
+    if Array === entity
+      entity.collect{|e| index[e] || e }
+    else
+      index[entity] || entity
+    end
+  end
+
+  def identify(name, entity)
+    identify_source(name, entity) || identify_target(name, entity)
+  end
+
+  def define_entity_modules
+    entity_options.each do |entity,options|
+      next unless options[:identifiers]
+      identifiers = options[:identifiers]
+      identifiers = identifiers.split(",") unless Array === identifiers
+      m = begin
+            Object.const_get entity
+          rescue
+            m = Module.new
+            m.extend Entity
+            m.include Entity::Identified
+            Object.const_set entity, m
+          end
+
+      identifiers.each do |file|
+        m.add_identifiers Path.setup(file), self.format
+      end
+    end
+  end
+end

data/lib/scout/knowledge_base/list.rb

@@ -0,0 +1,95 @@
+require 'scout/annotation'
+class KnowledgeBase
+
+  def list_file(id, entity_type = nil)
+    id = Path.sanitize_filename(id)
+
+    entity_type = entity_type.to_s.split(":").last
+
+    raise "Ilegal list id: #{ id }" unless Misc.path_relative_to(dir, File.join(dir, id))
+
+    if entity_type
+      if entity_type.to_s == "simple"
+        path = dir.lists[entity_type.to_s][id]
+      else
+        path = dir.lists[entity_type.to_s][id + ".tsv"]
+      end
+    else
+      path = dir.lists.glob("*/#{id}").first
+      path ||= dir.lists.glob("*/#{id}.tsv").first
+      raise "List not found #{id}" if path.nil?
+    end
+
+    path.find
+  end
+
+  def save_list(id, list)
+    if AnnotatedArray === list
+      path = list_file(id, list.base_entity)
+    else
+      path = list_file(id, :simple)
+    end
+
+    Open.lock path do
+      begin
+        if AnnotatedArray === list
+          Open.write(path, Annotation.tsv(list, :all).to_s)
+        else
+          Open.write(path, list * "\n")
+        end
+      rescue
+        FileUtils.rm(path) if File.exist?(path)
+        raise $!
+      end
+    end
+  end
+
+  def load_list(id, entity_type = nil)
+    if entity_type
+      path = list_file(id, entity_type) 
+      path = list_file(id) unless path.exists?
+    else
+      path = list_file(id)
+    end
+
+    raise "List not found: #{ id }" unless path and path.exists?
+
+    begin
+      if path.get_extension == 'tsv'
+        list = Annotation.load_tsv path.tsv
+        list.extend AnnotatedArray
+        list
+      else
+        path.list
+      end
+    rescue
+      Log.exception $!
+      nil
+    end
+  end
+
+  def lists
+    lists = {}
+    dir.lists.glob("*").each do |list_dir|
+      lists[list_dir.basename] = list_dir.glob("*").
+        collect(&:unset_extension).
+        collect(&:basename)
+    end
+    lists
+  end
+
+  def delete_list(id, entity_type = nil)
+    path = list_file(id, entity_type)
+    path = list_file(id) unless path.exists?
+
+    "This list does not belong to #{ user }: #{[entity_type, id] * ": "}" unless File.exist? path
+
+    Open.lock path do
+      begin
+        FileUtils.rm path if File.exist? path
+      rescue
+        raise $!
+      end
+    end
+  end
+end

data/lib/scout/knowledge_base/query.rb

@@ -0,0 +1,96 @@
+require_relative 'entity'
+
+class KnowledgeBase
+
+  def setup(name, matches, reverse = false)
+    AssociationItem.setup matches, self, name, reverse
+  end
+
+  def _subset(name, source = :all, target = :all, options = {})
+    repo = get_index name, options
+
+    repo.subset(source, target)
+  end
+
+  def subset(name, entities, options = {}, &block)
+    entities, options = options, entities if entities.nil? and Hash === options
+
+    entities = case entities
+               when :all
+                 {:target => :all, :source => :all}
+               when AnnotatedArray
+                 format = entities.format if entities.respond_to? :format 
+                 format ||= entities.base_entity.to_s
+                 {format => entities.purge}
+               when Hash
+                 entities
+               else
+                 raise "Entities are not a Hash or an AnnotatedArray: #{Log.fingerprint entities}"
+               end
+
+    identify, identify_source, identify_target = entities.merge(options || {}).values_at :identify, :identify_source, :identify_target
+
+    source, target = select_entities(name, entities, options)
+    
+    source = identify_source(name, source) if identify_source
+    target = identify_target(name, target) if identify_target
+
+    source = identify(name, source) if identify && !identify_source
+    target = identify(name, target) if identify && !identify_target
+
+    return [] if source.nil? or target.nil?
+    return [] if Array === target and target.empty?
+    return [] if Array === source and source.empty?
+
+    matches = _subset name, source, target, options
+
+    setup(name, matches)
+
+    matches = matches.select(&block) if block_given? 
+
+    matches
+  end
+
+  def all(name, options={})
+    repo = get_index name, options
+    setup name, repo.keys
+  end
+
+  def _children(name, entity)
+    repo = get_index name
+    repo.match(entity)
+  end
+
+  def children(name, entity)
+    entity = identify_source(name, entity)
+    setup(name, _children(name, entity))
+  end
+
+  def _parents(name, entity)
+    repo = get_index name
+    repo.reverse.match(entity)
+  end
+
+  def parents(name, entity)
+    entity = identify_target(name, entity)
+    matches = _parents(name, entity)
+    #matches.each{|m| m.replace(m.partition("~").reverse*"") } unless undirected(name)
+    setup(name, matches, true)
+  end
+
+  def _neighbours(name, entity)
+    if undirected(name) and source(name) == target(name)
+      {:children => _children(name, entity)}
+    else
+      {:parents => _parents(name, entity), :children => _children(name, entity)}
+    end
+  end
+
+  def neighbours(name, entity)
+    hash = _neighbours(name, entity)
+    IndiferentHash.setup(hash)
+    setup(name, hash[:children]) if hash[:children] 
+    setup(name, hash[:parents], true) if hash[:parents]
+    hash
+  end
+end