scout-gear 10.7.1 → 10.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e72b6d2bef9af0067bf2601e059aa992ffdab98fba772dba1e297f958edb02d
-  data.tar.gz: 2c2cc3591c1645ccf34dbb250f1b2acda1b69040c0b051806c02c97eb07cc253
+  metadata.gz: 469b6e128a39e5612de5698005de8f4859c015bf3d65a5490a58f2f9f0312a39
+  data.tar.gz: 0a76edd86052c35af31f6de227c0bc43c71bb6b64fdc757e1179bf88a4ce8701
 SHA512:
-  metadata.gz: d1be99dd4127f54c5cf16833197e527a3cfa41014c7db4306ec291e8f01b78f8340cad7ee6e342dee2ed25221f2b02a6f6b665f46e81365ad5a12eea087b300d
-  data.tar.gz: cc3a9b28e47520f6756cba18d6037d7e83b7cd4d0cded1597a7cafc2cc167234090c1014d2b38adcc0a6a69c8d3ba1a72255538bd924e14f55b95769f3eca977
+  metadata.gz: c8753c031fda58c461afdefa3412a0d7996c6d5139a30653b1e6fcfaeba7fdf31b4d300bb2cb07060e7eb12d63891fc7a157cec3964c59d5cc68584ffc8823c8
+  data.tar.gz: 05cbb772edf0384f2889ad1be2cc597425d071da8c488364a8c72b23e6b7098aea0089247c180da09165f3eb7d899de494dca9ce2a00f285dcfaa564c8dc3e68

data/.vimproject

@@ -38,38 +38,11 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
      orchestrator.rb
     }
    }
-   semaphore.rb
    work_queue.rb
    work_queue=work_queue{
     socket.rb
     worker.rb
    }
-
-   persist=persist{
-    engine.rb
-    engine=engine{
-     tokyocabinet.rb
-     fix_width_table.rb
-     tkrzw.rb
-     packed_index.rb
-     sharder.rb
-    }
-    tsv.rb
-    tsv=tsv{
-     adapter.rb
-     serialize.rb
-     adapter=adapter{
-      base.rb
-
-      fix_width_table.rb
-      packed_index.rb
-      tkrzw.rb
-      tokyocabinet.rb
-      sharder.rb
-     }
-    }
-   }
-
    tsv.rb
    tsv=tsv{
     util.rb
@@ -101,26 +74,60 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
     open.rb
     csv.rb
    }
+   persist=persist{
+    engine.rb
+    engine=engine{
+     tokyocabinet.rb
+     fix_width_table.rb
+     tkrzw.rb
+     packed_index.rb
+     sharder.rb
+    }
+    tsv.rb
+    tsv=tsv{
+     adapter.rb
+     serialize.rb
+     adapter=adapter{
+      base.rb
+
+      fix_width_table.rb
+      packed_index.rb
+      tkrzw.rb
+      tokyocabinet.rb
+      sharder.rb
+     }
+    }
+   }
    entity.rb
    entity=entity{
     property.rb
     object.rb
     format.rb
     identifiers.rb
+    named_array.rb
    }
    association.rb
    association=association{
-    fields.rb
     index.rb
+    fields.rb
     item.rb
    }
-
+   knowledge_base.rb
+   knowledge_base=knowledge_base{
+    registry.rb
+    entity.rb
+    query.rb
+    traverse.rb
+    enrichment.rb
+    list.rb
+   }
    offsite.rb
    offsite=offsite{
     ssh.rb
     sync.rb
     step.rb
    }
+   semaphore.rb
   }
   scout-gear.rb
   workflow-scout.rb
@@ -137,6 +144,15 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   update
   template
   offsite
+  kb=kb{
+   config
+   entities
+   register
+   show
+   query
+   traverse
+   list
+  }
   workflow=workflow{
    task
    list
@@ -158,8 +174,12 @@ scout-gear=/$PWD filter="*.rb *.yaml" {
   test_helper.rb
   test_scout-gear.rb
   test_scout.rb
-  data=data{
+  data=data filter="*"{
    person=person{
+    brothers
+    identifiers
+    marriages
+    parents
    }
   }
   scout=scout{

data/VERSION

@@ -1 +1 @@
-10.7.1
+10.7.3

data/lib/scout/association/index.rb

@@ -2,6 +2,10 @@ require 'scout/annotation'
 module Association
 
   def self.index(file, source: nil, target: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+    IndiferentHash.setup(kwargs)
+    source = kwargs.delete :source if kwargs.include?(:source)
+    target = kwargs.delete :target if kwargs.include?(:target)
+
     persist_options = IndiferentHash.pull_keys kwargs, :persist
     index_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
       prefix: "Association::Index", 
@@ -157,7 +161,7 @@ module Association
                        new.read
                      end
 
-                     new.unnamed = true
+                     new.unnamed = self.unnamed
 
                      new.undirected = undirected
 

data/lib/scout/association/item.rb

@@ -102,7 +102,7 @@ module AssociationItem
   property :info => :array2single do
     fields = self.info_fields
 
-    return [{}] * self.length if fields.nil? or fields.empty?
+    next [{}] * self.length if fields.nil? or fields.empty?
 
     value = self.value
     value.collect{|v|

data/lib/scout/association.rb

@@ -6,17 +6,37 @@ require_relative 'association/item'
 
 module Association
   def self.open(obj, source: nil, target: nil, fields: nil, source_format: nil, target_format: nil, format: nil, **kwargs)
+    IndiferentHash.setup(kwargs)
+    source = kwargs.delete :source if kwargs.include?(:source)
+    target = kwargs.delete :target if kwargs.include?(:target)
+
+    if Path.is_filename?(obj)
+      options = TSV.parse_options(obj).merge(kwargs)
+    else
+      options = kwargs.dup
+    end
+
+    if String === obj && options[:namespace] && obj.include?("NAMESPACE")
+      new_obj = obj.gsub(/\[?NAMESPACE\]?/, options[:namespace])
+      obj.annotate(new_obj)
+      obj = new_obj
+    end
+
     all_fields = TSV.all_fields(obj)
-    source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, kwargs.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
+    source_pos, field_pos, source_header, field_headers, source_format, target_format = headers(all_fields, fields, options.merge(source: source, target: target, source_format: source_format, target_format: target_format, format: format))
 
     original_source_header = all_fields[source_pos]
     original_field_headers = all_fields.values_at(*field_pos)
     original_target_header = all_fields[field_pos.first]
 
-    type, identifiers = IndiferentHash.process_options kwargs, :type, :identifiers
+    type, identifiers = IndiferentHash.process_options options, :type, :identifiers
 
-    if source_format
+    if source_format || target_format
       translation_files = [TSV.identifier_files(obj), Entity.identifier_files(source_format), identifiers].flatten.compact
+      translation_files.collect!{|f| (Path.is_filename?(f, false) && options[:namespace]) ? Path.setup(f.gsub(/\[?NAMESPACE\]?/, options[:namespace])) : f }
+    end
+
+    if source_format
       source_index = begin
                        TSV.translation_index(translation_files, source_header, source_format)
                      rescue
@@ -25,7 +45,6 @@ module Association
     end
 
     if target_format
-      translation_files = [TSV.identifier_files(obj), Entity.identifier_files(target_format), identifiers].flatten.compact
       target_index = begin
                        TSV.translation_index(translation_files, field_headers.first, target_format)
                      rescue
@@ -66,11 +85,11 @@ module Association
 
     if source_index.nil? && target_index.nil?
       if TSV === obj
-        IndiferentHash.pull_keys kwargs, :persist
-        type = kwargs[:type] || obj.type
-        res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **kwargs.merge(type: type, merge: true)
+        IndiferentHash.pull_keys options, :persist
+        type = options[:type] || obj.type
+        res = obj.reorder original_source_header, all_fields.values_at(*field_pos), **options.merge(type: type, merge: true)
       else
-        res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **kwargs.merge(type: type))
+        res = TSV.open(obj, key_field: original_source_header, fields: all_fields.values_at(*field_pos), **options.merge(type: type))
       end
       res.key_field = final_key_field
       res.fields = final_fields
@@ -93,8 +112,24 @@ module Association
     transformer
   end
 
-  def self.database(*args, **kwargs)
-    tsv = open(*args, **kwargs)
-    TSV::Transformer === tsv ? tsv.tsv(merge: true) : tsv
+  def self.database(file, *args, **kwargs)
+    persist_options = IndiferentHash.pull_keys kwargs, :persist
+
+    database_persist_options = IndiferentHash.add_defaults persist_options.dup, persist: true, 
+      prefix: "Association::Index", serializer: :list,
+      other_options: kwargs  
+
+    Persist.tsv(file, kwargs, engine: "BDB", persist_options: database_persist_options) do |data|
+      tsv = open(file, *args, **kwargs)
+      if TSV::Transformer === tsv 
+        tsv.tsv(merge: true, data: data) 
+      elsif data.respond_to?(:persistence_path)
+        data.merge!(tsv)
+        tsv.annotate(data)
+        data
+      else
+        tsv
+      end
+    end
   end
 end

data/lib/scout/entity/format.rb

@@ -17,16 +17,20 @@ module Entity
     def find(value)
       @find_cache ||= {}
 
-      @find_cache[value] ||= begin
+      if @find_cache.include?(value)
+        @find_cache[value]
+      else
+        @find_cache[value] = begin
                                if orig_include? value
-                                 @find_cache[value] = value
+                                 value
                                else
+                                 value = value.to_s
                                  found = nil
                                  each do |k,v|
-                                   if value.to_s == k.to_s
+                                   if value == k.to_s
                                      found = k
                                      break
-                                   elsif value.to_s =~ /\(#{Regexp.quote k}\)/
+                                   elsif value =~ /\(#{Regexp.quote k.to_s}\)/
                                      found = k
                                      break
                                    end
@@ -34,6 +38,7 @@ module Entity
                                  found
                                end
                              end
+      end
     end
 
     def [](value)

data/lib/scout/entity/identifiers.rb

@@ -1,6 +1,6 @@
 module Entity
   def self.identifier_files(field)
-    entity_type = Entity.formats[field]
+    entity_type = Entity.formats[Entity.formats.find(field)]
     return [] unless entity_type and entity_type.include? Entity::Identified 
     entity_type.identifier_files
   end
@@ -48,7 +48,7 @@ module Entity
     def identifier_files 
       files = identity_type.identifier_files.dup
       return [] if files.nil?
-      files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotations.include? :namespace and self.namespace
+      files.collect!{|f| f.annotate f.gsub(/\b#{NAMESPACE_TAG}\b/, namespace.to_s) } if annotation_hash.include? :namespace and self.namespace
       if files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ }.any?
         Log.warn "Rejecting some identifier files for lack of 'namespace': " << files.select{|f| f =~ /\b#{NAMESPACE_TAG}\b/ } * ", "
       end
@@ -91,6 +91,8 @@ module Entity
       end
     end
 
+    name = default if name.nil?
+
     self.send(:include, Entity::Identified) unless Entity::Identified === self
 
     self.format = all_fields
@@ -106,6 +108,4 @@ module Entity
     @identifier_files << file
     @identifier_files.uniq!
   end
-
-
 end

data/lib/scout/entity/named_array.rb

@@ -0,0 +1,13 @@
+require 'scout/named_array'
+require 'scout/entity'
+
+module NamedArray
+
+  def [](key)
+    pos = NamedArray.identify_name(@fields, key)
+    return nil if pos.nil?
+    v = super(pos)
+    field = @fields && Integer === key && ! @fields.include?(key) ? @fields[key] : key
+    Entity.prepare_entity(v, field)
+  end
+end

data/lib/scout/entity/property.rb

@@ -1,3 +1,4 @@
+require 'scout/exceptions'
 module Entity
   class << self
     attr_accessor :entity_property_cache
@@ -20,7 +21,8 @@ module Entity
         Persist.annotation_repo_persist(repo, [name, obj.id] * ":", &block)
       else
 
-        Persist.persist([name, obj.id] * ":", type, options.dup, &block)
+        _id = obj.nil? ? 'nil' : obj.id
+        Persist.persist([name, _id] * ":", type, options.dup, &block)
       end
     end
 

data/lib/scout/entity.rb

@@ -3,6 +3,7 @@ require_relative 'entity/format'
 require_relative 'entity/property'
 require_relative 'entity/object'
 require_relative 'entity/identifiers'
+require_relative 'entity/named_array'
 
 module Entity
   def self.extended(base)
@@ -12,17 +13,19 @@ module Entity
     base.instance_variable_set(:@persisted_methods, {})
     base.include Entity::Object
     base.include AnnotatedArray
+    base.format = base.to_s
     base
   end
 
   def self.prepare_entity(entity, field, options = {})
     return entity unless defined? Entity
-    return entity unless String === entity or Array === entity
-    options ||= {}
-
-    dup_array = options.delete :dup_array
+    return entity unless String === entity or Array === entity or Numeric === entity
 
     if Entity === field or (Entity.respond_to?(:formats) and (_format = Entity.formats.find(field)))
+      options ||= {}
+
+      dup_array = options.delete :dup_array
+
       params = options.dup
 
       params[:format] ||= params.delete "format"

data/lib/scout/knowledge_base/enrichment.rb

@@ -0,0 +1,9 @@
+require 'rbbt/knowledge_base/registry'
+class KnowledgeBase
+  def enrichment(name, entities, options = {})
+    require 'rbbt/statistics/hypergeometric'
+    database = get_database(name, options)
+    entities = identify_source name, entities
+    database.enrichment entities, database.fields.first, :persist => false
+  end
+end

data/lib/scout/knowledge_base/entity.rb

@@ -0,0 +1,143 @@
+require_relative '../entity'
+
+class KnowledgeBase
+
+  def select_entities(name, entities, options = {})
+    index = get_index(name, options)
+
+    source_field = index.source_field
+    target_field = index.target_field
+
+    source_type = source_type name
+    target_type = target_type name
+
+    source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] || entities[:both]
+    target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s] || entities[:both]
+
+    [source_entities, target_entities]
+  end
+
+  def entity_options_for(type, database_name = nil)
+    entity_options = self.entity_options
+    IndiferentHash.setup entity_options if entity_options and not IndiferentHash === entity_options
+    options = entity_options[type.to_s] || entity_options[Entity.formats[type].to_s] || {}
+    options[:format] = @format[type] if Hash === @format && @format.include?(type)
+    namespace = self.namespace
+    namespace = db_namespace(database_name) if namespace.nil? and database_name
+    if database_name  
+      database = get_database(database_name)
+      if database.entity_options and (database.entity_options[type] or database.entity_options[Entity.formats[type.to_s].to_s])
+        options = options.merge(database.entity_options[type] || database.entity_options[Entity.formats[type.to_s].to_s])
+      end
+    end
+    options
+  end
+
+  def annotate(entities, type, database = nil)
+    format = @format[type] || type
+    entity_options = entity_options_for(type, database)
+    Entity.prepare_entity(entities, format, entity_options)
+  end
+
+  def translate(entities, type)
+    if format = @format[type] and (entities.respond_to? :format and format != entities.format)
+      entities.to format
+    else
+      entities
+    end
+  end
+
+  def source_type(name)
+    Entity.formats[Entity.formats.find(source(name))]
+  end
+
+  def target_type(name)
+    Entity.formats[Entity.formats.find(target(name))]
+  end
+
+  def entities
+    all_databases.inject([]){|acc,name| acc << source(name); acc << target(name)}.uniq
+  end
+
+  def entity_types
+    entities.collect{|entity| Entity.formats[entity] }.uniq
+  end
+
+  def identifier_files(name)
+    get_database(name).identifier_files.dup + self.identifier_files
+  end
+
+  def db_namespace(name)
+    get_database(name).namespace
+  end
+
+  def source_index(name)
+    Persist.memory("Source index #{name}: KB directory #{dir}") do
+      identifier_files = identifier_files(name)
+      identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if not namespace and db_namespace(name)
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, nil, source(name), :persist => true
+    end
+  end
+  
+  def target_index(name)
+    Persist.memory("Target index #{name}: KB directory #{dir}") do
+      identifier_files = identifier_files(name)
+      identifier_files.concat Entity.identifier_files(target(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if self.namespace
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, db_namespace(name)))} if namespace.nil? and db_namespace(name)
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, nil, target(name), :persist => true
+    end
+  end
+
+  def identify_source(name, entity)
+    return :all if entity == :all
+    index = begin source_index(name) rescue nil end
+    return entity if index.nil?
+    if Array === entity
+      entity.collect{|e| index[e] || e }
+    else
+      index[entity] || entity
+    end
+  end
+  
+  def identify_target(name, entity)
+    return :all if entity == :all
+    index = begin target_index(name) rescue nil end
+    return entity if index.nil?
+    if Array === entity
+      entity.collect{|e| index[e] || e }
+    else
+      index[entity] || entity
+    end
+  end
+
+  def identify(name, entity)
+    identify_source(name, entity) || identify_target(name, entity)
+  end
+
+  def define_entity_modules
+    entity_options.each do |entity,options|
+      next unless options[:identifiers]
+      identifiers = options[:identifiers]
+      identifiers = identifiers.split(",") unless Array === identifiers
+      m = begin
+            Object.const_get entity
+          rescue
+            m = Module.new
+            m.extend Entity
+            m.include Entity::Identified
+            Object.const_set entity, m
+          end
+
+      identifiers.each do |file|
+        m.add_identifiers Path.setup(file), self.format
+      end
+    end
+  end
+end

data/lib/scout/knowledge_base/list.rb

@@ -0,0 +1,95 @@
+require 'scout/annotation'
+class KnowledgeBase
+
+  def list_file(id, entity_type = nil)
+    id = Path.sanitize_filename(id)
+
+    entity_type = entity_type.to_s.split(":").last
+
+    raise "Ilegal list id: #{ id }" unless Misc.path_relative_to(dir, File.join(dir, id))
+
+    if entity_type
+      if entity_type.to_s == "simple"
+        path = dir.lists[entity_type.to_s][id]
+      else
+        path = dir.lists[entity_type.to_s][id + ".tsv"]
+      end
+    else
+      path = dir.lists.glob("*/#{id}").first
+      path ||= dir.lists.glob("*/#{id}.tsv").first
+      raise "List not found #{id}" if path.nil?
+    end
+
+    path.find
+  end
+
+  def save_list(id, list)
+    if AnnotatedArray === list
+      path = list_file(id, list.base_entity)
+    else
+      path = list_file(id, :simple)
+    end
+
+    Open.lock path do
+      begin
+        if AnnotatedArray === list
+          Open.write(path, Annotation.tsv(list, :all).to_s)
+        else
+          Open.write(path, list * "\n")
+        end
+      rescue
+        FileUtils.rm(path) if File.exist?(path)
+        raise $!
+      end
+    end
+  end
+
+  def load_list(id, entity_type = nil)
+    if entity_type
+      path = list_file(id, entity_type) 
+      path = list_file(id) unless path.exists?
+    else
+      path = list_file(id)
+    end
+
+    raise "List not found: #{ id }" unless path and path.exists?
+
+    begin
+      if path.get_extension == 'tsv'
+        list = Annotation.load_tsv path.tsv
+        list.extend AnnotatedArray
+        list
+      else
+        path.list
+      end
+    rescue
+      Log.exception $!
+      nil
+    end
+  end
+
+  def lists
+    lists = {}
+    dir.lists.glob("*").each do |list_dir|
+      lists[list_dir.basename] = list_dir.glob("*").
+        collect(&:unset_extension).
+        collect(&:basename)
+    end
+    lists
+  end
+
+  def delete_list(id, entity_type = nil)
+    path = list_file(id, entity_type)
+    path = list_file(id) unless path.exists?
+
+    "This list does not belong to #{ user }: #{[entity_type, id] * ": "}" unless File.exist? path
+
+    Open.lock path do
+      begin
+        FileUtils.rm path if File.exist? path
+      rescue
+        raise $!
+      end
+    end
+  end
+end