scout-gear 10.7.2 → 10.7.3

data/lib/scout/knowledge_base/registry.rb

@@ -0,0 +1,173 @@
+require 'scout/association'
+require 'scout/association/item'
+
+class KnowledgeBase
+  def register(name, file = nil, options = {}, &block)
+    file = file.find if Path === file
+    @registry ||= IndiferentHash.setup({})
+    if block_given?
+      block.define_singleton_method(:filename) do name.to_s end
+      Log.debug("Registering #{ name } from code block")
+      @registry[name] = [block, options]
+    else
+      Log.debug("Registering #{ name }: #{ Log.fingerprint file } #{Log.fingerprint options}")
+      @registry[name] = [file, options]
+    end
+  end
+
+  def all_databases
+    return [] unless @registry
+    @registry.keys 
+  end
+
+  def include?(name)
+    all_databases.include? name
+  end
+
+  def fields(name)
+    @fields ||= {}
+    @fields[name] ||= get_index(name).fields
+  end
+
+  def description(name)
+    @descriptions ||= {}
+    @descriptions[name] ||= get_index(name).key_field.split("~")
+  end
+
+  def source(name)
+    description(name)[0]
+  end
+
+  def target(name)
+    description(name)[1]
+  end
+
+  def undirected(name)
+    description(name)[2]
+  end
+
+  def get_index(name, options = {})
+    name = name.to_s
+    options[:namespace] ||= self.namespace unless self.namespace.nil?
+    @indices ||= IndiferentHash.setup({})
+    @indices[[name, options]] ||= 
+      begin 
+        if options.empty?
+          key = name.to_s
+        elsif options[:key]
+          key = options[:key]
+          key = name if key == :name
+        else
+          fp = Misc.digest(options)
+          key = name.to_s + "_" + fp
+        end
+
+        Persist.memory("Index:" << [key, dir] * "@") do
+          options = options.dup
+
+          persist_dir = dir
+          persist_path = persist_dir[key].find
+          file, registered_options = registry[name]
+
+          options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist_dir => persist_dir, :persist => true
+
+          if entity_options
+            options[:entity_options] ||= {}
+            entity_options.each do |type, info|
+              options[:entity_options][type] ||= {}
+              options[:entity_options][type] = IndiferentHash.add_defaults options[:entity_options][type], info
+            end
+          end
+
+          persist_options = IndiferentHash.pull_keys options, :persist
+          persist_options = IndiferentHash.add_defaults persist_options
+
+          index = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
+                    Log.low "Re-opening index #{ name } from #{ Log.fingerprint persist_path }. #{options}"
+                    Association.index(file, **options, persist_options: persist_options.dup)
+                  else
+                    options = IndiferentHash.add_defaults options, registered_options if registered_options
+                    raise "Repo #{ name } not found and not registered" if file.nil?
+                    Log.medium "Opening index #{ name } from #{ Log.fingerprint file }. #{options}"
+                    file = file.call if Proc === file
+                    Association.index(file, **options, persist_options: persist_options.dup)
+                  end
+
+          index.namespace = self.namespace unless self.namespace
+
+          index
+        end
+      end
+  end
+
+  def get_database(name, options = {})
+    options = options.dup
+    if self.namespace == options[:namespace]
+      options.delete(:namespace) 
+    end
+    @databases ||= IndiferentHash.setup({})
+    @databases[[name, options]] ||= 
+      begin 
+        fp = Log.fingerprint([name,options])
+
+        if options.empty?
+          key = name.to_s
+        else
+          fp = Misc.digest(options)
+          key = name.to_s + "_" + fp
+        end
+
+        options[:namespace] ||= self.namespace unless self.namespace.nil?
+
+        key += '.database'
+        Persist.memory("Database:" << [key, dir] * "@") do
+          options = options.dup
+
+          persist_dir = dir
+          persist_path = persist_dir[key].find
+          file, registered_options = registry[name]
+
+          options = IndiferentHash.add_defaults options, registered_options if registered_options and registered_options.any?
+          options = IndiferentHash.add_defaults options, :persist_path => persist_path, :persist => true
+
+          if entity_options
+            options[:entity_options] ||= {}
+            entity_options.each do |type, info|
+              options[:entity_options][type] ||= {}
+              options[:entity_options][type] = IndiferentHash.add_defaults options[:entity_options][type], info
+            end
+          end
+
+          persist_options = IndiferentHash.pull_keys options, :persist
+
+          database = if persist_path.exists? and persist_options[:persist] and not persist_options[:update]
+                       Log.low "Re-opening database #{ name } from #{ Log.fingerprint persist_path }. #{options}"
+                       #Association.database(file, **options, persist_options: persist_options)
+                       Association.database(file, **options.merge(persist_options: persist_options))
+                     else
+                       options = IndiferentHash.add_defaults options, registered_options if registered_options
+                       undirected = IndiferentHash.process_options options, :undirected
+                       raise "Repo #{ name } not found and not registered" if file.nil?
+                       Log.medium "Opening database #{ name } from #{ Log.fingerprint file }. #{options}"
+                       file = file.call if Proc === file
+                       #Association.database(file, **options, persist_options: persist_options)
+                       Association.database(file, **options.merge(persist_options: persist_options))
+                     end
+
+          database.namespace = self.namespace if self.namespace
+
+          database
+        end
+      end
+  end
+
+  def index_fields(name)
+    get_index(name).fields
+  end
+
+  def produce(name, *rest,&block)
+    register(name, *rest, &block)
+    get_index(name)
+  end
+end

data/lib/scout/knowledge_base/traverse.rb

@@ -0,0 +1,329 @@
+class KnowledgeBase
+
+  class Traverser
+    attr_accessor :rules, :assignments, :matches, :kb
+
+    def initialize(kb, rules = [])
+      @kb = kb
+      @rules = rules
+      @assignments = {}
+      @matches = {}
+    end
+
+    def wildcard(name)
+      return name unless is_wildcard?(name)
+      assignments[name] || name
+    end
+
+    def is_wildcard?(name)
+      name[0] == '?'
+    end
+
+    def is_list?(name)
+      name[0] == ':'
+    end
+
+    def identify(db, source, target)
+      source_entities = if is_wildcard? source
+                          assignments[source] || :all
+                        elsif is_list? source
+                          kb.load_list(source[1..-1])
+                        else
+                          kb.identify_source db, source
+                        end
+
+      target_entities = if is_wildcard? target
+                          assignments[target] || :all
+                        elsif is_list? target
+                          kb.load_list(target[1..-1])
+                        else
+                          kb.identify_target db, target
+                        end
+
+      source_entities = [source_entities] unless Array === source_entities or source_entities == :all
+      target_entities = [target_entities] unless Array === target_entities or target_entities == :all
+
+      [source_entities, target_entities]
+    end
+    
+    def reassign(matches, source, target)
+      #assignments[source] = (matches.any? ? matches.collect{|m| m.source_entity }.uniq : nil) if is_wildcard? source
+      #assignments[target] = (matches.any? ? matches.collect{|m| m.target_entity }.uniq : nil) if is_wildcard? target
+      assignments[source] = (matches.any? ? matches.source_entity.uniq : nil) if is_wildcard? source
+      assignments[target] = (matches.any? ? matches.target_entity.uniq : nil) if is_wildcard? target
+    end
+
+    def clean_matches(rules, all_matches, assignments)
+      paths = {}
+
+      rules.zip(all_matches).each do |rule, matches|
+        source, db, target = rule.split /\s+/
+        next if matches.nil?
+
+        if is_wildcard? source
+          assigned = assignments[source] || []
+          matches = matches.select{|m| assigned.include? m.partition("~").first }
+        end
+        
+        if is_wildcard? target
+          assigned = assignments[target] || []
+          matches = matches.select{|m| assigned.include? m.partition("~").last }
+        end
+
+        paths[rule] = matches
+      end
+
+      paths
+    end
+
+    def _fp(rules, clean_matches, assignments)
+      return true if rules.empty?
+
+      rule, *rest = rules
+      source, db, target = rule.split /\s+/
+
+      wildcard_source = is_wildcard? source
+      wildcard_target = is_wildcard? target
+
+      paths = {}
+      matches = clean_matches[rule]
+      matches.each do |match|
+        new_assignments = nil
+        match_source, _sep, match_target = match.partition "~"
+
+        if wildcard_source
+          next if assignments[source] and assignments[source]  != match_source
+          new_assignments ||= assignments.dup
+          new_assignments[source] = match_source
+        end
+
+        if wildcard_target
+          next if assignments[target] and assignments[target]  != match_target
+          new_assignments ||= assignments.dup
+          new_assignments[target] = match_target
+        end
+
+        new_paths = _fp(rest, clean_matches, new_assignments)
+        next unless new_paths
+        paths[match] = new_paths
+      end
+
+      return false if paths.empty?
+
+      paths 
+    end
+
+    def _ep(paths)
+      found = []
+      paths.each do |match,_next|
+        case _next
+        when TrueClass
+          found << [match]
+        when FalseClass
+          next
+        else
+          _ep(_next).each do |_n|
+            found << [match] + _n
+          end
+        end
+      end
+      found
+    end
+
+    def find_paths(rules, all_matches, assignments)
+      clean_matches = clean_matches(rules, all_matches, assignments)
+
+      path_hash = _fp(rules, clean_matches, {})
+
+      return [] unless path_hash
+      _ep(path_hash).collect do |path|
+        path.zip(clean_matches.values_at(*rules)).collect do |item, matches|
+          matches.select{|m| m == item }.first
+        end
+      end
+    end
+
+    def traverse_db(db, source, target, conditions)
+      source_entities, target_entities = identify db, source, target
+
+      options = {:source => source_entities, :target => target_entities}
+      Log.debug "Traversing #{ db }: #{Log.fingerprint options}"
+      matches = kb.subset(db, options)
+
+      if conditions
+        Misc.tokenize(conditions).each do |condition|
+          if condition.index "="
+            key, value = condition.split("=")
+            matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
+          else
+            matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
+          end
+        end
+      end
+
+      matches
+    end
+
+    def id_dbs(db)
+      # ToDo: Revise this, I'm not sure what id does anymore
+      # I think it deals with syndication
+      if db.include? '?'
+        all_dbs = kb.registry.keys.collect{|k| k.to_s }
+        _name, _sep, _kb = db.partition("@")
+        case
+        when _name[0] == '?'
+          dbs = all_dbs.select{|_db| 
+            n,_s,d=_db.partition("@"); 
+            d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
+          }
+        when _kb[0] == '?'
+          dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
+        end
+      else
+        dbs = [db]
+      end
+
+      dbs
+    end
+
+    def traverse(nopaths = false)
+      all_matches = []
+      path_rules = []
+      acc_var = nil
+      pre_acc_var_assignments = nil
+      rules.each do |rule|
+        rule = rule.strip
+        next if rule.empty?
+
+        if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
+          Log.debug "Traverse rule: #{rule}"
+          path_rules << rule
+
+          source, db, target, conditions = m.captures
+
+          dbs = id_dbs(db)
+
+          rule_matches = nil
+          dbs.each do |_db|
+            matches = traverse_db(_db, source, target, conditions)
+
+            next if matches.nil? or matches.empty?
+
+            # ToDo: Revise this, I'm not sure what id does anymore
+            #
+            #if db.include? '?'
+            #  _name, _sep, _kb = db.partition("@")
+            #  case
+            #  when _kb[0] == '?'
+            #    assignments[_kb] ||= []
+            #    assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
+            #  when _name[0] == '?'
+            #    assignments[_name] ||= []
+            #    assignments[_name] << _db.partition("@").first
+            #  end
+            #end
+
+            if rule_matches.nil?
+              rule_matches = matches
+            else
+              matches.each do |m|
+                rule_matches << m
+              end
+            end
+
+            assignments.each{|k,v| v.uniq! if v}
+          end
+
+          reassign rule_matches, source, target if rule_matches
+
+          all_matches << rule_matches
+
+        elsif m = rule.match(/([^\s=]+)\s*=([^\s]*)\s*(.*)/)
+          Log.debug "Assign rule: #{rule}"
+          var, db, value_str = m.captures
+          names = value_str.split(",").collect{|v| v.strip}
+          if db.empty?
+            ids = names
+          else
+            dbs = id_dbs(db)
+            names = names.collect{|name| assignments.include?(name) ? assignments[name] : name}.flatten
+            ids = names.collect{|name| 
+              id = nil
+              dbs.each do |db|
+                sid, tid = identify db, name, name
+                id = (sid + tid).compact.first
+                break if id
+              end
+              id
+            }
+          end
+          assignments[var] = ids
+
+        elsif m = rule.match(/(\?[^\s{]+)\s*{/)
+          acc_var = m.captures.first
+          pre_acc_var_assignments = assignments.dup
+          Log.debug "Start assign block: #{acc_var}"
+        elsif m = rule.match(/^\s*}\s*$/)
+          Log.debug "Close assign block: #{acc_var}"
+          saved_assign = assignments[acc_var]
+          assignments.clear
+          assignments.merge!(pre_acc_var_assignments)
+          pre_acc_var_assignments = nil
+          assignments[acc_var] = saved_assign
+          all_matches = []
+          path_rules = []
+        else
+          raise "Rule not understood: #{rule}"
+        end
+      end
+
+      return [assignments, nil] if nopaths
+
+      Log.debug "Finding paths: #{all_matches.length}"
+      paths = find_paths path_rules, all_matches, assignments
+      Log.debug "Found paths: #{paths.length}"
+
+      [assignments, paths]
+    end
+
+    #def traverse
+    #  all_matches = []
+
+    #  rules.each do |rule|
+    #    rule = rule.strip
+    #    next if rule.empty?
+    #    source, db, target, conditions = rule.match(/([^\s]+)\s+([^\s]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/).captures
+
+    #    source_entities, target_entities = identify db, source, target
+
+    #    matches = kb.subset(db, :source => source_entities, :target => target_entities)
+
+    #    if conditions
+    #      conditions.split(/\s+/).each do |condition|
+    #        if condition.index "="
+    #          key, value = conditions.split("=")
+    #          matches = matches.select{|m| m.info[key.strip].to_s =~ /\b#{value.strip}\b/}
+    #        else
+    #          matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
+    #        end
+    #      end
+    #    end
+
+    #    reassign matches, source, target
+
+    #    all_matches << matches
+    #  end
+
+    #  paths = find_paths rules, all_matches, assignments
+
+    #  [assignments, paths]
+    #end
+
+  end
+
+  def traverse(rules, nopaths=false)
+    traverser = KnowledgeBase::Traverser.new self, rules
+    traverser.traverse nopaths
+  end
+    
+end

data/lib/scout/knowledge_base.rb

@@ -0,0 +1,91 @@
+require_relative 'association'
+require_relative 'association/item'
+require_relative 'knowledge_base/registry'
+require_relative 'knowledge_base/entity'
+require_relative 'knowledge_base/query'
+require_relative 'knowledge_base/traverse'
+require_relative 'knowledge_base/list'
+#require 'scout/knowledge_base/query'
+#require 'scout/knowledge_base/syndicate'
+
+class KnowledgeBase
+
+  attr_accessor :dir, :namespace, :registry, :entity_options, :format, :identifier_files
+
+  def initialize(dir, namespace = nil)
+    @dir = Path.setup(dir.dup)
+
+    @namespace = namespace
+
+    @identifier_files = []
+
+    @registry       ||= IndiferentHash.setup({})
+    @entity_options ||= IndiferentHash.setup({})
+
+    @format         ||= IndiferentHash.setup({})
+    @descriptions   ||= IndiferentHash.setup({})
+    @indices        ||= IndiferentHash.setup({})
+  end
+
+  def config_file(name)
+    @dir.config[name.to_s]
+  end
+
+  def save_variable(name)
+    file = config_file(name)
+    variable = "@#{name}".to_sym
+    Open.write(file, self.instance_variable_get(variable).to_yaml)
+  end
+
+  def load_variable(name)
+    file = config_file(name)
+    variable = "@#{name}".to_sym
+    self.instance_variable_set(variable, YAML.load(Open.read(file))) if file.exists?
+  end
+
+  def save
+    save_variable(:namespace)
+    save_variable(:registry)
+    save_variable(:entity_options)
+    save_variable(:identifier_files)
+  end
+
+  def load
+    load_variable(:namespace)
+    load_variable(:registry)
+    load_variable(:entity_options)
+    load_variable(:identifier_files)
+  end
+
+  def self.load(dir)
+    dir = Path.setup("var").knowledge_base[dir.to_s] if Symbol === dir
+    kb = KnowledgeBase.new dir
+    kb.load
+    kb
+  end
+
+  def info(name)
+
+    source = self.source(name)
+    target = self.target(name)
+    source_type = self.source_type(name)
+    target_type = self.target_type(name)
+    fields = self.fields(name)
+    source_entity_options = self.entity_options_for source_type, name
+    target_entity_options = self.entity_options_for target_type, name
+    undirected = self.undirected(name) == 'undirected'
+
+    info = {
+      :source => source,
+      :target => target,
+      :source_type => source_type,
+      :target_type => target_type,
+      :source_entity_options => source_entity_options,
+      :target_entity_options => target_entity_options,
+      :fields => fields,
+      :undirected => undirected,
+    }
+
+    info
+  end
+end

data/lib/scout/tsv/annotation.rb

@@ -48,12 +48,12 @@ module Annotation
 
     fields = fields.flatten.compact.uniq
 
-    annotations = if Annotation.is_annotated?(objs) 
-                    objs.annotations 
+    annotations = if Annotation.is_annotated?(objs)
+                    objs.annotation_hash.keys
                   elsif (Array === objs && objs.any?)
                     first = objs.compact.first
                     if Annotation.is_annotated?(first)
-                      objs.compact.first.annotations
+                      objs.compact.first.annotation_hash.keys
                     else
                       raise "Objects didn't have annotations"
                     end
@@ -158,7 +158,7 @@ module Annotation
         Annotation.load_tsv_values(id, values, tsv.fields)
       end
 
-      case tsv.key_field 
+      case tsv.key_field
       when "List"
         annotated_objects.first
       else

data/lib/scout/tsv/index.rb

@@ -66,8 +66,6 @@ module TSV
         index = TSV.setup({}, :type => :single)
       end
 
-      tsv_file = TSV.open(tsv_file, **data_options) if ! TSV === tsv_file
-
       log_msg = "Index #{Log.fingerprint tsv_file} target #{Log.fingerprint target}"
       Log.low log_msg
       bar = log_msg if TrueClass === bar

data/lib/scout/tsv/parser.rb

@@ -7,7 +7,7 @@ module TSV
       if Proc === cast
         cast.call value
       else
-        if value.nil? || value.empty?
+        if value.nil? || value == ""
           nil
         else
           value.send(cast)

data/lib/scout/tsv/stream.rb

@@ -1,5 +1,5 @@
 module TSV
-  def self.paste_streams(streams, type: nil, sort: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
+  def self.paste_streams(streams, type: nil, sort: nil, sort_cmd_args: nil, sort_memory: nil, sep: nil, preamble: nil, header: nil, same_fields: nil, fix_flat: nil, all_match: nil, one2one: true, field_prefix: nil)
     sep = "\t" if sep.nil?
 
     streams = streams.collect do |stream|
@@ -20,7 +20,7 @@ module TSV
     num_streams = streams.length
 
     streams = streams.collect do |stream|
-      Open.sort_stream(stream, memory: sort_memory)
+      Open.sort_stream(stream, memory: sort_memory, cmd_args: sort_cmd_args)
     end if sort
 
     begin
@@ -193,7 +193,7 @@ module TSV
       dumper.close
 
       streams.each do |stream|
-        stream.close if stream.respond_to?(:close) && ! stream.closed?
+        stream.close if stream.respond_to?(:close)
         stream.join if stream.respond_to?(:join)
       end
       end

data/lib/scout/tsv.rb

@@ -133,6 +133,8 @@ module TSV
 
       tsv.unnamed = unnamed unless unnamed.nil?
 
+      tsv.entity_options = entity_options
+
       tsv
     end
   end

data/lib/scout/workflow/step/info.rb

@@ -50,6 +50,15 @@ class Step
     @info
   end
 
+  def pid
+    info[:pid]
+  end
+
+  def pid=(pid)
+    set_info :pid, pid
+  end
+
+
   def merge_info(new_info)
     info = self.info
     new_info.each do |key,value|
@@ -121,7 +130,7 @@ class Step
     if message.nil?
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
     else
-      message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
+      message = Log.fingerprint(message.split("\n").first).sub(/^'/,'').sub(/'$/,'')
       Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
     end
   end