rbbt-util 5.14.33 → 5.14.34

data/lib/rbbt/knowledge_base/registry.rb

@@ -0,0 +1,106 @@
+require 'rbbt/association'
+require 'rbbt/association/item'
+
+class KnowledgeBase
+
+  def register(name, file = nil, options = {}, &block)
+    if block_given?
+      block.define_singleton_method(:filename) do name.to_s end
+      Log.debug("Registering #{ name } from code block")
+      @registry[name] = [block, options]
+    else
+      Log.debug("Registering #{ name }: #{ Misc.fingerprint file }")
+      @registry[name] = [file, options]
+    end
+  end
+
+  def all_databases
+    @registry.keys 
+  end
+
+  def description(name)
+    @descriptions[name] ||= get_index(name).key_field.split("~")
+  end
+
+  def source(name)
+    description(name)[0]
+  end
+
+  def target(name)
+    description(name)[1]
+  end
+
+  def undirected(name)
+    description(name)[2]
+  end
+
+  def get_index(name, options = {})
+    name = name.to_s
+    key = name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options]))
+    @indices[key] ||= 
+      begin 
+        Persist.memory("Index:" << [key, dir] * "@") do
+          options = options.dup
+          persist_dir = dir
+          persist_file = persist_dir[key]
+          file, registered_options = registry[name]
+
+          options = Misc.add_defaults options, :persist_file => persist_file, :persist_dir => persist_dir, :namespace => namespace, :format => format, :persist => true
+          options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
+
+          persist_options = Misc.pull_keys options, :persist
+
+          index = if persist_file.exists? and persist_options[:persist] and not persist_options[:update]
+                    Log.low "Re-opening index #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
+                    Association.index(nil, options, persist_options.dup)
+                  else
+                    options = Misc.add_defaults options, registered_options if registered_options
+                    raise "Repo #{ name } not found and not registered" if file.nil?
+                    Log.medium "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
+                    Association.index(file, options, persist_options.dup)
+                  end
+
+          index.namespace = self.namespace
+
+          index
+        end
+      end
+  end
+
+  def get_database(name, options = {})
+    name = name.to_s
+    key = "Index:" + name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options.dup]))
+    @indices[key] ||= 
+      begin 
+        Persist.memory("Database:" << [key, dir] * "@") do
+          options = options.dup
+          persist_file = dir.indices[key]
+          file, registered_options = registry[name]
+
+          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format, :persist => true
+          options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
+
+          persist_options = Misc.pull_keys options, :persist
+
+          database = if persist_file.exists?
+                    Log.low "Re-opening database #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
+                    Association.open(nil, options, persist_options)
+                  else
+                    options = Misc.add_defaults options, registered_options if registered_options
+                    raise "Repo #{ name } not found and not registered" if file.nil?
+                    Log.medium "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
+                    Association.open(file, options, persist_options)
+                  end
+
+          database.namespace = self.namespace
+
+          database
+        end
+      end
+  end
+
+  def index_fields(name)
+    get_index(name).fields
+  end
+
+end

data/lib/rbbt/knowledge_base/syndicate.rb

@@ -0,0 +1,22 @@
+class KnowledgeBase
+  def syndicate(name, kb)
+    kb.all_databases.each do |database|
+      db_name = [database, name] * "@"
+      file, kb_options = kb.registry[database]
+      options = {}
+      options[:entity_options] = kb_options[:entity_options]
+      options[:undirected] = true if kb_options and kb_options[:undirected]
+      if kb.entity_options
+        options[:entity_options] = kb.entity_options.merge(options[:entity_options] || {})
+      end
+
+      register(db_name, nil, options) do
+        kb.get_database(database)
+      end
+    end
+  end
+
+  def all_databases
+    @registry.keys 
+  end
+end

data/lib/rbbt/knowledge_base.rb

@@ -1,23 +1,11 @@
+
 require 'rbbt/association'
 require 'rbbt/association/item'
-require 'rbbt/entity'
+require 'rbbt/knowledge_base/entity'
+require 'rbbt/knowledge_base/query'
+require 'rbbt/knowledge_base/syndicate'
 
 class KnowledgeBase
-  class << self
-    attr_accessor :knowledge_base_dir, :registry
-
-    def registry
-      @registry ||= IndiferentHash.setup({})
-    end
-    
-    def knowledge_base_dir
-      @knowledge_base_dir ||= Rbbt.var.knowledge_base
-    end
-  end
-
-  def setup(name, matches, reverse = false)
-    AssociationItem.setup matches, self, name, reverse
-  end
 
   attr_accessor :namespace, :dir, :indices, :registry, :format, :databases, :entity_options
   def initialize(dir, namespace = nil)
@@ -36,348 +24,7 @@ class KnowledgeBase
     @databases = {}
   end
 
-  def version(new_namespace, force = false)
-    return self if new_namespace == namespace and not force
-    new_kb = KnowledgeBase.new dir[new_namespace], new_namespace
-    new_kb.format.merge! self.format
-    new_kb.entity_options.merge! self.entity_options
-    new_kb.registry = self.registry
-    new_kb
-  end
-
-  #{{{ Descriptions
- 
-  def register(name, file = nil, options = {}, &block)
-    if block_given?
-      block.define_singleton_method(:filename) do name.to_s end
-      Log.debug("Registering #{ name } from code block")
-      @registry[name] = [block, options]
-    else
-      Log.debug("Registering #{ name }: #{ Misc.fingerprint file }")
-      @registry[name] = [file, options]
-    end
-  end
-
-  def syndicate(name, kb)
-    kb.all_databases.each do |database|
-      db_name = [database, name] * "@"
-      file, kb_options = kb.registry[database]
-      options = {}
-      options[:entity_options] = kb_options[:entity_options]
-      options[:undirected] = true if kb_options and kb_options[:undirected]
-      if kb.entity_options
-        options[:entity_options] = kb.entity_options.merge(options[:entity_options] || {})
-      end
-
-      register(db_name, nil, options) do
-        kb.get_database(database)
-      end
-    end
-  end
-
-  def all_databases
-    @registry.keys 
-  end
-
-  def description(name)
-    @descriptions[name] ||= get_index(name).key_field.split("~")
-  end
-
-  def source(name)
-    description(name)[0]
-  end
-
-  def target(name)
-    description(name)[1]
-  end
-
-  def undirected(name)
-    description(name)[2]
-  end
-
-  def source_type(name)
-    Entity.formats[source(name)]
-  end
-
-  def target_type(name)
-    Entity.formats[target(name)]
-  end
-
-  def index_fields(name)
-    get_index(name).fields
-  end
-
-  def entities
-    all_databases.inject([]){|acc,name| acc << source(name); acc << target(name)}.uniq
-  end
-
-  def entity_types
-    entities.collect{|entity| Entity.formats[entity] }.uniq
-  end
-
-  #{{{ Open and get
- 
-  def open_options
-    {:namespace => namespace, :format => @format}
-  end
- 
-  def get_database(name, options = {})
-    key = name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options,format,namespace]))
-    @databases[key] ||= 
-      begin 
-        file, registered_options = registry[name]
-        database = Persist.memory("Database:" << [key, dir] * "@") do
-          persist_file = dir.databases[key]
-
-          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format
-          options = Misc.add_defaults options, registered_options if registered_options
-
-          persist_options = Misc.pull_keys options, :persist
-
-          database = if persist_file.exists?
-                    Log.low "Re-opening database #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
-                    Association.open(file, options, persist_options)
-                  else
-                    raise "Repo #{ name } not found and not registered" if file.nil?
-                    Log.low "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
-                    Association.open(file, options, persist_options)
-                  end
-
-          database.namespace = self.namespace
-
-          database
-        end
-
-        database.entity_options ||= {}
-        database.entity_options.merge! registered_options[:entity_options] if registered_options.include? :entity_options
-        database
-      end
-  end
-
-  def get_index(name, options = {})
-    name = name.to_s
-    key = name.to_s + "_" + Misc.digest(Misc.fingerprint([name,options]))
-    @indices[key] ||= 
-      begin 
-        Persist.memory("Index:" << [key, dir] * "@") do
-          persist_file = dir.indices[key]
-          file, registered_options = registry[name]
-
-          options = Misc.add_defaults options, :persist_file => persist_file, :namespace => namespace, :format => format
-          options = Misc.add_defaults options, registered_options if registered_options and registered_options.any?
-
-          persist_options = Misc.pull_keys options, :persist
-
-          index = if persist_file.exists?
-                    Log.low "Re-opening index #{ name } from #{ Misc.fingerprint persist_file }. #{options}"
-                    Association.index(nil, options, persist_options)
-                  else
-                    options = Misc.add_defaults options, registered_options if registered_options
-                    raise "Repo #{ name } not found and not registered" if file.nil?
-                    Log.low "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
-                    Association.index(file, options, persist_options)
-                  end
-
-          index.namespace = self.namespace
-
-          index
-        end
-      end
-  end
-
-
-  #{{{ Add manual database
-  
-  def add_index(name, source_type, target_type, *fields)
-    options = fields.pop if Hash === fields.last
-    options ||= {}
-    undirected = Misc.process_options options, :undirected 
-
-    undirected = nil unless undirected 
-
-    repo_file = dir[name].find
-    index = Association.index(nil, {:namespace => namespace, :key_field => [source_type, target_type, undirected].compact * "~", :fields => fields}.merge(options), :file => repo_file, :update => true)
-    @indices[name] = index
-  end
-
-  def add(name, source, target, *rest)
-    code = [source, target] * "~"
-    repo = @indices[name]
-    repo[code] = rest
-  end
-
-  def write(name)
-    repo = @indices[name]
-    repo.write_and_read do
-      yield
-    end
-  end
-
-  #{{{ Annotate
-  
-  def entity_options_for(type, database_name = nil)
-    options = entity_options[Entity.formats[type]] || {}
-    options[:format] = @format[type] if @format.include? :type
-    options = {:organism => namespace}.merge(options)
-    if database_name and 
-      (database = get_database(database_name)).entity_options and
-      (database = get_database(database_name)).entity_options[type]
-      options = options.merge database.entity_options[type] 
-    end
-    options
-  end
-
-  def annotate(entities, type, database = nil)
-    format = @format[type] || type
-    Misc.prepare_entity(entities, format, entity_options_for(type, database))
-  end
-
-  #{{{ Identify
-  
-
-  def database_identify_index(database, target)
-    if database.identifier_files.any?
-      id_file =  database.identifier_files.first
-      identifier_fields = TSV.parse_header(id_file).all_fields
-      if identifier_fields.include? target
-        TSV.index(database.identifiers, :target => target, :persist => true, :order => true)
-      else
-        {}
-      end
-    else
-      if TSV.parse_header(Organism.identifiers(namespace)).all_fields.include? target
-        Organism.identifiers(namespace).index(:target => target, :persist => true, :order => true)
-      else
-        {}
-      end
-    end
-  end
-
-  def identify_source(name, entity)
-    database = get_database(name, :persist => true)
-    return entity if Symbol === entity or (String === entity and database.include? entity)
-    source = source(name)
-    @identifiers[name] ||= {}
-    @identifiers[name]['source'] ||= begin
-                                       database_identify_index(database, source)
-                                     end
-
-    if Array === entity
-      @identifiers[name]['source'].chunked_values_at(entity).zip(entity).collect{|p|
-        p.compact.first
-      }
-    else
-      @identifiers[name]['source'][entity]
-    end
-  end
-
-  def identify_target(name, entity)
-    database = get_database(name, :persist => true)
-    return entity if Symbol === entity #or (String === entity) # and database.values.collect{|v| v.first}.compact.flatten.include?(entity))
-    target = target(name)
-
-    @identifiers[name] ||= {}
-    @identifiers[name]['target'] ||= begin
-                                       database_identify_index(database, target)
-                                     end
-    if Array === entity
-      @identifiers[name]['target'].chunked_values_at(entity).zip(entity).collect{|p|
-        p.compact.first
-      }
-    else
-      @identifiers[name]['target'][entity] 
-    end
-  end
-
-  def identify(name, entity)
-    identify_source(name, entity) || identify_target(name, entity)
-  end
-
-  def normalize(entity)
-    source_matches = all_databases.collect{|d|
-      identify_source(d, entity)
-    }.flatten.compact.uniq
-    return entity if source_matches.include? entity
-
-    target_matches = all_databases.collect{|d|
-      identify_target(d, entity)
-    }.flatten.compact.uniq
-    return entity if target_matches.include? entity
-
-    (source_matches + target_matches).first
-  end
-
-  #{{{ Query
-
-  def all(name, options={})
-    repo = get_index name, options
-    setup name, repo.keys
-  end
-
-  def children(name, entity)
-    repo = get_index name
-    setup(name, repo.match(entity))
-  end
-
-  def parents(name, entity)
-    repo = get_index name
-    setup(name, repo.reverse.match(entity), true)
-  end
-
-  def neighbours(name, entity)
-    if undirected(name) and source(name) == target(name)
-      IndiferentHash.setup({:children => children(name, entity)})
-    else
-      IndiferentHash.setup({:parents => parents(name, entity), :children => children(name, entity)})
-    end
-  end
-
-  def subset(name, entities, &block)
-    entities = case entities
-               when :all
-                 {:target => :all, :source => :all}
-               when AnnotatedArray
-                 format = entities.format if entities.respond_to? :format 
-                 format ||= entities.base_entity.to_s
-                 {format => entities.clean_annotations}
-               when Hash
-                 entities
-               else
-                 raise "Entities are not a Hash or an AnnotatedArray: #{Misc.fingerprint entities}"
-               end
-
-    repo = get_index name
-
-    matches = setup(name, repo.subset_entities(entities))
-
-    block_given? ? matches.select(&block) : matches
-  end
-
-  def translate(entities, type)
-    if format = @format[type] and (entities.respond_to? :format and format != entities.format)
-      entities.to format
-    else
-      entities
-    end
-  end
-
-  def pair_matches(source, target, undirected = nil)
-    all_databases.inject([]){|acc,database|
-      match = [source, target] * "~"
-      index = get_index(database)
-
-      if index.include? match 
-        acc << setup(database, match) 
-
-      elsif undirected or undirected(database) 
-        inv = [target, source] * "~"
-        if index.include? inv 
-          setup(database, inv)
-          acc <<  inv 
-        end
-      end
-
-      acc
-    }
+  def setup(name, matches, reverse = false)
+    AssociationItem.setup matches, self, name, reverse
   end
 end

data/lib/rbbt/tsv/accessor.rb

@@ -1,6 +1,8 @@
 require 'yaml'
 require 'rbbt/annotations'
 require 'rbbt/tsv/dumper'
+require 'set'
+
 module TSV
 
   TSV_SERIALIZER = YAML
@@ -601,6 +603,7 @@ module TSV
   end
 
   def summary
+
     key = nil
     values = nil
     self.each do |k, v|
@@ -608,6 +611,7 @@ module TSV
       values = v
       break
     end
+
     with_unnamed do
       <<-EOF
 Filename = #{Path === filename ? filename.find : (filename || "No filename")}

data/lib/rbbt/tsv/change_id.rb

@@ -1,4 +1,6 @@
 require 'rbbt/tsv'
+require 'rbbt/persist'
+
 
 module TSV
   def self.change_key(tsv, format, options = {}, &block)
@@ -84,5 +86,122 @@ module TSV
     TSV.swap_id(self, *args)
   end
 
+  def self.translation_index(files, target = nil, source = nil, options = {})
+    return nil if source == target
+    options = Misc.add_defaults options.dup, :persist => true
+    fields = source ? [source] : nil
+    files.each do |file|
+      if TSV === file
+        all_fields = file.all_fields
+        target = file.fields.first if target.nil?
+        return file.index(options.merge(:target => target, :fields => fields, :order => true)) if (source.nil? or all_fields.include? source) and all_fields.include? target
+      else
+        all_fields = TSV.parse_header(file).all_fields
+        target = all_fields[1] if target.nil?
+        return TSV.index(file, options.merge(:target => target, :fields => fields, :order => true)) if (source.nil? or all_fields.include? source) and all_fields.include? target
+      end
+    end
+
+    files.each do |file|
+      all_fields = TSV === file ? file.all_fields : TSV.parse_header(file).all_fields 
+
+      files.each do |other_file|
+        next if file == other_file
+
+        other_all_fields = TSV === other_file ? other_file.all_fields : TSV.parse_header(other_file).all_fields 
+
+        common_field = (all_fields & other_all_fields).first
+
+        if common_field and (source.nil? or fields.include? source) and all_fields.include? common_field and 
+          other_all_fields.include? common_field and other_all_fields.include? target 
+
+          return Persist.persist_tsv(nil, Misc.fingerprint(files), {:files => files, :source => source, :target => target}, :prefix => "Translation index", :persist => options[:persist]) do |data|
+            index = TSV === file ? 
+              file.index(options.merge(:target => common_field, :fields => fields)) :
+              TSV.index(file, options.merge(:target => common_field, :fields => fields))
+
+            other_index = TSV === other_file ? 
+              other_file.index(options.merge(:target => target, :fields => [common_field])) :
+              TSV.index(other_file, options.merge(:target => target, :fields => [common_field]))
+
+            data.serializer = :clean
+            data.merge! index.to_list.attach(other_index.to_list).slice([target]).to_single
+
+            data
+          end
+        end
+      end
+    end
+    return nil
+  end
+
+  def self.translate(tsv, *args)
+    new = TSV.open translate_stream(tsv, *args)
+    new.identifiers = tsv.identifiers
+    new
+  end
+
+  def self.translate_stream(tsv, field, format, options = {}, &block)
+    options = Misc.add_defaults options, :persist => false, :identifier_files => tsv.identifier_files, :compact => true
+
+    identifier_files, identifiers, persist_input, compact = Misc.process_options options, :identifier_files, :identifiers, :persist, :compact
+    identifier_files = [tsv, identifiers].compact if identifier_files.nil? or identifier_files.empty?
+
+    identifier_files.uniq!
+
+    index = translation_index identifier_files, format, field, options.dup
+    raise "No index: #{Misc.fingerprint([identifier_files, field, format])}" if index.nil?
+
+    orig_type = tsv.type 
+    tsv = tsv.to_double if orig_type != :double
+
+    pos = tsv.identify_field field
 
+    new_options = tsv.options
+    new_options[:identifiers] = tsv.identifiers.find if tsv.identifiers
+
+    case pos
+    when :key
+      new_options[:key_field] = format if tsv.key_field == field
+      dumper = TSV::Dumper.new new_options
+      dumper.init
+      TSV.traverse tsv, :into => dumper do |key,values|
+        new_key = index[key]
+        [new_key, values]
+      end
+    else
+      new_options[:fields] = tsv.fields.collect{|f| f == field ? format : f }
+      dumper = TSV::Dumper.new new_options
+      dumper.init
+
+      case tsv.type
+      when :double
+        TSV.traverse tsv, :into => dumper do |key,values|
+          original = values[pos]
+          new = index.values_at *original
+          values[pos] = new
+          [key, values]
+        end
+      when :list
+        TSV.traverse tsv, :into => dumper do |key,values|
+          original = values[pos]
+          new = index[original]
+          values[pos] = new
+          [key, values]
+        end
+      when :flat
+        TSV.traverse tsv, :into => dumper do |key,values|
+          new = index.values_at *values
+          [key, new]
+        end
+      when :single
+        TSV.traverse tsv, :into => dumper do |key,original|
+          new = index[original]
+          [key, new]
+        end
+      end
+    end
+
+    dumper.stream
+  end
 end

data/lib/rbbt/tsv/index.rb

@@ -32,12 +32,16 @@ module TSV
 
           new_key_field, new_fields = through target, fields, true do |key, values|
             next if key.empty? 
-            if type == :single
+            case type
+            when :single
               values = [values]
               values.unshift key
-            else
+            when :double
               values = values.dup
               values.unshift [key]
+            when :list, :flat
+              values = values.dup
+              values.unshift key
             end
 
             values.each_with_index do |list, i|

data/lib/rbbt/tsv/parser.rb

@@ -343,6 +343,8 @@ module TSV
         when String === key_field
           @key_position = @fields.dup.unshift(@key_field).index key_field
           raise "Key field #{ key_field } was not found" if @key_position.nil?
+        when :key == key_field
+          @key_position = 0
         else
           raise "Format of key_field not understood: #{key_field.inspect}"
         end
@@ -503,11 +505,11 @@ module TSV
       # get parser
 
       # grep
-      if grep
-        stream.rewind if stream.eof?
-        stream = Open.grep(stream, grep, invert_grep)
-        self.first_line = stream.gets
-      end
+      #if grep and false
+      #  stream.rewind if stream.eof?
+      #  stream = Open.grep(stream, grep, invert_grep)
+      #  self.first_line = stream.gets
+      #end
 
       # first line
       line = self.rescue_first_line