rbbt-util 5.14.33 → 5.14.34

data/lib/rbbt/association.rb

@@ -1,383 +1,4 @@
-require 'rbbt-util'
-require 'rbbt/tsv/change_id'
+require 'rbbt/association/open'
 require 'rbbt/association/index'
-
 module Association
-  class << self
-    attr_accessor :databases
-    def databases
-      @databases ||= {}
-    end
-  end
-
-  def self.add_reciprocal(tsv)
-    tsv = tsv.type == :double ? tsv : tsv.to_double
-    new = TSV.open(tsv.dumper_stream)
-    tsv.with_unnamed do
-      tsv.through do |source, values|
-        next if values.flatten.compact.empty?
-        if values.length > 1
-          Misc.zip_fields(values).each do |_target_values|
-            target, *target_values = _target_values
-            if new[target].nil?
-              new_values = [[source]] + target_values.collect{|v| [v] }
-              new[target] = new_values
-            else
-              new_values = new[target].collect{|l| l.dup }
-              targets = new_values.shift
-              targets << source
-              rest = new_values.zip(target_values).collect do |o,n|
-                o << n
-                o
-              end
-              new_values = [targets] + rest
-              new[target] = new_values
-            end
-          end
-        else
-          values.first.each do |target|
-            if new[target].nil?
-              new[target] = [[source]]
-            else
-              new[target] = [new[target][0] + [source]]
-            end
-          end
-        end
-      end
-    end
-
-    tsv.annotate(new)
-
-    new
-  end
-
-  def self.resolve_field(name, fields)
-    entity_type = Entity.formats[name]
-    return "Field #{ name } could not be resolved: #{fields}" if entity_type.nil?
-    field = fields.select{|f| Entity.formats[f] == entity_type}.first
-    [field, nil, name]
-  end
-
-  def self.parse_field_specification(spec)
-    return [2,nil,nil] if Fixnum === spec
-    spec = spec.split "=>" unless Array === spec
-    field_part, final_format = spec
-
-    field, format = field_part.split "=~"
-
-    [field, format, final_format]
-  end
-
-  def self.calculate_headers(key_field, fields, spec)
-    all_fields = [key_field].concat fields if fields and key_field
-    field, header, format = parse_field_specification spec if spec
-
-    if field and key_field == field and not all_fields.include? field
-      field, header, format = resolve_field field, all_fields
-    end
-
-    [field, header, format]
-  end
-
-  #{{{ Open
-
-  def self.open_tsv(file, source, source_header, target, target_header, all_fields, options)
-    fields = Misc.process_options options, :fields
-    fields ||= all_fields.dup
-
-    fields.delete source 
-    fields.delete target
-    fields.unshift target 
-
-    open_options = options.merge({
-      :persist => false,
-      :key_field => all_fields.index(source), 
-      :fields => fields.collect{|f| String === f ? all_fields.index(f): f },
-      :unnamed => true,
-      :merge => (options[:merge] or (options[:type] and options[:type].to_sym == :flat) ? false : true)
-    })
-
-    open_options["header_hash"] = "#" if options["header_hash"] == ""
-
-    field_headers = all_fields.values_at *open_options[:fields]
-
-    tsv = case file
-          when TSV
-            if file.fields == field_headers
-              file
-            else
-              file.reorder(source, field_headers)
-            end
-          else
-            TSV.open(file, open_options)
-          end
-
-    tsv.fields = field_headers
-    tsv.key_field = source
-
-    # Fix source header
-    if source_header and tsv.key_field != source_header
-      tsv.key_field = source_header
-    end
-
-    # Fix target header
-    if target_header and tsv.fields.first != target_header
-      tsv.fields = tsv.fields.collect{|f| f == target ? target_header : f }
-    end
-
-    tsv
-  end
-
-  def self.translate_tsv(tsv, source_final_format, target_final_format)
-    source_field = tsv.key_field
-    target_field = tsv.fields.first
-
-    if source_final_format and source_field != source_final_format and
-      Entity.formats[source_field] 
-
-      Log.debug("Changing source format from #{tsv.key_field} to #{source_final_format}")
-
-      tsv.with_unnamed do
-        identifiers = tsv.identifiers || Organism.identifiers(tsv.namespace)
-        tsv = tsv.change_key source_final_format, :identifiers => identifiers, :persist => true
-      end
-    end
-
-
-    # Translate target 
-    if target_final_format and target_field != target_final_format and
-      Entity.formats[target_field] and
-      Entity.formats[target_field] == Entity.formats[target_final_format]
-
-      Log.debug("Changing target format from #{tsv.fields.first} to #{target_final_format}")
-
-      save_key_field = tsv.key_field
-      tsv.key_field = "MASKED"
-
-      tsv.with_unnamed do
-        tsv = tsv.swap_id tsv.fields.first, target_final_format, :identifiers => Organism.identifiers(tsv.namespace), :persist => true, :compact => true
-      end
-
-      tsv.key_field = save_key_field 
-    end
-
-    tsv
-  end
-
-  def self.specs(all_fields, options = {})
-    source_spec, source_format, target_spec, target_format, format, key_field, fields = Misc.process_options options, :source, :source_format, :target, :target_format, :format, :key_field, :fields
-
-    if key_field and all_fields
-      key_pos = (Fixnum === key_field ? key_field : all_fields.index(key_field) )
-      key_field = all_fields[key_pos]
-    else
-      key_field = all_fields.first if all_fields
-    end
-
-    if fields and all_fields
-      field_pos = fields.collect{|f| Fixnum === f ? f : all_fields.index(f) }
-      fields = all_fields.values_at *field_pos
-    else
-      #fields = all_fields[1..-1] if all_fields
-    end
-
-    source, source_header, orig_source_format = calculate_headers(key_field, fields || all_fields, source_spec)
-    source_format ||= orig_source_format 
-    source = key_field if source.nil? 
-    source = key_field if source == :key
-    source_header ||= source
-
-    target, target_header, orig_target_format = calculate_headers(key_field, fields || all_fields, target_spec)
-    target_format ||= orig_target_format 
-    if target.nil?
-      target = case
-               when fields
-                 fields.first
-               when key_field == source
-                 all_fields[1]
-               else
-                 (([key_field] + all_fields) - [source]).first
-               end
-    end
-
-    target = key_field if target == :key
-    target_header ||= target
-
-    case format
-    when String
-      source_format ||= format if Entity.formats[source_header] == Entity.formats[format]
-      target_format ||= format if Entity.formats[target_header] == Entity.formats[format]
-    when Hash
-      _type = Entity.formats[source_header].to_s
-      source_format ||= format[_type] if format.include? _type 
-      _type = Entity.formats[target_header].to_s
-      target_format ||= format[_type] if format.include? _type 
-    end
-
-    [source, source_header, source_format, target, target_header, target_format, fields || all_fields]
-  end
-
-  def self.load_tsv(file, options)
-    undirected = options[:undirected]
-
-    case file
-    when Proc
-      res = file.call
-      tsv = case res
-            when TSV, Path
-              return load_tsv(res, options)
-            else
-              tsv = TSV.open(res, :unnamed => true)
-              return load_tsv(tsv, options)
-            end 
-    when TSV
-      key_field, *fields = all_fields = file.all_fields
-    else 
-      key_field, *fields = all_fields = TSV.parse_header(file, options.merge(:fields => nil, :key_field => nil)).all_fields
-    end
-
-    source, source_header, source_format, target, target_header, target_format, fields = specs(all_fields, options)
- 
-    Log.low("Loading associations from: #{ Misc.fingerprint file }")
-    Log.low("sources: #{ [source, source_header, source_format].join(", ") }")
-    Log.low("targets: #{ [target, target_header, target_format].join(", ") }")
-
-    tsv = open_tsv(file, source, source_header, target, target_header, all_fields, options.merge(:fields => fields.dup))
-
-    tsv = translate_tsv(tsv, source_format, target_format)
-
-    tsv = add_reciprocal(tsv) if undirected
-
-    tsv
-  end
-
-  def self.open(file, options = {}, persist_options = {})
-    options = {} if options.nil?
-    persist_options = {} if persist_options.nil?
-
-    namespace = options[:namespace]
-    old_file, file = file, file.sub('NAMESPACE', namespace) if namespace and String === file
-    old_file.annotate file if Path === old_file
-
-    Persist.persist_tsv(file, nil, options, {:persist => true, :prefix => "Association"}.merge(persist_options)) do |data|
-      options = options.clone
-
-      tsv = load_tsv(file, options)
-
-      tsv.annotate(data)
-
-      data.serializer = tsv.type 
-
-      tsv.with_unnamed do
-        tsv.each do |k,v|
-          next if v.nil?
-          data[k] = v
-        end
-      end
-
-      data
-    end
-  end
-
-  #{{{ Index
-
-  #def self.get_index(index_file, write = false)
-  #  Persist.open_tokyocabinet(index_file, write, :list, TokyoCabinet::BDB).tap{|r| r.unnamed = true; Association::Index.setup r }
-  #end
-
-  def self.index(file, options = {}, persist_options = nil)
-    options = {} if options.nil?
-    options = Misc.add_defaults options, :persist => true, :undirected => false
-    persist_options = Misc.pull_keys options, :persist if persist_options.nil?
-
-    expanded_persist_options = {:persist => true, :prefix => "Association Index"}.
-      merge(persist_options).
-      merge(:engine => TokyoCabinet::BDB, :serializer => :clean)
-
-    Persist.persist_tsv(file, nil, options, expanded_persist_options) do |assocs|
-      undirected = options[:undirected]
-      recycle = options[:recycle]
-      if file
-        tsv = TSV === file ? file : Association.open(file, options, persist_options.merge(:persist => false))
-
-        fields = tsv.fields
-        source_field = tsv.key_field
-        target_field = fields.first.split(":").last
-
-        key_field = [source_field, target_field, undirected ? "undirected" : nil].compact * "~"
-
-        TSV.setup(assocs, :key_field => key_field, :fields => fields[1..-1], :type => :list, :serializer => :list)
-
-        tsv.with_unnamed do
-          tsv.with_monitor :desc => "Extracting associations" do
-            case tsv.type
-            when :list
-              tsv.through do |source, values|
-                target, *rest = values
-                next if source.nil? or source.empty? or target.nil? or target.empty?
-
-                key = [source, target] * "~"
-                assocs[key] = rest
-              end
-            when :flat
-              tsv.through do |source, targets|
-                next if source.nil? or source.empty? or targets.nil? or targets.empty?
-
-                targets.each do |target|
-                  next if target.nil? or target.empty?
-                  key = [source, target] * "~"
-                  assocs[key] = nil
-                end
-              end
-
-            when :double
-              tsv.through do |source, values|
-                next if values.empty?
-                next if source.nil? or source.empty?
-                next if values.empty?
-
-                targets = values.first
-
-                rest =  values[1..-1]
-
-                size = values.first ? values.first.length : 0
-
-                rest.each_with_index do |list,i|
-                  list.replace [list.first] * size if list.length == 1
-                end if recycle and size > 1
-
-                rest = Misc.zip_fields rest
-
-                annotations = rest.length > 1 ?
-                  targets.zip(rest) :
-                  targets.zip(rest * targets.length) 
-
-                annotations.each do |target, info|
-                  next if target.nil? or target.empty?
-                  key = [source, target] * "~"
-                  if assocs[key].nil? or info.nil?
-                    assocs[key] = info
-                  else
-                    old_info = assocs[key]
-                    info = old_info.zip(info).collect{|p| p * ";;" }
-                    assocs[key] = info
-                  end
-                end
-              end
-            else
-              raise "Type not supported: #{tsv.type}"
-            end
-          end
-        end
-      else
-        key_field, fields = options.values_at :key_field, :fields
-        TSV.setup(assocs, :key_field => key_field, :fields => fields[1..-1], :type => :list, :serializer => :list)
-      end
-      assocs.close
-
-      assocs
-    end.tap do |assocs|
-      Association::Index.setup assocs
-    end
-  end
 end

data/lib/rbbt/entity/identifiers.rb

@@ -0,0 +1,106 @@
+module Entity
+
+  def self.identifier_files(field)
+    entity_type = Entity.formats[field]
+    return [] unless entity_type and entity_type.include? Entity::Identified 
+    entity_type.identifier_files
+  end
+
+  module Identified
+
+    def self.included(base)
+      base.annotation :format
+      base.annotation :organism
+
+      base.module_eval do
+        def identity_type
+          self.annotation_types.select{|m| m.include? Entity::Identified }.last
+        end
+
+        def identifier_files 
+          files = identity_type.identifier_files.dup
+          files.collect!{|f| f.annotate f.gsub(/\bNAMESPACE\b/, organism) } if annotations.include? :organism and self.organism
+          files.reject!{|f| f =~ /\bNAMESPACE\b/ } 
+          files
+        end
+
+        def identifier_index(format = nil, source = nil)
+          Persist.memory("Entity index #{identity_type}: #{format} (from #{source || "All"})", :format => format, :source => source) do
+            source ||= self.respond_to?(:format)? self.format : nil
+
+            index = TSV.translation_index(identifier_files, format, source, :persist => true)
+            index.unnamed = true
+            index
+          end
+        end
+      end
+
+      base.property :to => :both do |target_format|
+
+        target_format = case target_format
+                        when :name
+                          identity_type.name_format 
+                        when :default
+                          identity_type.default_format 
+                        when :ensembl
+                          identity_type.formats.select{|f| f =~ /Ensembl/}.first
+                        else
+                          target_format
+                        end
+
+        return self if target_format == format
+        if Array === self
+          self.annotate(identifier_index(target_format, self.format).values_at(*self))
+        else
+          self.annotate(identifier_index(target_format, self.format)[self])
+        end.tap{|o| o.format = target_format unless o.nil? }
+      end
+
+      base.property :name => :both do
+        to(:name)
+      end
+
+      base.property :default => :both do
+        to(:name)
+      end
+
+      base.property :ensembl => :both do
+        to(:ensembl)
+      end
+
+    end
+
+  end
+
+  attr_accessor :identifier_files, :formats, :default_format, :name_format, :description_format
+
+  def add_identifiers(file, default = nil, name = nil, description = nil)
+    if TSV === file
+      all_fields = file.all_fields
+    else
+      if file =~ /NAMESPACE/
+        all_fields = file.sub(/NAMESPACE/,'**').glob.collect do |f|
+        TSV.parse_header(f).all_fields
+        end.flatten.compact.uniq
+      else
+        all_fields = TSV.parse_header(file).all_fields
+      end
+    end
+
+    self.format = all_fields
+    @formats ||= []
+    @formats.concat all_fields
+    @formats.uniq!
+
+    @default_format = default if default
+    @name_format = name if name
+    @description_format = description if description
+
+    @identifier_files ||= []
+    @identifier_files << file
+
+
+    self.include Entity::Identified unless Entity::Identified === self
+  end
+
+end

data/lib/rbbt/entity.rb

@@ -1,4 +1,5 @@
 require 'rbbt/annotations'
+require 'rbbt/entity/identifiers'
 
 module Entity 
 

data/lib/rbbt/knowledge_base/entity.rb

@@ -0,0 +1,107 @@
+require 'rbbt/entity'
+require 'rbbt/knowledge_base/registry'
+
+class KnowledgeBase
+
+  def select_entities(name, entities, options = {})
+    index = get_index(name, options)
+    source_field = index.source_field
+    target_field = index.target_field
+
+    source_type = Entity.formats[source_field] 
+    target_type = Entity.formats[target_field]
+
+    source_entities = entities[:source] || entities[source_field] || entities[Entity.formats[source_field].to_s] 
+    target_entities = entities[:target] || entities[target_field] || entities[Entity.formats[target_field].to_s]
+
+    [source_entities, target_entities]
+  end
+
+
+  def entity_options_for(type, database_name = nil)
+    options = entity_options[Entity.formats[type]] || {}
+    options[:format] = @format[type] if @format.include? :type
+    options = {:organism => namespace}.merge(options)
+    if database_name and 
+      (database = get_database(database_name)).entity_options and
+      (database = get_database(database_name)).entity_options[type]
+      options = options.merge database.entity_options[type] 
+    end
+    options
+  end
+
+  def translate(entities, type)
+    if format = @format[type] and (entities.respond_to? :format and format != entities.format)
+      entities.to format
+    else
+      entities
+    end
+  end
+
+  def annotate(entities, type, database = nil)
+    format = @format[type] || type
+    Misc.prepare_entity(entities, format, entity_options_for(type, database))
+  end
+
+  def source_type(name)
+    Entity.formats[source(name)]
+  end
+
+  def target_type(name)
+    Entity.formats[target(name)]
+  end
+
+  def entities
+    all_databases.inject([]){|acc,name| acc << source(name); acc << target(name)}.uniq
+  end
+
+  def entity_types
+    entities.collect{|entity| Entity.formats[entity] }.uniq
+  end
+
+  def identifier_files(name)
+    get_database(name).identifier_files.dup
+  end
+
+  def source_index(name)
+    Persist.memory("Source index #{name}: KB directory #{dir}") do
+      identifier_files = identifier_files(name)
+      identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, source(name), nil, :persist => true
+    end
+  end
+  
+  def target_index(name)
+    Persist.memory("Target index #{name}: KB directory #{dir}") do
+      identifier_files = identifier_files(name)
+      identifier_files.concat Entity.identifier_files(source(name)) if defined? Entity
+      identifier_files.uniq!
+      identifier_files.collect!{|f| f.annotate(f.gsub(/\bNAMESPACE\b/, namespace))} if namespace
+      identifier_files.reject!{|f| f.match(/\bNAMESPACE\b/)}
+      TSV.translation_index identifier_files, target(name), nil, :persist => true
+    end
+  end
+
+  def identify_source(name, entity)
+    return :all if entity == :all
+    index = source_index(name)
+    return nil if index.nil?
+    index.values_at *entity
+  end
+
+  
+  def identify_target(name, entity)
+    return :all if entity == :all
+    index = target_index(name)
+    return nil if index.nil?
+    index.values_at *entity
+  end
+
+  def identify(name, entity)
+    identify_source(name, entity) || identify_target(name, entity)
+  end
+end
+

data/lib/rbbt/knowledge_base/query.rb

@@ -0,0 +1,83 @@
+require 'rbbt/entity'
+require 'rbbt/knowledge_base/registry'
+
+class KnowledgeBase
+
+  def _subset(name, source = :all, target = :all, options = {})
+    repo = get_index name, options
+
+    repo.subset(source, target)
+  end
+
+  def subset(name, entities, options = {}, &block)
+    entities, options = options, entities if entities.nil? and Hash === options
+    entities = case entities
+               when :all
+                 {:target => :all, :source => :all}
+               when AnnotatedArray
+                 format = entities.format if entities.respond_to? :format 
+                 format ||= entities.base_entity.to_s
+                 {format => entities.clean_annotations}
+               when Hash
+                 entities
+               else
+                 raise "Entities are not a Hash or an AnnotatedArray: #{Misc.fingerprint entities}"
+               end
+
+    source, target = select_entities(name, entities, options)
+
+    return [] if source.nil? or target.nil?
+    return [] if Array === target and target.empty?
+    return [] if Array === source and source.empty?
+
+    matches = _subset name, source, target, options
+
+    setup(name, matches)
+
+    matches = matches.select(&block) if block_given? 
+
+    matches
+  end
+
+  def all(name, options={})
+    repo = get_index name, options
+    setup name, repo.keys
+  end
+
+  def _children(name, entity)
+    repo = get_index name
+    repo.match(entity)
+  end
+
+  def children(name, entity)
+    entity = identify_source(name, entity)
+    setup(name, _children(name, entity))
+  end
+
+  def _parents(name, entity)
+    repo = get_index name
+    repo.reverse.match(entity)
+  end
+
+  def parents(name, entity)
+    entity = identify_target(name, entity)
+    setup(name, _parents(name, entity), true)
+  end
+
+  def _neighbours(name, entity)
+    if undirected(name) and source(name) == target(name)
+      {:children => _children(name, entity)}
+    else
+      {:parents => _parents(name, entity), :children => _children(name, entity)}
+    end
+  end
+
+  def neighbours(name, entity)
+    hash = _neighbours(name, entity)
+    IndiferentHash.setup(hash)
+    setup(name, hash[:children]) if hash[:children] 
+    setup(name, hash[:parents], true) if hash[:parents]
+    hash
+  end
+end
+