rbbt-util 5.4.1 → 5.5.0

data/lib/rbbt/association/index.rb

@@ -0,0 +1,92 @@
+require 'rbbt/tsv'
+module Association
+  module Index
+
+    attr_accessor :source_field, :target_field, :undirected
+    def parse_key_field
+      @source_field, @target_field, @undirected = key_field.split("~")
+    end
+
+    def self.setup(repo)
+      repo.extend Association::Index
+      repo.parse_key_field
+      repo.unnamed = true
+    end
+
+    def reverse
+      @reverse ||= begin
+                     reverse_filename = persistence_path + '.reverse'
+
+                     if File.exists?(reverse_filename)
+                       new = Persist.open_tokyocabinet(reverse_filename, false, serializer, TokyoCabinet::BDB)
+                     else
+                       new = Persist.open_tokyocabinet(reverse_filename, true, serializer, TokyoCabinet::BDB)
+                       new.write
+                       through do |key, value|
+                         new_key = key.split("~").reverse.join("~")
+                         new[new_key] = value
+                       end
+                       annotate(new)
+                       new.key_field = key_field.split("~").values_at(1,0,2).compact * "~"
+                       new.close
+                     end
+
+                     new.unnamed = true
+
+                     Association::Index.setup new
+                     new
+                   end
+    end
+
+    def match(entity)
+      return [] if entity.nil?
+      prefix(entity + "~")
+    end
+
+    def matches(entities)
+      entities.inject(nil) do |acc,e| 
+        m = match(e); 
+        if acc.nil? or acc.empty?
+          acc = m
+        else
+          acc.concat m
+        end
+        acc
+      end
+    end
+ 
+    #{{{ Subset
+
+    def select_entities(entities)
+      source_type = Entity.formats[source_field] 
+      target_type = Entity.formats[target_field]
+
+      source_entities = entities[source_field] || entities[Entity.formats[source_field].to_s]  
+      target_entities = entities[target_field] || entities[Entity.formats[target_field].to_s]
+
+      [source_entities, target_entities]
+    end
+
+    def subset(source, target)
+      return [] if source.nil? or source.empty? or target.nil? or target.empty?
+
+      matches = source.uniq.inject([]){|acc,e| acc.concat(match(e)) }
+
+      target_matches = {}
+
+      matches.each{|code| 
+        s,sep,t = code.partition "~"
+        next if (undirected and t > s) 
+        target_matches[t] ||= []
+        target_matches[t] << code
+      }
+
+      target_matches.values_at(*target.uniq).flatten.compact
+    end
+
+    def subset_entities(entities)
+      source, target = select_entities(entities)
+      subset source, target
+    end
+  end
+end

data/lib/rbbt/association/item.rb

@@ -0,0 +1,44 @@
+require 'rbbt/entity'
+
+module AssociationItem
+  extend Entity
+
+  annotation :knowledge_base
+  annotation :database
+  annotation :reverse
+
+  property :part => :array2single do
+    self.clean_annotations.collect{|p| p.partition("~") }
+  end
+
+  property :target => :array2single do
+    self.part.collect{|p| p[2]}
+  end
+
+  property :source => :array2single do
+    self.clean_annotations.collect{|p| p[/[^~]+/] }
+  end
+
+  property :target_entity => :array2single do
+    type = reverse ? knowledge_base.source(database) : knowledge_base.target(database)
+    knowledge_base.annotate self.target, type if self.target.any?
+  end
+
+  property :source_entity => :array2single do
+    type = reverse ? knowledge_base.target(database) : knowledge_base.source(database)
+    knowledge_base.annotate self.source, type if self.target.any?
+  end
+
+  property :value => :array2single do
+    value = knowledge_base.get_index(database).chunked_values_at self
+    value.collect{|v| NamedArray.setup(v, knowledge_base.get_index(database).fields)}
+  end
+
+  property :info => :array2single do
+    fields = knowledge_base.index_fields(database)
+    return [{}] * self.length if fields.nil? or fields.empty?
+    value.collect{|v|
+      Hash[*v.zip(fields).flatten]
+    }
+  end
+end

data/lib/rbbt/entity.rb

@@ -32,6 +32,10 @@ module Entity
         end
       end
 
+      def base_entity
+        self.annotation_types.select{|m| Entity === m}.last
+      end
+
       def property(*args, &block)
         class << self; self; end.property(*args,&block)
       end

data/lib/rbbt/fix_width_table.rb

@@ -1,7 +1,7 @@
 class FixWidthTable
 
   attr_accessor :filename, :file, :value_size, :record_size, :range, :size
-  def initialize(filename, value_size = nil, range = nil, update = false)
+  def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
     @filename = filename
 
     if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exists?(filename)
@@ -23,7 +23,11 @@ class FixWidthTable
       @size = 0
     else
       Log.debug "FixWidthTable up-to-date: #{ filename }"
-      @file        = File.open(@filename, 'r')
+      if in_memory
+        @file        = StringIO.new(Open.read(@filename, :mode => 'rb'), 'r')
+      else
+        @file        = File.open(@filename, 'r')
+      end
       @value_size  = @file.read(4).unpack("L").first
       @range       = @file.read(1).unpack("C").first == 1
       @record_size = @value_size + (@range ? 12 : 4)
@@ -145,18 +149,19 @@ class FixWidthTable
 
     while(upper >= lower) do
       idx = lower + (upper - lower) / 2
-      comp = pos <=> pos(idx)
+      pos_idx = pos(idx)
 
-      if comp == 0
-        break 
-      elsif comp > 0
-        lower = idx + 1
-      else
+      case pos <=> pos_idx
+      when 0
+        break
+      when -1
         upper = idx - 1
+      when 1
+        lower = idx + 1
       end
     end
 
-    if pos(idx) > pos
+    if pos_idx > pos
       idx = idx - 1
     end
 

data/lib/rbbt/knowledge_base.rb

@@ -0,0 +1,269 @@
+require 'rbbt/association'
+require 'rbbt/association/item'
+require 'rbbt/entity'
+
+class KnowledgeBase
+  class << self
+    attr_accessor :knowledge_base_dir, :registry
+
+    def registry
+      @registry ||= IndiferentHash.setup({})
+    end
+    
+    def knowledge_base_dir
+      @knowledge_base_dir ||= Rbbt.var.knowledge_base
+    end
+  end
+
+  attr_accessor :namespace, :dir, :indices, :registry, :format, :databases, :entity_options
+  def initialize(dir, namespace = nil)
+    @dir = Path.setup dir
+
+    @namespace = namespace
+    @format = IndiferentHash.setup({})
+
+    @registry = IndiferentHash.setup({})
+    @entity_options = IndiferentHash.setup({})
+
+    @indices = IndiferentHash.setup({})
+    @databases = IndiferentHash.setup({})
+    @identifiers = IndiferentHash.setup({})
+    @descriptions = {}
+    @databases = {}
+  end
+
+  def version(new_namespace, force = false)
+    return self if new_namespace == namespace and not force
+    new_kb = KnowledgeBase.new dir[new_namespace], new_namespace
+    new_kb.format.merge! self.format
+    new_kb.entity_options.merge! self.entity_options
+    new_kb.registry = self.registry
+    new_kb
+  end
+
+  #{{{ Descriptions
+ 
+  def register(name, file = nil, options = {}, &block)
+    if block_given?
+      Log.debug("Registering #{ name } from code block")
+      @registry[name] = [block, options]
+    else
+      Log.debug("Registering #{ name }: #{ Misc.fingerprint file }")
+      @registry[name] = [file, options]
+    end
+  end
+
+  def all_databases
+    (@indices.keys + @registry.keys).uniq
+  end
+
+  def description(name)
+    @descriptions[name] ||= get_index(name).key_field.split("~")
+  end
+
+  def source(name)
+    description(name)[0]
+  end
+
+  def target(name)
+    description(name)[1]
+  end
+
+  def undirected(name)
+    description(name)[2]
+  end
+
+  def source_type(name)
+    Entity.formats[source(name)]
+  end
+
+  def target_type(name)
+    Entity.formats[target(name)]
+  end
+
+  def index_fields(name)
+    get_index(name).fields
+  end
+
+  def entities
+    all_databases.inject([]){|acc,name| acc << source(name); acc << target(name)}.uniq
+  end
+
+  def entity_types
+    entities.collect{|entity| Entity.formats[entity] }.uniq
+  end
+
+  #{{{ Open and get
+ 
+  def open_options
+    {:namespace => namespace, :format => @format}
+  end
+ 
+  def get_database(name, options = {})
+    persist_options = Misc.pull_keys options, :persist
+
+    file, registered_options = registry[name]
+    options = open_options.merge(registered_options || {}).merge(options)
+    raise "Repo #{ name } not found and not registered" if file.nil?
+
+    @databases[name] ||= begin 
+                           Log.debug "Opening database #{ name } from #{ Misc.fingerprint file }. #{options}"
+                           Association.open(file, options, persist_options).
+                             tap{|tsv| tsv.namespace = self.namespace}
+                         end
+  end
+
+ 
+  def get_index(name, options = {})
+    persist_options = Misc.pull_keys options, :persist
+
+    file, registered_options = registry[name]
+    options = open_options.merge(registered_options || {}).merge(options)
+    raise "Repo #{ name } not found and not registered" if file.nil?
+
+    @indices[name] ||= begin 
+                           Log.debug "Opening index #{ name } from #{ Misc.fingerprint file }. #{options}"
+                           Association.index(file, options, persist_options).
+                             tap{|tsv| tsv.namespace = self.namespace}
+                         end
+  end
+
+  def index(name, file, options = {}, persist_options = {})
+    @indices[name] = Association.index(file, open_options.merge(options), persist_options)
+  end
+
+  #{{{ Add manual database
+  
+  def add_index(name, source_type, target_type, *fields)
+    options = fields.pop if Hash === fields.last
+    options ||= {}
+    undirected = Misc.process_options options, :undirected 
+
+    undirected = nil unless undirected 
+
+    repo_file = dir[name].find
+    index = Association.index(nil, {:namespace => namespace, :key_field => [source_type, target_type, undirected].compact * "~", :fields => fields}.merge(options), :file => repo_file, :update => true)
+    @indices[name] = index
+  end
+
+  def add(name, source, target, *rest)
+    code = [source, target] * "~"
+    repo = @indices[name]
+    repo[code] = rest
+  end
+
+  def write(name)
+    repo = @indices[name]
+    repo.write_and_read do
+      yield
+    end
+  end
+
+  #{{{ Annotate
+  
+  def entity_options_for(type)
+    options = entity_options[Entity.formats[type]] || {}
+    options[:format] = @format[type] if @format.include? :type
+    options = {:organism => namespace}.merge(options)
+    options
+  end
+
+  def annotate(entities, type)
+    Misc.prepare_entity(entities, type, entity_options_for(type))
+  end
+
+  #{{{ Identify
+  
+  def identify_source(name, entity)
+    database = get_database(name, :persist => true)
+    return entity if database.include? entity
+    source = source(name)
+    @identifiers[name] ||= {}
+    @identifiers[name]['source'] ||= begin
+                                       if database.identifier_files.any?
+                                         if TSV.parse_header(database.identifier_files.first).all_fields.include? source
+                                           TSV.index(database.identifiers, :target => source, :persist => true)
+                                         else
+                                           {}
+                                         end
+                                       else
+                                         if TSV.parse_header(Organism.identifiers(namespace)).all_fields.include? source
+                                           Organism.identifiers(namespace).index(:target => source, :persist => true)
+                                         else
+                                           {}
+                                         end
+                                       end
+                                     end
+
+    @identifiers[name]['source'][entity]
+  end
+
+  def identify_target(name, entity)
+    database = get_database(name, :persist => true)
+    target = target(name)
+
+    @identifiers[name] ||= {}
+    @identifiers[name]['target'] ||= begin
+                                       if database.identifier_files.any?
+                                         if TSV.parse_header(database.identifier_files.first).all_fields.include? target
+                                           TSV.index(database.identifiers, :target => target, :persist => true)
+                                         else
+                                           {}
+                                         end
+                                       else
+                                         if TSV.parse_header(Organism.identifiers(namespace)).all_fields.include? target
+                                           Organism.identifiers(namespace).index(:target => target, :persist => true)
+                                         else
+                                          database.index(:target => database.fields.first, :fields => [database.fields.first], :persist => true)
+                                         end
+                                       end
+                                     end
+    @identifiers[name]['target'][entity]
+  end
+
+  def identify(name, entity)
+    identify_source(name, entity) || identify_target(name, entity)
+  end
+
+  #{{{ Query
+
+  def children(name, entity)
+    repo = get_index name
+    AssociationItem.setup repo.match(entity), self, name, false
+  end
+
+  def parents(name, entity)
+    repo = get_index name
+    AssociationItem.setup repo.reverse.match(entity), self, name, true
+  end
+
+  def neighbours(name, entity)
+    if undirected(name)
+      IndiferentHash.setup({:children => children(name, entity)})
+    else
+      IndiferentHash.setup({:parents => parents(name, entity), :children => children(name, entity)})
+    end
+  end
+
+  def subset(name, entities)
+    case entities
+    when AnnotatedArray
+      format = entities.format if entities.respond_to? :format 
+      format ||= entities.base_entity.to_s
+      {format => entities.clean_annotations}
+    when Hash
+    else
+      raise "Entities are not a Hash or an AnnotatedArray: #{Misc.fingerprint entities}"
+    end
+    repo = get_index name
+    AssociationItem.setup repo.subset_entities(entities), self, name, false
+  end
+
+  def translate(entities, type)
+    if format = @format[type] and format != entities.format
+      entities.to format
+    else
+      entities
+    end
+  end
+end