rbbt-util 1.2.1 → 2.0.1

data/lib/rbbt/util/tsv/accessor.rb

@@ -0,0 +1,273 @@
+require 'rbbt/util/misc'
+
+class TSV
+  ## Make sure we overwrite the methods declared by attr_accessor
+  MAIN_ACCESSORS = :data,  :key_field, :fields, :cast
+  EXTRA_ACCESSORS = :filename, :identifiers, :namespace, :datadir, :type, :case_insensitive
+  attr_accessor *(MAIN_ACCESSORS + EXTRA_ACCESSORS)
+
+  def self.zip_fields(list, fields = nil)
+    return [] if list.nil? || list.empty?
+    fields ||= list.fields if list.respond_to? :fields
+    zipped = list[0].zip(*list[1..-1])
+    zipped = zipped.collect{|v| NamedArray.name(v, fields)} if fields 
+    zipped 
+  end
+ 
+  module Field
+    attr_accessor :namespace
+
+    def self.field(field, namespace = nil)
+      field.extend Field
+      field.namespace = namespace
+      field
+    end
+
+    def self.namespace(string)
+      return nil unless string.match(/(.+):/)
+      namespace_str = $1
+      return nil if namespace_str.nil? or namespace_str.empty?
+      namespace_str
+    end
+
+    def fullname
+      return self if self =~ /:/ or namespace.nil?
+      namespace + ":" << self
+    end
+
+    def ==(string)
+      return false unless String === string
+      return true if self.casecmp(string) == 0
+      if Field === string
+        return true if self.fullname.casecmp(string.fullname) == 0
+      else
+        return true if self.fullname.casecmp(string)  == 0
+      end
+      return true if self.sub(/.*:/,'').casecmp(string) == 0
+      return false
+    end
+
+    def namespace
+      Field.namespace(self) || @namespace
+    end
+
+    def matching_namespaces(other)
+      return true if namespace.nil?
+      return namespace == other.namespace
+    end
+  end
+
+  #{{{{ Field END
+
+  def identifier_files
+    case
+    when (identifiers and TSV === identifiers)
+      [identifiers]
+    when (identifiers and Array === identifiers)
+      case
+      when (TSV === identifiers.first or identifiers.empty?)
+        identifiers
+      when
+        identifiers.collect{|f| Path.path(f, datadir, namespace)}
+      end
+    when (identifiers and not Array === identifiers)
+      [Path.path(identifiers, datadir)]
+    when filename
+      Path.path(filename, datadir).identifier_files
+    else
+      []
+    end
+  end
+
+  def fields_in_namespace(namespace = nil)
+    namespace = self.namespace if namespace == nil or TrueClass === namespace
+    fields.select{|f| f.namespace.nil? or f.namespace == namespace}
+  end
+
+  def fields
+    return nil if @fields.nil?
+    fds = @fields
+    fds.each do |f| f.extend Field end if Array === @fields
+    fds.each do |f| f.namespace = namespace end unless namespace.nil?
+    NamedArray.name(fds, @fields)
+  end
+
+  def all_fields
+    return nil if @fields.nil?
+    all_fields = @fields.dup
+    all_fields.unshift key_field
+    all_fields.each do |f| f.extend Field end if Array === @fields
+    all_fields.each do |f| f.namespace = namespace end unless namespace.nil?
+    NamedArray.name(all_fields, [key_field] +  @fields)
+    all_fields
+  end
+
+  def all_namespace_fields(namespace = nil)
+    namespace = self.namespace if namespace == nil or TrueClass === namespace
+    all_fields = self.all_fields
+    return nil if all_fields.nil?
+    return all_fields if namespace.nil?
+    all_fields.select{|f| f.namespace.nil? or f.namespace == namespace}
+  end
+
+  def self.identify_field(key, fields, field)
+    return field if Integer === field
+    return :key if field.nil? or field == 0 or field.to_sym == :key or key == field 
+    return nil if fields.nil?
+    return fields.collect{|f| f.to_s}.index field if fields.collect{|f| f.to_s}.index field
+    return fields.index field 
+  end
+
+  def identify_field(field)
+    TSV.identify_field(key_field, fields, field)
+  end
+
+  def fields=(new_fields)
+    new_fields.collect! do |field| 
+      if Field === field
+        if field !~ /:/ and field.namespace != nil and field.namespace != namespace
+          field.namespace + ":" + field.to_s
+        else
+          field
+        end
+      else
+        field
+      end
+    end if Array === new_fields
+    @fields = new_fields
+    @data.fields = new_fields if @data.respond_to? :fields=
+  end
+
+  def old_fields=(new_fields)
+    @fields = new_fields
+    @data.fields = new_fields if @data.respond_to? :fields=
+  end
+
+  def keys
+    @data.keys
+  end
+
+  def values
+    @data.values
+  end
+
+  def size
+    @data.size
+  end
+
+  # Write
+
+  def []=(key, value)
+    key = key.downcase if @case_insensitive
+    @data[key] = value
+  end
+
+
+  def merge!(new_data)
+    new_data.each do |key, value|
+      self[key] = value
+    end
+  end
+
+  # Read
+
+  def follow(value)
+    return nil if value.nil?
+    if String === value && value =~ /__Ref:(.*)/
+      return self[$1]
+    else
+      value = NamedArray.name value, fields if Array === value and fields 
+      value
+    end
+  end
+
+  def [](key)
+    if Array === key
+      return @data[key] if @data[key] != nil
+      key.each{|k| v = self[k]; return v unless v.nil?}
+      return nil
+    end
+
+    key = key.downcase if @case_insensitive and key !~ /^__Ref:/
+    follow @data[key]
+  end
+
+  def values_at(*keys)
+    keys.collect{|k|
+      self[k]
+    }
+  end
+
+  def each(&block)
+    @data.each do |key, value|
+      block.call(key, follow(value))
+    end
+  end
+
+  def collect
+    if block_given?
+      @data.collect do |key, value|
+        value = follow(value)
+        yield key, value
+      end
+    else
+      @data.collect do |key, value|
+        [key, follow(value)]
+      end
+    end
+  end
+
+  def sort(&block)
+    collect.sort(&block).collect{|p|
+      key, value = p
+      value = NamedArray.name value, fields if fields
+      [key, value]
+    }
+  end
+
+  def sort_by(&block)
+    collect.sort_by &block
+  end
+
+  def values_to_s(values)
+      case
+      when (values.nil? and fields.nil?)
+        "\n"
+      when (values.nil? and not fields.nil?)
+        "\t" << ([""] * fields.length) * "\t" << "\n"
+      when (not Array === values)
+        "\t" << values.to_s << "\n"
+      when Array === values.first
+        "\t" << values.collect{|list| (list || []) * "|"} * "\t" << "\n"
+      else
+        "\t" << values * "\t" << "\n"
+      end
+  end
+
+  def include?(key)
+    data.include? key
+  end
+
+  def to_s(keys = nil)
+    str = ""
+
+    str << "#: " << Misc.hash2string(EXTRA_ACCESSORS.collect{|key| [key, self.send(key)]}) << "\n"
+    if fields
+      str << "#" << key_field << "\t" << fields * "\t" << "\n"
+    end
+
+    if keys.nil?
+      each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup << values_to_s(values)
+      end
+    else
+      keys.zip(values_at(*keys)).each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup << values_to_s(values)
+      end
+    end
+
+    str
+  end
+end

data/lib/rbbt/util/tsv/attach.rb

@@ -0,0 +1,228 @@
+class TSV
+
+  #{{{ Attach Methods
+  
+  def attach_same_key(other, fields = nil)
+    fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+
+    through do |key, values|
+      if other.include? key
+        new_values = other[key].values_at *fields
+        new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
+        new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+        self[key] = self[key].concat new_values
+      else
+        if type == :double
+          self[key] = self[key].concat [[]] * fields.length
+        else
+          self[key] = self[key].concat [""] * fields.length
+        end
+      end
+    end
+
+    self.fields = self.fields.concat other.fields.values_at *fields
+  end
+
+  def attach_source_key(other, source, fields = nil)
+    fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+
+    other = other.tsv unless TSV === other
+    field_positions = fields.collect{|field| other.identify_field field}
+    field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
+
+    through do |key, values|
+      source_keys = values[source]
+      if source_keys.nil? or source_keys.empty?
+        all_new_values = []
+      else
+        all_new_values = []
+        source_keys.each do |source_key|
+          next unless other.include? source_key
+          new_values = field_positions.collect do |pos|
+            if pos == :key
+              source_key
+            else
+              other[source_key][pos]
+            end
+          end
+
+          new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
+          new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+          all_new_values << new_values
+        end
+      end
+
+      if all_new_values.empty?
+        if type == :double
+          self[key] = self[key].concat [[]] * field_positions.length
+        else
+          self[key] = self[key].concat [""] * field_positions.length
+        end
+      else
+        if type == :double
+          self[key] = self[key].concat TSV.zip_fields(all_new_values).collect{|l| l.flatten}
+        else
+          self[key] = self[key].concat all_new_values.first
+        end
+      end
+    end
+
+    self.fields = self.fields.concat field_names
+  end
+
+  def attach_index(other, index, fields = nil)
+    fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+
+    other = other.tsv unless TSV === other
+    field_positions = fields.collect{|field| other.identify_field field}
+    field_names     = field_positions.collect{|pos| pos == :key ? other.key_field : other.fields[pos] }
+
+    through do |key, values|
+      source_keys = index[key]
+      if source_keys.nil? or source_keys.empty?
+        all_new_values = []
+      else
+        all_new_values = []
+        source_keys.each do |source_key|
+          next unless other.include? source_key
+          new_values = field_positions.collect do |pos|
+            if pos == :key
+              source_key
+            else
+              other[source_key][pos]
+            end
+          end
+          new_values.collect!{|v| [v]}     if     type == :double and not other.type == :double
+          new_values.collect!{|v| v.first} if not type == :double and     other.type == :double
+          all_new_values << new_values
+        end
+      end
+
+      if all_new_values.empty?
+        if type == :double
+          self[key] = self[key].concat [[]] * field_positions.length
+        else
+          self[key] = self[key].concat [""] * field_positions.length
+        end
+      else
+        if type == :double
+          self[key] = self[key].concat TSV.zip_fields(all_new_values).collect{|l| l.flatten}
+        else
+          self[key] = self[key].concat all_new_values.first
+        end
+      end
+    end
+
+    self.fields = self.fields.concat field_names
+  end
+
+  #{{{ Attach Helper
+
+  # May make an extra index!
+  def self.find_path(files, in_namespace = false)
+    if in_namespace
+      ids = [files.first.all_namespace_fields(in_namespace)]
+      ids += files[1..-1].collect{|f| f.all_fields}
+    else
+      ids = files.collect{|f| f.all_fields}
+    end
+    id_list = []
+
+    ids.flatten.each do |field|
+    end
+
+    ids.each_with_index do |list, i|
+      break if i == ids.length - 1
+      match = list.select{|field| 
+        ids[i + 1].select{|f| field == f}.any?
+      }
+      return nil if match.empty?
+      id_list << match.first
+    end
+    
+    if id_list.last.first != files.last.all_fields.first
+      id_list << files.last.all_fields.first
+      id_list.zip(files)
+    else
+      id_list.zip(files[0..-1])
+    end
+  end
+
+  def self.build_traverse_index(files, in_namespace = false)
+    path = find_path(files, in_namespace)
+
+    return nil if path.nil?
+    
+    traversal_ids = path.collect{|p| p.first}
+    
+    Log.medium "Found Traversal: #{traversal_ids * " => "}"
+
+    current_key = files.first.all_fields.first
+    target = files.last.all_fields.first
+    target = nil
+    current_id, current_file = path.shift
+    index   = current_file.index :target => current_id, :fields =>  current_key, :persistence => false
+
+    while not path.empty?
+      current_id, current_file = path.shift
+      current_index   = current_file.index :target => current_id, :fields => index.fields.first, :persistence => true
+      index.process 0 do |value|
+        current_index.values_at(*value).flatten.uniq
+      end
+      index.fields = current_index.fields
+    end
+
+    index
+  end
+
+  def self.find_traversal(tsv1, tsv2, in_namespace = false)
+    identifiers1 = tsv1.identifier_files || []
+    identifiers2 = tsv2.identifier_files || []
+
+    identifiers1.unshift tsv1
+    identifiers2.unshift tsv2
+
+    files1 = []
+    files2 = []
+    while identifiers1.any?
+      files1.push identifiers1.shift
+      identifiers2.each_with_index do |e,i|
+        files2 = identifiers2[(0..i)]
+        index  = build_traverse_index(files1 + files2.reverse, in_namespace)
+        return index if not index.nil?
+      end
+    end
+
+    return nil
+  end
+
+  def attach(other, fields = nil, options = {})
+    options = Misc.add_defaults options, :in_namespace => true
+    in_namespace = Misc.process_options options, :in_namespace
+
+    fields = other.fields - [key_field].concat(self.fields) if fields == :all
+    fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
+    Log.high("Attaching fields:#{fields.inspect} from #{other.filename.inspect}.")
+    case
+    when key_field == other.key_field
+      attach_same_key other, fields
+    when (not in_namespace and self.fields.include?(other.key_field))
+      attach_source_key other, other.key_field, fields
+    when (in_namespace and self.fields_in_namespace.include?(other.key_field))
+      attach_source_key other, other.key_field, fields
+    else
+      index = TSV.find_traversal(self, other, in_namespace)
+      raise "Cannot traverse identifiers" if index.nil?
+      attach_index other, index, fields
+    end
+    Log.medium("Attachment of fields:#{fields.inspect} from #{other.filename.inspect} finished.")
+  end
+
+  def detach(file)
+    file_fields = file.fields.collect{|field| field.fullname}
+    detached_fields = []
+    self.fields.each_with_index{|field,i| detached_fields << i if file_fields.include? field.fullname}
+    reorder :key, detached_fields
+  end
+
+end