rbbt-util 3.2.1 → 4.0.0

data/lib/rbbt/tsv/accessor.rb

@@ -0,0 +1,345 @@
+require 'rbbt/util/chain_methods'
+
+module TSV
+  extend ChainMethods
+  self.chain_prefix = :tsv
+
+  attr_accessor :unnamed
+
+  def with_unnamed
+    saved_unnamed = @unnamed 
+    @unnamed = true
+    res = yield
+    @unnamed = saved_unnamed
+    res
+  end
+
+  def with_monitor(value = true)
+    saved_monitor = @monitor
+    @monitor = value
+    res = yield
+    @monitor = saved_monitor
+    res
+  end
+
+  def self.extended(data)
+    setup_chains(data)
+
+    if not data.respond_to? :write
+      class << data
+        attr_accessor :writable
+
+        def close
+        end
+
+        def read(force = false)
+          @writable = false
+          self
+        end
+
+        def write(force = false)
+          @writable = true
+          self
+        end
+
+        def write?
+          @writable
+        end
+      end
+    end
+  end
+
+  KEY_PREFIX = "__tsv_hash_"
+
+  ENTRIES = []
+  ENTRY_KEYS = []
+
+  def serialized_get(key)
+    raise "Uninitialized serializer" if serializer == :type
+    serialized_value = tsv_clean_get_brackets(key) 
+    SERIALIZER_ALIAS[serializer.to_sym].load(serialized_value) unless serialized_value.nil?
+  end
+
+  def serialized_set(key, value)
+    raise "Uninitialized serializer" if serializer == :type
+    if value.nil?
+      tsv_clean_set_brackets(key, nil)
+    else
+      tsv_clean_set_brackets(key, SERIALIZER_ALIAS[serializer.to_sym].dump(value))
+    end
+  end
+
+  #{{{ Chained Methods
+  def tsv_get_brackets(key)
+    value = if serializer.nil?
+            tsv_clean_get_brackets(key)
+          else
+            serialized_get(key)
+          end
+
+    NamedArray.setup value, fields if Array === value and not @unnamed
+    value
+  end
+
+  def tsv_set_brackets(key,value)
+    if serializer.nil?
+      tsv_clean_set_brackets(key, value)
+    else
+      serialized_set(key, value)
+    end
+  end
+
+  def tsv_keys
+    tsv_clean_keys - ENTRY_KEYS
+  end
+
+  def tsv_values
+    values = values_at(*keys)
+    values.each{|value| NamedArray.setup value, fields} if Array === values.first and not @unnamed
+    values
+  end
+
+  def tsv_each
+    tsv_clean_each do |key, value|
+      next if ENTRY_KEYS.include? key
+
+      value = SERIALIZER_ALIAS[serializer].load(value) unless serializer.nil?
+      NamedArray.setup value, fields if Array === value and not @unnamed
+      yield key, value if block_given?
+      [key, value]
+    end
+  end
+
+  def tsv_collect
+    tsv_clean_collect do |key, value|
+      next if ENTRY_KEYS.include? key
+      value = SERIALIZER_ALIAS[serializer].load(value) unless serializer.nil? or not String === value 
+      NamedArray.setup value, fields if Array === value and not @unnamed
+      if block_given?
+        yield key, value
+      else
+        [key, value]
+      end
+    end
+  end
+
+  def tsv_size
+    keys.length
+  end
+
+  def tsv_length
+    keys.length
+  end
+
+  def tsv_values_at(*keys)
+    keys.collect do |key|
+      self[key]
+    end
+  end
+
+  #{{{ Sorting
+
+  def tsv_sort_by(field = nil, just_keys = false, &block)
+    field = :all if field.nil?
+    if field == :all
+      elems = collect
+    else
+      elems = []
+      through :key, field do |key, fields|
+        elems << [key, fields.first]
+      end
+    end
+
+    if not block_given?
+      if fields == :all
+        if just_keys
+          elems.sort_by{|key, value| key }.collect{|key, values| key}
+        else
+          elems.sort_by{|key, value| key }
+        end
+      else
+        if just_keys
+          elems.sort_by{|key, value| value }.collect{|key, value| key}
+        else
+          elems.sort_by{|key, value| value }.collect{|key, value| [key, self[key]]}
+        end
+      end
+    else
+      if just_keys
+        elems.sort_by(&block).collect{|key, value| key}
+      else
+        elems.sort_by(&block).collect{|key, value| [key, self[key]]}
+      end
+    end
+  end
+
+  def tsv_sort(&block)
+    collect.sort &block
+  end
+
+  # Starts in page 1
+  def page(pnum, psize, field = nil, just_keys = false, &block)
+    if pnum.to_s =~ /-(.*)/
+      reverse = true
+      pnum = $1.to_i
+    else
+      reverse = false
+    end
+
+    with_unnamed do
+      pstart = psize * (pnum - 1)
+      pend = psize * pnum - 1
+      field = :key if field == "key"
+      keys = sort_by(field || :key, true, &block)
+      keys.reverse! if reverse
+
+      if just_keys
+        keys[pstart..pend]
+      else
+        select :key => keys[pstart..pend]
+      end
+    end
+  end
+
+  
+  def self.entry(*entries)
+    entries = entries.collect{|entry| entry.to_s}
+    ENTRIES.concat entries
+    entries.each do |entry|
+      key = KEY_PREFIX + entry
+      ENTRY_KEYS << key
+      self.module_eval "
+attr_accessor :#{entry}
+
+def #{ entry }
+  if not defined? @#{entry}
+    @#{entry} = YAML.load(self.tsv_clean_get_brackets('#{key}') || nil.to_yaml)
+  end
+  @#{entry}
+end
+
+def #{ entry }=(value)
+  @#{entry} = value
+  self.tsv_clean_set_brackets '#{key}', value.to_yaml
+end
+"
+    end
+  end
+
+  entry :key_field, 
+    :fields, 
+    :type,
+    :cast,
+    :identifiers,
+    :namespace,
+    :filename,
+    :serializer
+
+  def fields
+    @fields ||= YAML.load(self.tsv_clean_get_brackets("__tsv_hash_fields") || nil.to_yaml)
+    if @fields.nil? or @unnamed
+      @fields
+    else
+      NamedArray.setup @fields, @fields
+    end
+  end
+
+  def self.zip_fields(list, fields = nil)
+    return [] if list.nil? || list.empty?
+    fields ||= list.fields if list.respond_to? :fields
+    zipped = list[0].zip(*list[1..-1])
+    zipped = zipped.collect{|v| NamedArray.setup(v, fields)} if fields 
+    zipped 
+  end
+
+  def identifier_files
+    case
+    when (identifiers and TSV === identifiers)
+      [identifiers]
+    when (identifiers and Array === identifiers)
+      case
+      when (TSV === identifiers.first or identifiers.empty?)
+        identifiers
+      when
+        identifiers.collect{|f| Path === f ? f : Path.setup(f, nil, namespace)}
+      end
+    when identifiers
+      [ Path === identifiers ? identifiers : Path.setup(identifiers, nil, namespace) ]
+    when Path === filename
+      filename.identifier_files
+    when filename
+      Path.setup(filename).identifier_files
+    else
+      []
+    end
+  end
+
+  def options
+    options = {}
+    ENTRIES.each do |entry|
+      options[entry] = self.send(entry)
+    end
+    IndiferentHash.setup options
+  end
+
+
+  def all_fields
+    [key_field] + fields
+  end
+
+  def values_to_s(values)
+      case
+      when (values.nil? and fields.nil?)
+        "\n"
+      when (values.nil? and not fields.nil?)
+        "\t" << ([""] * fields.length) * "\t" << "\n"
+      when (not Array === values)
+        "\t" << values.to_s << "\n"
+      else
+        "\t" << values.collect{|v| Array === v ? v * "|" : v} * "\t" << "\n"
+      end
+  end
+
+  def to_s(keys = nil, no_options = false)
+    if FalseClass === keys or TrueClass === keys
+      no_options = keys
+      keys = nil
+    end
+
+    if keys == :sort
+      keys = self.keys.sort
+    end
+
+    str = ""
+
+    str << "#: " << Misc.hash2string(ENTRIES.collect{|key| [key.to_sym, self.send(key)]}) << "\n" unless no_options
+    if fields
+      str << "#" << key_field << "\t" << fields * "\t" << "\n"
+    end
+
+    saved_unnamed = @unnamed
+    @unnamed = false
+    if keys.nil?
+      each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup 
+        str << values_to_s(values)
+      end
+    else
+      keys.zip(values_at(*keys)).each do |key, values|
+        key = key.to_s if Symbol === key
+        str << key.dup << values_to_s(values)
+      end
+    end
+
+    @unnamed = saved_unnamed
+    str
+  end
+
+  def value_peek
+    peek = {}
+    keys[0..10].zip(values[0..10]).each do |k,v| peek[k] = v end
+    peek
+  end
+end
+

data/lib/rbbt/tsv/attach.rb

@@ -0,0 +1,183 @@
+require 'rbbt/tsv'
+require 'rbbt/tsv/attach/util'
+module TSV
+
+  # Merge columns from different rows of a file
+  def self.merge_row_fields(input, output, sep = "\t")
+    is = case
+         when (String === input and not input.index("\n") and input.length < 250 and File.exists?(input))
+           CMD.cmd("sort -k1,1 -t'#{sep}' #{ input } | grep -v '^#{sep}' ", :pipe => true)
+         when (String === input or StringIO === input)
+           CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => input, :pipe => true)
+         else
+           input
+         end
+ 
+    current_key  = nil
+    current_parts = []
+
+    done = false
+    Open.write(output) do |os|
+
+      done = is.eof?
+      while not done
+        key, *parts = is.gets.sub("\n",'').split(sep, -1)
+        current_key ||= key
+        case
+        when key.nil?
+        when current_key == key
+          parts.each_with_index do |part,i|
+            if current_parts[i].nil?
+              current_parts[i] = part
+            else
+              current_parts[i] = current_parts[i] << "|" << part
+            end
+          end
+        when current_key != key
+          os.puts [current_key, current_parts].flatten * sep
+          current_key = key
+          current_parts = parts
+        end
+
+        done = is.eof?
+      end
+
+    end
+  end
+
+  # Merge two files with the same keys and different fields
+  def self.merge_different_fields(file1, file2, output, sep = "\t")
+    case
+    when (String === file1 and not file1.index("\n") and file1.length < 250 and File.exists?(file1))
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file1 } | grep -v '^#{sep}' ", :pipe => true)
+    when (String === file1 or StringIO === file1)
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1, :pipe => true)
+    when TSV === file1
+      file1 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file1.to_s(:sort, true), :pipe => true)
+    end
+
+    case
+    when (String === file2 and not file2.index("\n") and file2.length < 250 and File.exists?(file2))
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' #{ file2 } | grep -v '^#{sep}' ", :pipe => true)
+    when (String === file2 or StringIO === file2)
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2, :pipe => true)
+    when TSV === file2
+      file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2.to_s(:sort, true), :pipe => true)
+    end
+
+    output = File.open(output, 'w') if String === output
+
+    cols1 = nil
+    cols2 = nil
+
+    done1 = false
+    done2 = false
+
+    key1 = key2 = nil
+    while key1.nil?
+      while (line1 = file1.gets) =~ /#/; end
+      key1, *parts1 = line1.sub("\n",'').split(sep, -1)
+      cols1 = parts1.length
+    end
+
+    while key2.nil?
+      while (line2 = file2.gets) =~ /#/; end
+      key2, *parts2 = line2.sub("\n",'').split(sep, -1)
+      cols2 = parts2.length
+    end
+
+    key = key1 < key2 ? key1 : key2
+    parts = [""] * (cols1 + cols2)
+    while not (done1 and done2)
+      while (not done1 and key1 == key)
+        parts1.each_with_index do |part, i|
+          parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
+        end
+        key1 = nil
+        while key1.nil? and not done1
+          if file1.eof?; done1 = true; else key1, *parts1 = file1.gets.sub("\n",'').split(sep, -1) end
+        end
+      end
+      while (not done2 and key2 == key)
+        parts2.each_with_index do |part, i|
+          i += cols1
+          parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
+        end
+        key2 = nil
+        while key2.nil? and not done2
+          if file2.eof?; done2 = true; else key2, *parts2 = file2.gets.sub("\n",'').split(sep, -1) end
+        end
+      end
+
+      output.puts [key, parts].flatten * sep
+      parts = [""] * (cols1 + cols2)
+
+      case
+      when done1
+        key = key2
+      when done2
+        key = key1
+      else
+        key = key1 < key2 ? key1 : key2
+      end
+    end
+
+    output.close
+  end
+
+  # Merge columns from different files
+  def self.merge_paste(files, delim = "$")
+    CMD.cmd("paste #{ files.collect{|f| "'#{f}'"} * " "} -d'#{delim}' |sed 's/#{delim}[^\\t]*//g'", :pipe => true)
+  end
+
+  def attach(other, options = {})
+    options      = Misc.add_defaults options, :in_namespace => false, :persist_input => true
+    fields, one2one = Misc.process_options options, :fields, :one2one
+    in_namespace = options[:in_namespace]
+
+    fields = other.fields - [key_field].concat(self.fields) if fields.nil?  or fields == :all 
+    if in_namespace
+      fields = other.fields_in_namespace - [key_field].concat(self.fields) if fields.nil?
+    else
+      fields = other.fields - [key_field].concat(self.fields) if fields.nil?
+    end
+
+    Log.medium("Attaching fields:#{fields.inspect} from #{other.filename.inspect}.")
+
+    other = other.tsv(:persist => options[:persist_input] == true) unless TSV === other 
+    case
+    when key_field == other.key_field
+      attach_same_key other, fields
+    when (not in_namespace and self.fields.include?(other.key_field))
+      Log.debug "Found other's key field: #{other.key_field}"
+      attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
+    when (in_namespace and self.fields_in_namespace.include?(other.key_field))
+      Log.debug "Found other's key field in #{in_namespace}: #{other.key_field}"
+      attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
+    else
+      index = TSV.find_traversal(self, other, options)
+      raise "Cannot traverse identifiers" if index.nil?
+      attach_index other, index, fields
+    end
+    Log.debug("Attachment of fields:#{fields.inspect} from #{other.filename.inspect} finished.")
+
+    self
+  end
+
+  def detach(file)
+    file_fields = file.fields.collect{|field| field.fullname}
+    detached_fields = []
+    self.fields.each_with_index{|field,i| detached_fields << i if file_fields.include? field.fullname}
+    reorder :key, detached_fields
+  end
+
+  def merge_different_fields(other, options = {})
+    TmpFile.with_file do |output|
+      TSV.merge_different_fields(self, other, output, options[:sep] || "\t")
+      tsv = TSV.open output, options
+      tsv.key_field = self.key_field unless self.key_field.nil?
+      tsv.fields = self.fields + other.fields unless self.fields.nil? or other.fields.nil?
+      tsv
+    end
+  end
+end