rbbt-util 3.2.1 → 4.0.0

data/lib/rbbt/tsv/manipulate.rb

@@ -0,0 +1,361 @@
+require 'progress-bar'
+require 'rbbt/persist'
+require 'rbbt/tsv/util'
+
+module TSV
+  
+  attr_accessor :monitor
+
+  class Traverser
+    attr_accessor :new_key_field, :new_fields, :new_key_field_name, :new_field_names, :type, :uniq
+
+    def process_null(key, values)
+      [[key], values]
+    end
+
+    def process_subset_list(key, values)
+      [key, @new_fields.collect{|field| field == :key ?  key : values[field] }]
+    end
+
+    def process_subset_all_but_list(key, values)
+      new = values.dup
+      new.delete_at(0 - @new_fields)
+      [key, new]
+    end
+
+    def process_reorder_list(key, values)
+      [ [values[@new_key_field]], 
+        @new_fields.collect{|field| field == :key ?  key : values[field] }]
+    end
+
+    def process_reorder_double_uniq(key, values)
+      [ values[@new_key_field].uniq, 
+        @new_fields.collect{|field| field == :key ?  
+          [key] : values[field] }
+      ]
+    end
+
+    def process_subset_double(key, values)
+      [[key], @new_fields.collect{|field| field == :key ?  [key] : values[field] }]
+    end
+
+    def process_subset_all_but_double(key, values)
+      new = values.dup
+      new.delete_at(0 - @new_fields)
+      [[key], new]
+    end
+
+    def process_reorder_double(key, values)
+      [ values[@new_key_field], 
+        @new_fields.collect{|field| field == :key ?  
+          [key] : values[field] }
+      ]
+    end
+
+    def initialize(key_field, fields, new_key_field, new_fields, type, uniq)
+      @new_key_field = TSV.identify_field(key_field, fields, new_key_field)
+
+      raise "Key field #{ new_key_field } not found" if @new_key_field.nil?
+      @new_fields = case
+                   when new_fields.nil?
+                     case 
+                     when @new_key_field == :key
+                       :all
+                     when fields.nil?
+                       - @new_key_field
+                     else 
+                       new = (0..fields.length - 1).to_a
+                       new.delete_at(@new_key_field)
+                       new.unshift :key
+                       new
+                     end
+                   else
+                     if Array === new_fields
+                       new_fields.collect do |field|
+                         TSV.identify_field(key_field, fields, field)
+                       end
+                     else
+                       [TSV.identify_field(key_field, fields, new_fields)]
+                     end
+                   end
+
+      @new_key_field_name = case 
+                            when @new_key_field == :key
+                              key_field
+                            else
+                              fields[@new_key_field] if Array === fields
+                            end
+
+      if Array === fields
+        @new_field_names = case
+                           when fields.nil?
+                             nil
+                           when Array === @new_fields
+                             @new_field_names = @new_fields.collect do |field|
+                               case 
+                               when field == :key
+                                 key_field
+                               else
+                                 fields[field]
+                               end
+                             end
+                           when @new_fields == :all
+                             fields
+                           when (Fixnum === @new_fields and @new_fields <= 0)
+                             new = fields.dup
+                             new.delete_at(- @new_fields)
+                             new.unshift key_field
+                             new
+                           end
+      end
+
+      case
+      when (@new_key_field == :key and (@new_fields == :all or fields.nil? or @new_fields == (0..fields.length - 1).to_a))
+        self.instance_eval do alias process process_null end
+      when @new_key_field == :key 
+        if type == :double
+          if Fixnum === @new_fields and @new_fields <= 0
+            self.instance_eval do alias process process_subset_all_but_double end
+          else
+            self.instance_eval do alias process process_subset_double end
+          end
+        else
+          if Fixnum === @new_fields and @new_fields <= 0
+            self.instance_eval do alias process process_subset_all_but_list end
+          else
+            self.instance_eval do alias process process_subset_list end
+          end
+        end
+      else
+        if type == :double
+          if uniq
+            self.instance_eval do alias process process_reorder_double_uniq end
+          else
+            self.instance_eval do alias process process_reorder_double end
+          end
+        else
+          self.instance_eval do alias process process_reorder_list end
+        end
+      end
+    end
+
+  end
+
+  #{{{ Methods
+
+  def through(new_key_field = nil, new_fields = nil, uniq = false)
+
+    traverser = Traverser.new @key_field, @fields, new_key_field, new_fields, type, uniq
+
+    if @monitor
+      desc = "Iterating TSV"
+      step = 100
+      if Hash === @monitor
+        desc = @monitor[:desc] if @monitor.include? :desc 
+        step = @monitor[:step] if @monitor.include? :step 
+      end
+      progress_monitor = Progress::Bar.new(size, 0, step, desc)
+    else
+      progress_monitor = nil
+    end
+
+    each do |key, value|
+      progress_monitor.tick if progress_monitor
+
+      keys, value = traverser.process(key, value)
+      next if keys.nil?
+      keys.each do |key|
+        NamedArray.setup value, traverser.new_field_names if Array === value and not @unnamed
+        yield key, value
+      end
+    end
+
+    [traverser.new_key_field_name, traverser.new_field_names]
+  end
+
+  def reorder(new_key_field = nil, new_fields = nil, persist = false)
+    Persist.persist_tsv self, self.filename, {:key_field => new_key_field, :fields => new_fields}, {:persist => persist, :persist_prefix => "Reorder:"} do |data|
+
+      with_unnamed do
+        new_key_field_name, new_field_names = through new_key_field, new_fields do |key, value|
+          if data.include?(key) and type == :double
+            data[key] = data[key].zip(value).collect do |old_list, new_list| old_list + new_list end
+          else
+            data[key] = value
+          end
+        end
+
+        data.extend TSV unless TSV === data
+        data.key_field = new_key_field_name
+        data.fields = new_field_names
+        data.filename = filename
+        data.type = type
+      end
+    end
+  end
+
+  def slice(fields)
+    reorder :key, fields
+  end
+
+  def sort(*fields)
+    fields = nil if fields.empty?
+
+    elems = []
+    through :key, fields do |key, value|
+      elems << case
+      when block_given?
+        [key, yield(*value)]
+      else
+        case
+        when type == :single
+          [key, value]
+        when type == :double
+          [key, value.first.first]
+        else
+          [key, value.first]
+        end
+      end
+    end
+
+    elems.sort_by{|k,v| v}.collect{|k,v| k}
+  end
+
+  def select(method = nil)
+    new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
+
+    new.key_field = key_field
+    new.fields    = fields.dup
+    new.type      = type
+    new.filename  = filename
+    
+   case
+    when (method.nil? and block_given?)
+      through do |key, values|
+        new[key] = values if yield key, values
+      end
+    when Array === method
+      with_unnamed do
+        through do |key, values|
+          new[key] = values if ([key,values].flatten & method).any?
+        end
+      end
+    when Regexp === method
+      with_unnamed do
+        through do |key, values|
+          new[key] = values if [key,values].flatten.select{|v| v =~ method}.any?
+        end
+      end
+    when String === method
+      if block_given?
+        pos = identify_field method
+        with_unnamed do
+          through do |key, values|
+            new[key] = values if yield((method == key_field or method == :key)? key : values[pos])
+          end
+        end
+      else
+        with_unnamed do
+          through do |key, values|
+            new[key] = values if [key,values].flatten.select{|v| v == method}.any?
+          end
+        end
+      end
+    when Hash === method
+      key  = method.keys.first
+      method = method.values.first
+      case
+      when (Array === method and (key == :key or key_field == key))
+        with_unnamed do
+          method.each{|key| 
+            new[key] = self[key] if self.include? key
+          }
+        end
+      when Array === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if (values.flatten & method).any?
+          end
+        end
+      when Regexp === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if values.flatten.select{|v| v =~ method}.any?
+          end
+        end
+      when String === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if values.flatten.select{|v| v == method}.any?
+          end
+        end
+      when Proc === method
+        with_unnamed do
+          through :key, key do |key, values|
+            values = [values] if type == :single
+            new[key] = self[key] if values.flatten.select{|v| method.call(v)}.any?
+          end
+        end
+      end
+ 
+    end
+
+    new
+  end
+
+  def process(field, &block)
+    field_pos = identify_field field
+
+    through do |key, values|
+      next if values.nil?
+
+      case
+      when type == :single
+        field_values = values
+      when type == :flat
+        field_values = values
+      else
+        next if values[field_pos].nil? 
+        field_values = values[field_pos]
+      end
+
+      new_values = case 
+                   when block.arity == 1
+                     yield(field_values)
+                   when block.arity == 2
+                     yield(field_values, key)
+                   when block.arity == 3
+                     yield(field_values, key, values)
+                   else
+                     raise "Unexpected arity in block, must be 1, 2 or 3: #{block.arity}"
+                   end
+
+      case
+      when type == :single
+        self[key] = new_values
+      when type == :flat
+        self[key] = new_values
+      else
+        values[field].replace new_values
+        self[key] = values
+      end
+    end
+  end
+
+  def add_field(name = nil)
+    through do |key, values|
+      new_values = yield(key, values)
+      new_values = [new_values] if type == :double and not Array === new_values
+
+      values << new_values
+      self[key] = values
+    end
+
+    self.fields = self.fields + [name] if fields != nil and name != nil
+
+    self
+  end
+end

data/lib/rbbt/tsv/parser.rb

@@ -0,0 +1,231 @@
+require 'rbbt/util/cmd'
+module TSV
+  class Parser
+    attr_accessor :header_hash, :sep, :sep2, :type, :key_position, :field_positions, :cast, :key_field, :fields, :fix, :select, :serializer, :straight
+
+    class SKIP_LINE < Exception; end
+
+    def all_fields
+      all = [key_field] + fields
+      NamedArray.setup all, all
+    end
+
+    def parse_header(stream)
+      options = {}
+
+      # Get line
+
+      line = stream.gets
+      raise "Empty content" if line.nil?
+      line.chomp!
+
+      # Process options line
+
+      if line and line =~ /^#{@header_hash}: (.*)/
+        options = Misc.string2hash $1
+        line = stream.gets
+      end
+
+      # Determine separator
+
+      @sep = options[:sep] if options[:sep]
+
+      # Process fields line
+
+      if line and line =~ /^#{@header_hash}/
+        line.chomp!
+        @fields = line.split(@sep)
+        @key_field = @fields.shift
+        @key_field = @key_field[(0 + header_hash.length)..-1] # Remove initial hash character
+        line = stream.gets
+      end
+
+      @first_line = line
+
+      options
+    end
+
+    def process(line)
+      l = line.chomp
+      raise Parser::SKIP_LINE if Proc === @select and not @select.call l
+      l = @fix.call l if Proc === @fix
+      l
+    end
+
+    def cast?
+      !! @cast
+    end
+
+    def chop_line(line)
+      line.split(@sep, -1)
+    end
+
+    def get_values_single(parts)
+      return parts.shift, parts.first if field_positions.nil?
+      key = parts[key_position]
+      value = parts[field_positions.first]
+      [key, value]
+    end
+
+    def get_values_list(parts)
+      return parts.shift, parts if field_positions.nil?
+      key = parts[key_position]
+      values = parts.values_at *field_positions
+      [key, values]
+    end
+
+    def get_values_double(parts)
+      return parts.shift.split(@sep2, -1), parts.collect{|value| value.split(@sep2, -1)} if field_positions.nil?
+      keys = parts[key_position].split(@sep2, -1)
+      values = parts.values_at(*field_positions).collect{|value| value.split(@sep2, -1)}
+      [keys, values]
+    end
+
+    def add_to_data_no_merge_list(data, key, values)
+      data[key] = values unless data.include? key
+    end
+
+    def add_to_data_no_merge_double(data, keys, values)
+      keys.each do |key|
+        data[key] = values unless data.include? key
+      end
+    end
+
+    def add_to_data_merge(data, keys, values)
+      keys.each do |key|
+        if data.include? key
+          data[key] = data[key].zip(values).collect do |old, new|
+            old.concat new
+            old
+          end
+        else
+          data[key] = values
+        end
+      end
+    end
+
+    def cast_values_single(value)
+      case
+      when Symbol === cast
+        value.send(cast)
+      when Proc === cast
+        cast.call value
+      end
+    end
+
+    def cast_values_list(values)
+      case
+      when Symbol === cast
+        values.collect{|v| v.send(cast)}
+      when Proc === cast
+        values.collect{|v| cast.call v}
+      end
+    end
+
+    def cast_values_double(values)
+      case
+      when Symbol === cast
+        values.collect{|list| list.collect{|v| v.send(cast)}}
+      when Proc === cast
+        values.collect{|list| list.collect{|v| cast.call v }}
+      end
+    end
+
+    def rescue_first_line
+      @first_line
+    end
+
+    def fix_fields(options)
+      key_field = Misc.process_options options, :key_field
+      fields    = Misc.process_options options, :fields
+
+      if (key_field.nil? or key_field == 0 or key_field == :key) and
+        (fields.nil? or fields == @fields or (not @fields.nil? and fields == (1..@fields.length).to_a))
+
+        @straight = true
+        return
+      else
+        @straight = false
+
+        case
+        when (key_field.nil? or key_field == @key_field or key_field == 0)
+          @key_position = 0
+        when Integer === key_field
+          @key_position = key_field
+        when String === key_field
+          @key_position = @fields.dup.unshift(@key_field).index key_field
+        else
+          raise "Format of key_field not understood: #{key_field.inspect}"
+        end
+
+        if (fields.nil? or fields == @fields or (not @fields.nil? and fields == (1..@fields.length).to_a))
+          @field_positions = (0..@fields.length).to_a
+          @field_positions.delete @key_position
+        else
+          fields = [fields] if not Array === fields
+          @field_positions = fields.collect{|field|
+            case
+            when Integer === field
+              field
+            when String === field
+              @fields.dup.unshift(@key_field).index field
+            else
+              raise "Format of fields not understood: #{fields.inspect}"
+            end
+          }
+        end
+
+        new_key_field = @fields.dup.unshift(@key_field)[@key_position] if not @fields.nil?
+        @fields = @fields.dup.unshift(@key_field).values_at *@field_positions if not @fields.nil?
+        @key_field = new_key_field
+      end
+    end
+
+    def initialize(stream = nil, options = {})
+      @header_hash = Misc.process_options(options, :header_hash) || "#"
+      @sep = Misc.process_options(options, :sep) || "\t"
+
+      options = parse_header(stream).merge options
+
+      @type = Misc.process_options(options, :type) || :double
+      merge = Misc.process_options(options, :merge) || false
+
+      @sep2 = Misc.process_options(options, :sep2) || "|"
+      @cast = Misc.process_options options, :cast
+      @type ||= Misc.process_options options, :type
+      @fix = Misc.process_options(options, :fix) 
+      @select= Misc.process_options options, :select
+
+      if @type == :double
+        self.instance_eval do alias get_values get_values_double end
+        self.instance_eval do alias cast_values cast_values_double end
+        if merge
+          self.instance_eval do alias add_to_data add_to_data_merge end
+        else
+          self.instance_eval do alias add_to_data add_to_data_no_merge_double end
+        end
+      else
+        if @type == :single
+          self.instance_eval do alias get_values get_values_single end
+          self.instance_eval do alias cast_values cast_values_single end
+        else
+          self.instance_eval do alias get_values get_values_list end
+          self.instance_eval do alias cast_values cast_values_list end
+        end
+        self.instance_eval do alias add_to_data add_to_data_no_merge_list end
+      end
+
+      fix_fields(options)
+
+      @straight = false if @sep != "\t" or not @cast.nil? or merge
+    end
+
+    def setup(data)
+      data.extend TSV unless TSV === data
+      data.type = @type
+      data.key_field = @key_field
+      data.fields = @fields
+      data
+    end
+  end
+end