carray-dataframe 1.0.0 → 1.1.1

data/lib/carray-dataframe/dataframe.rb

@@ -0,0 +1,1279 @@
+require "carray"
+require "carray/table"
+
+def CADataFrame (*argv)
+  return CADataFrame.new(*argv)
+end
+
+class CADataFrame
+
+  #
+  # Constructor
+  #
+  def initialize (data, index: nil, columns: nil, order: nil, clone: false, &block)
+    # @column_names = Array holds column names and its order
+    # @column_data  = Hash holds data entities
+    # @row_number   = Integer holds number of rows
+    # @row_index    = CArray stores row index (any object)
+    # @__methods__  = ...
+    # Stores data entity
+    
+    case data
+    when Hash
+      raise "columns option is not needed for hash data" if columns
+      @column_data  = columns_to_columns(data)
+      @column_names = @column_data.keys
+    when CArray
+      if columns
+        @column_names = columns.map(&:to_s)
+      else
+        if data.respond_to?(:column_names)
+          @column_names = data.column_names.map(&:to_s)
+        elsif order
+          @column_names = order.map(&:to_s)
+        else
+          raise "can't determin column names use columns or order option"
+        end
+      end
+      if @column_names.size != data.dim1
+        raise "mismatch between 'column_names' and table columns"
+      end
+      @column_data = table_to_columns(data)
+    when Array
+      case data.first
+      when Hash
+        @column_data = {}
+        dummy = {}
+        data.each do |hash|
+          dummy.update(hash)
+        end
+        @column_names = []
+        dummy.each_key do |k|
+          list = []
+          data.each do |hash|
+            list << (hash[k] || UNDEF)
+          end
+          name = k.to_s
+          @column_names << name
+          @column_data[name] = list.to_ca
+        end
+      else
+        if columns
+          @column_names = columns.map(&:to_s)
+        elsif order
+          @column_names = order.map(&:to_s)
+        else
+          raise "columns or order option should be given"
+        end
+        @column_data = array_to_columns(data)
+      end
+    else
+      raise "unknown data"
+    end
+    
+    if order
+      if @column_names.size != order.size
+        raise 'invalid order option'
+      end
+      new_column_data = {}
+      order.each do |key|
+        if @column_data.has_key?(key.to_s)
+          new_column_data[key.to_s] = @column_data[key.to_s]
+        else
+          raise 'invalid column name '#{key.to_s}' in order option'
+        end
+      end
+      @column_data  = new_column_data
+      @column_names = new_column_data.keys
+    end
+
+    # Sets @row_number and check column length
+    @row_number   = @column_data.first[1].size
+    if @column_names.any?{ |key| @column_data[key].size != @row_number }
+      raise "column sizes mismatch"
+    end
+
+    # Processing option 'index'
+    set_index(index, inplace: true)
+    @__methods__ = {}
+
+    if clone
+      raise NotImplementedError, "copy option is not implemented"
+    end
+
+    if block_given?
+      arrange(&block)
+    end
+  end
+  
+  attr_reader :column_data, :column_names, :row_index, :row_number
+
+  def columns
+    @column_data
+  end
+
+  def __methods__
+    return @__methods__
+  end
+
+  def table_to_columns (table)
+    new_columns = {}
+    @column_names.each_with_index do |name, i|
+      new_columns[name.to_s] = table[nil,i].to_ca
+    end
+    return new_columns
+  end
+
+  private :table_to_columns
+
+  def columns_to_columns (columns)
+    new_columns = {}
+    row_number = []
+    columns.each_with_index do |(key, col), k|
+      case col
+      when CArray
+        column = col
+        column = column.flatten unless column.rank == 1
+      when Array
+        column = col.to_ca
+        unless column.rank == 1
+          list = col.clone
+          column = CArray.object(list.size).convert { list.shift }
+        end
+      else
+        begin
+          column = col.to_ca
+          column = column.flatten unless column.rank == 1
+        rescue
+          raise "#{k}-th column can't be converted to CArray"
+        end
+      end
+      if key == ""
+        @row_index = column
+      else
+        new_columns[key.to_s] = column
+      end
+    end
+    return new_columns
+  end
+
+  private :columns_to_columns
+
+  def array_to_columns (array)
+    new_columns = {}
+    case array.first
+    when CArray
+      if @column_names.size != data.size
+        raise "mismatch between 'columns' and table columns"
+      end
+      @column_names.each_with_index do |key, k|
+        column = array[k]
+        column = column.flatten unless column.rank == 1
+        new_columns[key.to_s] = column
+      end
+    when Array
+      table = array.transpose
+      @column_names.each_with_index do |key, k|
+        new_columns[key] = table[k].to_ca
+      end
+    else
+      raise "invalid array content for CADataFrame"
+    end
+    return new_columns
+  end
+  
+  private :array_to_columns
+
+  def set_index (index, drop: true, inplace: false)
+    if inplace
+      case index
+      when nil
+      when String, Symbol
+        index = index.to_s
+        raise "can't find column named '#{index}'" unless @column_names.include?(index)
+        if drop
+          @row_index = @column_data.delete(index)
+          @column_names.delete(index)
+        else
+          @row_index = @column_data[index]
+        end
+      else
+        @row_index = index.to_ca
+      end
+      self
+    else
+      return to_df.set_index(index, drop: drop, inplace: true)
+    end
+  end
+
+  def replace (other)
+    @column_names = other.column_names
+    @column_data  = other.column_data
+    @row_index    = other.row_index
+    @row_number   = other.row_number
+    @__methods__  = other.__methods__
+    return self
+  end
+
+  def has_column?(name)
+    return @column_names.include?(name)
+  end
+
+  def column_types
+    return @column_names.map{|name| @column_data[name].data_type_name }
+  end
+
+  #
+  # Column, Row Access
+  #
+
+  def column (spec)
+    case spec
+    when Integer
+      return @column_data[@column_names[spec]]
+    when String, Symbol
+      return @column_data[spec.to_s]
+    else
+      raise "invalid column specifier"
+    end
+  end
+  alias col column
+
+  def loc
+    @loc ||= CADataFrame::LocAccessor.new(self)
+    return @loc 
+  end
+
+  def iloc (&block)
+    @iloc ||= CADataFrame::ILocAccessor.new(self)
+    return @iloc 
+  end
+
+  # TO BE FIXED
+  def index
+    if @row_index
+      return @row_index.to_ca
+    else
+      return CArray.int(@row_number).seq
+    end
+  end
+
+  def head (n=10)
+    rmax = [@row_number, n].min
+    return row[0..rmax-1]
+  end
+
+  def tail (n=10)
+    rmin = -([@row_number, n].min)
+    return row[rmin..-1]
+  end
+
+  def method (hash)
+    new_hash = {}
+    hash.each do |key, value|
+      new_hash[key.to_s] = value.to_s
+    end
+    @__methods__.update(new_hash)
+  end
+
+  def method_missing (name, *args)
+    if args.size == 0
+      name = name.to_s
+      if has_column?(name)
+        return @column_data[name]
+      elsif has_column?(name.gsub(/_/,'.')) ### For R
+        return @column_data[name.gsub(/_/,'.')]
+      elsif @__methods__.include?(name)
+        return @column_data[@__methods__[name]]
+      end
+    end
+    raise "no method '#{name}' for CADataFrame"
+  end
+
+  #
+  # Iterators
+  #
+
+  def each_column (&block)
+    return @column_data.each(&block)
+  end
+
+  def each_column_name (&block)
+    return @column_names.each(&block)
+  end
+
+  def each_row_index (&block)
+    if @row_index
+      @row_index.each(&block)
+    else
+      @row_number.times(&block)
+    end
+  end
+
+  def each_row (with: Array, columns: nil, &block)
+    case columns
+    when Array
+      column_names = columns
+    when Regexp
+      column_names = @column_names.grep(columns)
+    else
+      column_names = @column_names
+    end
+    if with == Array
+      @row_number.times do |i|
+        yield column_names.map{|n| @column_data[n][i] }
+      end
+    elsif with == Hash
+      row = {}
+      @row_number.times do |i|
+        column_names.each do |c|
+          row[c] = @column_data[c][i]
+        end
+        yield row
+      end
+    elsif with == CArray
+      joined = CArray.join(@column_data.values_at(*column_names))
+      joined[:i,nil].each do |block|
+        yield block.to_ca.compact
+      end
+    else
+      raise "invalid data type for loop variable"
+    end
+  end
+
+  def each_row_with_row_index (with: Array, &block)
+    if with == Array
+      if @row_index
+        @row_index.each_with_index do |idx, i|
+          yield @column_data.map{|n,c| c[i] }, idx
+        end
+      else
+        @row_number.times do |i|
+          yield @column_data.map{|n,c| c[i] }, i
+        end
+      end
+    elsif with == Hash
+      row = {}
+      if @row_index
+        @row_index.each_with_index do |idx, i|
+          @column_names.each do |c|
+            row[c] = @column_data[c][i]
+          end
+          yield row, @row_index[i]
+        end
+      else
+        @row_number.times do |idx, i|
+          @column_names.each do |c|
+            row[c] = @column_data[c][i]
+          end
+          yield row, @row_index[i]
+        end
+      end
+    else
+      raise "invalid  data type for loop variable"
+    end
+  end
+
+  def where (mask, *args)
+    mask.column_names.each do |key|
+      if has_column?(key)
+        case args.size
+        when 1
+          column(key)[mask.column(key).boolean.not] = args[0]
+        when 2
+          column(key)[mask.column(key).boolean.not] = args[0]
+          column(key)[mask.column(key).boolean] = args[1]
+        end
+      end
+    end
+  end
+
+  def fill (*names, value)
+    names.each do |name|
+      if has_column?(name)
+        column(name).fill(value)
+      end
+    end
+    return self
+  end
+
+  #
+  # Arrange
+  #
+
+  def arrange (&block)
+    return Arranger.new(self).arrange(&block)
+  end
+
+  def rename (name1, name2)
+    if idx = @column_names.index(name1.to_s)
+      @column_names[idx] = name2.to_s
+      column = @column_data[name1.to_s]
+      @column_data.delete(name1.to_s)
+      @column_data[name2.to_s] = column
+    else
+      raise "unknown column name #{name1}"
+    end
+  end
+
+  def downcase
+    new_column_names = []
+    new_columns = {}
+    each_column_name do |name|
+      new_column_names << name.downcase
+      new_columns[name.downcase] = @column_data[name]
+    end
+    @column_names = new_column_names
+    @column_data = new_columns
+    return self
+  end
+
+  def append_column (name, new_column = nil, &block)
+    if new_column
+      # do nothing
+    elsif block
+      new_column = instance_exec(&block)
+    else
+      new_column = @column_data.first[1].template(:object)
+    end
+    unless new_column.is_a?(CArray)
+      new_column = new_column.to_ca
+    end
+    if new_column.rank != 1 or new_column.size != @row_number
+      raise "invalid shape of appended column"
+    end
+    @column_names.push(name.to_s)
+    @column_data[name.to_s] = new_column
+    return new_column
+  end
+
+  alias append append_column
+
+  def prepend_column (name, new_column = nil, &block)
+    if new_column
+      # do nothing
+    elsif block
+      new_column = instance_exec(&block)
+    else
+      new_column = @column_data.first[1].template(:object)
+    end
+    unless new_column.is_a?(CArray)
+      new_column = new_column.to_ca
+    end
+    if new_column.rank != 1 or new_column.size != @row_number
+      raise "invalid shape of appended column"
+    end
+    @column_names.unshift(name.to_s)
+    @column_data[name.to_s] = new_column
+    return new_column
+  end
+
+  alias lead prepend_column
+
+  def drop_column (*columns)
+    if columns.empty?
+      return self
+    else
+      names = []
+      columns.each do |c|
+        case c
+        when String
+          names << c
+        when Symbol
+          names << c.to_s
+        when Regexp
+          names.push *@column_names.grep(c)
+        else
+          raise "invalid column specification"
+        end
+      end
+    end
+    new_columns = {}
+    each_column_name do |name|
+      unless names.include?(name)
+        new_columns[name] = column(name)
+      end
+    end
+    return replace CADataFrame.new(new_columns, index: @row_index)
+  end
+
+  alias eliminate_column drop_column
+
+  def vacant_copy
+    new_columns = {}
+    each_column_name do |key|
+      new_columns[key] = CArray.object(0)
+    end
+    return CADataFrame.new(new_columns)
+  end
+
+  def merge (*args)
+    return CADataFrame.merge(self, *args)
+  end
+
+  def execute (&block)
+    case block.arity
+    when 1
+      return instance_exec(self, &block)
+    else
+      return instance_exec(&block)
+    end
+  end
+
+  def calculate (label, &block)
+    hash = {}
+    each_column_name do |name|
+      begin
+        if block
+          hash[name] = [yield(name, column(name))]
+        else
+          hash[name] = [column(name).send(label.intern)]
+        end
+      rescue
+        hash[name] = [UNDEF]
+      end
+    end
+    return CADataFrame.new(hash, index: [label])
+  end
+
+  def resample (&block)
+    new_columns = {}
+    each_column_name do |name|
+      begin
+        new_columns[name] = yield(name, column(name))
+      rescue
+      end
+    end
+    return CADataFrame.new(new_columns)
+  end
+
+  def select (*columns, &block)
+    if columns.empty?
+      names = @column_names
+    else
+      names = []
+      columns.each do |c|
+        case c
+        when String
+          names << c
+        when Symbol
+          names << c.to_s
+        when Regexp
+          names.push *@column_names.grep(c)
+        else
+          raise "invalid column specification"
+        end
+      end
+    end
+    if block
+      row = instance_exec(&block)
+    else
+      row = nil
+    end
+    new_columns = {}
+    names.map(&:to_s).each do |name|
+      new_columns[name] = column(name)[row]
+    end
+    return CADataFrame.new(new_columns, index: @row_index ? @row_index[row] : nil)
+  end
+  #
+  # Maintenance
+  #
+
+  def unmask! (value = nil)
+    each_column_name do |name|
+      column(name).unmask(value)
+    end
+    return self
+  end
+
+  def unmask (value = nil)
+    return to_df.unmask!(value)
+  end
+
+  def detouch
+    @column_data = @column_data.clone
+    each_column_name do |name|
+      @column_data[name] = @column_data[name].to_ca
+    end
+    if @row_index
+      @row_index = @row_index.clone
+    end
+    return self
+  end
+
+  def delete_masked_rows
+    not_masked = @column_data.first[1].template(:boolean) { true }
+    @column_names.each do |name|
+      not_masked &= @column_data[name].is_not_masked
+    end
+    columns = {}
+    @column_names.each_with_index do |name, i|
+      columns[name] = @column_data[name].to_ca[not_masked]
+    end
+    return CADataFrame.new(columns)
+  end
+
+  def delete_rows (&block)
+    masked = instance_eval(&block)
+    columns = {}
+    @column_names.each_with_index do |name, i|
+      columns[name] = @column_data[name].to_ca[masked]
+    end
+    return CADataFrame.new(columns)
+  end
+  #
+  # Transformation
+  #
+
+  def reorder (&block)
+    index = instance_exec(&block)
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name)[index]
+    end
+    return CADataFrame.new(new_columns, index: @row_index ? @row_index[index] : nil)
+  end
+
+  def order_by (*names, &block)
+    if names.empty?
+      if block
+        ret = instance_exec(&block)
+        case ret
+        when CArray
+          list = [ret]
+        when Array
+          list = ret
+        end
+      end
+    else
+      list = @column_data.values_at(*names.map{|s| s.to_s})
+    end
+    return reorder { CA.sort_addr(*list) }
+  end
+
+  def reverse
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name).reverse
+    end
+    return CADataFrame.new(new_columns, index: @row_index ? @row_index.reverse : nil)
+  end
+
+  def transpose (columns: nil)
+    if columns
+      columns = columns.map(&:to_s)
+    else
+      if @row_index
+        columns = @row_index.convert(:object) {|v| v.to_s }
+      else
+        columns = CArray.object(@row_number).seq("a",:succ)
+      end
+    end
+    return CADataFrame.new(ca.transpose, index: @column_names.to_ca, columns: columns)
+  end
+
+  def add_suffix (suf)
+    new_columns = {}
+    each_column_name do |name|
+      new_name = (name.to_s + suf).to_s
+      new_columns[new_name] = column(name)
+    end
+    return CADataFrame.new(new_columns, index: @row_index)
+  end
+  #
+  # Conversions
+  #
+
+  def to_df
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name)
+    end
+    return CADataFrame.new(new_columns, index: @row_index).detouch
+  end
+
+  def objectify
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name).object
+    end
+    return CADataFrame.new(new_columns, index: @row_index)
+  end
+
+  def ca (*names)
+    if names.empty?
+      return CADFArray.new(@column_names, @column_data)
+    else
+      return CADFArray.new(names.map(&:to_s), @column_data)
+    end
+  end
+
+  def to_ca (*names)
+    return ca(*names).to_ca
+  end
+
+  def to_hash
+    hash = {}
+    if @row_index
+      hash["index"] = @row_index
+    end
+    @column_data.each do |k,v|
+      hash[k] = v.to_a
+    end
+    return hash
+  end
+  
+  alias to_h to_hash
+
+  def columns_to_hash (key_name, value_names)
+    hash = {}
+    unless @column_names.include?(key_name)
+      raise ArgumentError, "include invalid key column name #{key_name}"
+    end
+    case value_names
+    when String
+      unless @column_names.include?(value_names)
+        raise ArgumentError, "invalid key column name #{value_names}"
+      end
+      key_columns = @column_data[key_name]
+      value_columns = @column_data[value_names]
+      @row_number.times do |i|
+        hash[key_columns[i]] = value_columns[i]
+      end
+    when Array
+      unless value_names.all?{|s| @column_names.include?(s) }
+        raise ArgumentError, "include invalid column name in #{value_names.join(' ')}"
+      end
+      key_columns = @column_data[key_name]
+      value_columns = @column_data.values_at(*value_names)
+      @row_number.times do |i|
+        hash[key_columns[i]] = value_columns.map{|c| c[i]}
+      end
+    else
+      raise ArgumentError, "invalud argument"
+    end
+    return hash
+  end
+  
+  private
+
+  def __obj_to_string__ (obj)
+    case obj
+    when Float
+      "%.6g" % obj
+    when nil
+      "nil"
+    else
+      obj.to_s
+    end
+  end
+
+  def __strwidth__ (string)
+    if string.ascii_only?
+      return string.length
+    else
+      return string.each_char.inject(0){|s,c| s += c.bytesize > 1 ? 2 : 1 }
+    end
+  end
+  public
+
+  def ascii_table (rowmax = :full, time_format: nil, index: true)
+    columns = @column_data.clone
+    @column_names.each do |name|
+      if columns[name].is_a?(CATimeIndex)
+        if time_format
+          columns[name] = columns[name].time.time_strftime(time_format)
+        else
+          columns[name] = columns[name].time.time_strftime("%F %T%:z")
+        end
+      end
+    end
+    if index
+      if @row_index
+        namelist = ["    "] + @column_names
+        if @row_index.is_a?(CATimeIndex)
+          if time_format
+            row_index = @row_index.time.time_strftime(time_format)
+          else
+            row_index = @row_index.time.time_strftime("%F %T%:z")
+          end
+        else
+          row_index = @row_index
+        end
+        tbl = CADFArray.new(namelist, columns.update("    " => row_index))
+      else
+        namelist = ["    "] + @column_names
+        tbl = CADFArray.new(namelist, columns.update("    " => CArray.int(@row_number).seq))
+      end
+    else
+      namelist = @column_names
+      tbl = CADFArray.new(namelist, columns)
+    end
+    if rowmax.is_a?(Integer) and @row_number > rowmax
+      list = tbl[0..(rowmax/2),nil].to_a
+      list.push namelist.map { "..." }
+      list.push *(tbl[-rowmax/2+1..-1,nil].to_a)
+      tbl = list.to_ca
+    end
+    datastr = tbl.convert {|c| __obj_to_string__(c) }.unmask("")
+    datamb  = datastr.convert(:boolean, &:"ascii_only?").not.sum(0).ne(0)
+    namemb  = namelist.to_ca.convert(:boolean) {|c| c.to_s.ascii_only? }.eq(0)
+    mb      = datamb.or(namemb)
+    namelen = namelist.map(&:length).to_ca
+    datalen = datastr.convert(&:length)
+    if mb.max == 0
+      if datalen.size == 0
+        lengths  = namelen.to_a
+      else
+        lengths  = datalen.max(0).pmax(namelen).to_a
+      end
+      hrule  = "-" + lengths.map {|len| "-"*len}.join("--") + "-"
+      header = " " +
+               [namelist, lengths].transpose.map{|name, len|
+                                            "#{name.to_s.ljust(len)}" }.join("  ") + " "
+      ary = [hrule, header, hrule]
+			if datalen.size > 0
+	      datastr[:i,nil].each_with_index do |blk, i|
+	        list = blk.flatten.to_a
+	        ary << " " + [list, lengths].transpose.map{|value, len|
+	                                           "#{value.ljust(len)}"}.join("  ") + " "
+	      end
+			end
+      ary << hrule
+      return "DataFrame: rows#=#{@row_number}: \n" + ary.join("\n")
+    else
+      namewidth  = namelist.to_ca.convert{|c| __strwidth__(c.to_s) }
+      if datalen.size == 0
+        maxwidth   = namewidth
+      else
+        datawidth  = datastr.convert{|c| __strwidth__(c.to_s) }
+        maxwidth   = datawidth.max(0).pmax(namewidth)
+      end
+      len = maxwidth[:*,nil] - datawidth + datalen
+      hrule  = "-" + maxwidth.map {|len| "-"*len}.join("--") + "-"
+      header = " " +
+               [namelist, maxwidth.to_a].transpose.map{|name, len|
+                                            "#{name.to_s.ljust(len-__strwidth__(name.to_s)+name.to_s.length)}" }.join("  ") + " "
+      ary = [hrule, header, hrule]
+			if datalen.size > 0
+	      datastr[:i,nil].each_with_addr do |blk, i|
+	        list = blk.flatten.to_a
+	        ary << " " + list.map.with_index {|value, j|
+	                  "#{value.ljust(len[i,j])}"}.join("  ") + " "
+	      end
+			end
+      ary << hrule
+      return "DataFrame: row#=#{@row_number}: \n" + ary.join("\n")
+    end
+  end
+
+  def inspect
+    return ascii_table(8)
+  end
+
+  def to_s
+    return ascii_table
+  end
+
+  def to_ary
+    return [to_s]
+  end
+end
+
+#############################################################
+#
+# Class methods
+#
+#############################################################
+class CADataFrame
+
+  def self.merge (*args)
+    ref = args.first
+    new_columns = {}
+    args.each do |table|
+      table.column_names.each do |name|
+        new_columns[name] = table.col(name)
+      end
+    end
+    return CADataFrame.new(new_columns, index: ref.row_index)
+  end
+
+  def self.concat (*args)
+    ref = args.first
+    column_names = ref.column_names
+    new_columns = {}
+    column_names.each do |name|
+      list = args.map{|t| t.column(name) }
+      if list.first.is_a?(CATimeIndex)
+        new_columns[name] = CATimeIndex.concat(*list)
+      else
+        data_type = list.first.data_type
+        new_columns[name] = CArray.bind(data_type, list, 0)
+      end
+    end
+    list = args.map(&:row_index)
+    if list.all?
+      if list.first.is_a?(CATimeIndex)
+        new_row_index = CATimeIndex.concat(*list)
+      else
+        new_row_index = CArray.join(*list).flatten
+      end
+    else
+      new_row_index = nil
+    end
+    return CADataFrame.new(new_columns, index: new_row_index)
+  end
+end
+
+#############################################################
+#
+# BASIC Comparison
+#
+#############################################################
+class CADataFrame
+
+  def -@
+    return cmp(:-@)
+  end
+
+  def < (other)
+    return cmp(:<, other)
+  end
+
+  def <= (other)
+    return cmp(:<=, other)
+  end
+
+  def > (other)
+    return cmp(:>, other)
+  end
+
+  def >= (other)
+    return cmp(:>=, other)
+  end
+
+  def is_masked
+    return cmp(:is_masked)
+  end
+
+  def is_finite
+    return cmp(:is_finite)
+  end
+  
+  private
+
+  def cmp (method, *argv)
+    return CADataFrame.new(ca.send(method,*argv), columns: @column_names)
+  end
+
+end
+
+#############################################################
+#
+# BASIC Manipulations
+#
+#############################################################
+class CADataFrame
+
+  def matchup (keyname, reference)
+    key = column(keyname.to_s)
+    idx = reference.matchup(key)
+    new_columns = {}
+    each_column_name do |name|
+      if name == keyname
+        new_columns[name] = reference
+      else
+        new_columns[name] = column(name).project(idx)
+      end
+    end
+    if @row_index
+      new_row_index = @row_index.project(idx).unmask(nil)
+    else
+      new_row_index = nil
+    end
+    return CADataFrame.new(new_columns, index: new_row_index) {
+      self.send(keyname)[] = reference
+    }
+  end
+
+  def histogram (name, scale = nil, options = nil)
+    if scale.nil?
+      return group_by(name).table{ { :count => col(name).count_valid } }
+    else
+      if options
+        hist = CAHistogram.int(scale, options)
+      else
+        hist = CAHistogram.int(scale)
+      end
+      hist.increment(@column_data[name.to_s])
+      hash = {
+        name.to_s => hist.midpoints[0],
+        "#{name}_L".to_s => scale[0..-2],
+        "#{name}_R".to_s => scale.shift(-1)[0..-2],
+        :count => hist[0..-2].to_ca,
+      }
+      return CADataFrame.new(hash)
+    end
+  end
+
+  def classify (name, scale = nil, opt = {})
+    if not scale
+      column = @column_data[name.to_s]
+      mids   = column.uniq
+      mapper = {}
+      mids.each_with_index do |v,i|
+        mapper[v] = i
+      end
+      cls = columns.convert(:int32) {|v| mapper[v] }
+      hash = {
+        "#{name}_M" => mids,
+        "#{name}_L" => mids,
+        "#{name}_R" => mids,
+        "#{name}_CLASS" => cls
+      }
+    else
+      option = {
+        :include_upper  => false,
+        :include_lowest => true,
+        :offset => 0,
+      }.update(opt)
+      column = @column_data[name.to_s]
+      cls = scale.bin(column,
+                      option[:include_upper],
+                      option[:include_lowest],
+                      option[:offset])
+      mids = ((scale + scale.shifted(-1))/2)[0..-2].to_ca
+      left = scale[0..-2]
+      right = scale.shift(-1)[0..-2]
+      hash = {
+        "#{name}_M" => mids.project(cls).to_ca,
+        "#{name}_L" => left.project(cls).to_ca,
+        "#{name}_R" => right.project(cls).to_ca,
+        "#{name}_CLASS" => cls
+      }
+    end
+    return CADataFrame.new(hash)
+  end
+
+  def cross (name1, name2)
+    col1 = column(name1)
+    col2 = column(name2)
+    var1 = col1.uniq.sort
+    var2 = col2.uniq.sort
+    hash = {}
+    count = Hash.new {0}
+    var1.each do |v1|
+      var2.each do |v2|
+        hash[[v1,v2]] = 0
+      end
+    end
+    list = CArray.join([col1, col2]).to_a
+    list.each do |item|
+      hash[item] += 1
+    end
+    out = CArray.object(var1.size, var2.size) { 0 }
+    var1.each_with_index do |v1, i|
+      var2.each_with_index do |v2, j|
+        out[i,j] = hash[[v1,v2]]
+      end
+    end
+    return CADataFrame.new(out, index: var1, columns: var2)
+  end
+end
+
+#############################################################
+#
+# CArray
+#
+#############################################################
+class CADataFrame
+
+  def sum
+    new_columns = {}
+    each_column do |name, col|
+      new_columns[name] = [col.sum]
+    end
+    return CADataFrame.new(new_columns, index: ["sum"])
+  end
+
+  def mean
+    new_columns = {}
+    each_column do |name, col|
+      new_columns[name] = [col.mean]
+    end
+    return CADataFrame.new(new_columns, index: ["mean"])
+  end
+end
+
+class CArray
+  
+  def describe_type
+    type = nil
+    case true
+    when numeric?
+      type = :numeric
+    when boolean?
+      type = :categorical
+    else
+      begin
+        self / 1
+        type = :numeric
+      rescue 
+        type = :categorical
+      end
+    end
+    type
+  end
+  
+  private :describe_type
+  
+  def describe (as: nil)
+    if as
+      type = as.intern
+    else
+      type = describe_type
+    end
+    case type
+    when :numeric
+      describe_numeric
+    when :categorical
+      describe_categorical
+    else
+      raise "unknown"
+    end
+  end
+  
+  def describe_numeric
+    min, q25, median, q75, max = *quantile
+    {
+      count:  is_masked.count_false,
+      mean:   mean,
+      std:    stddev,
+      max:    max,
+      q75:    q75,
+      median: median,
+      q25:    q25,
+      min:    min,
+    }
+  end
+  
+  def describe_categorical
+    hash = {}
+    each do |v|
+      hash[v] ||= 0
+      hash[v] += 1
+    end
+    top, freq = hash.max_by{|x| x[1]}
+    {
+      count:  is_masked.count_false,
+      unique: hash.size,
+      top:    top,
+      freq:   freq,
+    }
+  end
+  
+  def summary 
+    summary_categorical
+  end
+  
+  def summary_categorical
+    hash = {}
+    each do |v|
+      hash[v] ||= 0
+      hash[v] += 1
+    end
+    hash
+  end
+  
+end
+
+class CADataFrame
+  
+  def describe
+    list = []
+    @column_data.each do |name, column|
+      list << column.describe
+    end
+    CADataFrame.new(list, index: @column_names).transpose
+  end
+
+  def summary (*names)
+    data = []
+    names.each do |name|
+      data << @column_data[name].summary
+    end
+    CADataFrame.new(data, index: names).transpose        
+  end
+
+end
+
+
+class CArray
+  
+  def get_dummies
+    keys = uniq
+    hash = {}
+    keys.each do |k|
+      hash[k] = self.eq(k)
+    end
+    return hash
+  end
+  
+end
+
+class CADataFrame
+  
+  def get_dummies (*names, prefix: nil, prefix_sep: "_")
+    keep_columns = {}
+    new_columns  = {}
+    k = 0
+    @column_names.each do |name|
+      unless names.include?(name)
+        keep_columns[name] = @column_data[name]
+        next
+      end
+      hash = @column_data[name].get_dummies
+      case prefix
+      when nil
+        hash.each do |v, dummy|
+          new_columns["#{name}#{prefix_sep}#{v}"] = dummy
+        end
+      when String
+        hash.each do |v, dummy|
+          new_columns["#{prefix}#{prefix_sep}#{v}"] = dummy
+        end        
+      when Array
+        hash.each do |v, dummy|
+          new_columns["#{prefix[k]}#{prefix_sep}#{v}"] = dummy
+        end
+      when Hash
+        hash.each do |v, dummy|
+          new_columns["#{prefix[name]}#{prefix_sep}#{v}"] = dummy
+        end
+      end
+      k += 1
+    end
+    CADataFrame.new(keep_columns.update(new_columns), index: @row_index)
+  end
+  
+end
+
+
+class CADataFrame
+  
+  def save (filename)
+    open(filename, "w") {|io|
+      Marshal.dump(self, io)
+    }
+  end
+  
+  def self.load (filename)
+    out = open(filename, "r") {|io|
+      Marshal.load(io)
+    }    
+    raise "invalid data" unless out.is_a?(CADataFrame)
+    return out
+  end
+  
+end