carray-dataframe 1.0.0

data/lib/carray/dataframe/dataframe.rb

@@ -0,0 +1,1640 @@
+require "carray"
+require "carray/io/table"
+
+module CA::TableMethods
+
+  def to_dataframe (&block)
+    df = CADataFrame.new(self, &block) 
+    if @header or @note
+      df.instance_variable_set(:@header, @header)
+      df.instance_variable_set(:@note, @note)
+      class << df
+        attr_reader :note
+        def header (name=nil)
+          if name
+            return @header[name.to_s]
+          else
+            return @column_names
+          end
+        end
+      end
+    end
+    return df 
+  end
+  
+  alias to_df to_dataframe
+  
+end
+
+class CADataFrame
+
+  #
+  # Constructor
+  #
+
+  def initialize (columns_or_table, row_index: nil, column_names: nil, &block)
+    case columns_or_table
+    when Hash
+      columns = columns_or_table
+      @column_names = columns.keys.map(&:to_s)
+      @columns = normalize_columns(columns)
+      @row_number = @columns.first[1].size
+      if @column_names.any?{ |key| @columns[key].size != @row_number }
+        raise "column sizes mismatch"
+      end
+    when CArray
+      table = columns_or_table
+      if column_names
+        @column_names = column_names.map(&:to_s)
+      else
+        if table.respond_to?(:column_names)
+          @column_names = table.column_names.map(&:to_s)
+        else
+          raise "data table (CArray) has no method 'column_names'."
+        end
+      end
+      @columns = table_to_columns(table)
+      @row_number = table.dim0
+    else
+      raise "unknown data"
+    end
+    if row_index
+      @row_index = row_index.to_ca.object
+    else
+      @row_index = nil
+    end
+    @__methods__ = {}
+    if block_given?
+      arrange(&block)
+    end
+  end
+
+  def __methods__
+    return @__methods__
+  end
+
+  def replace (other)
+    @column_names = other.column_names
+    @columns      = other.columns
+    @row_index    = other.row_index
+    @row_number   = other.row_number
+    @__methors__  = other.__methods__
+    return self
+  end
+
+  private
+
+  def table_to_columns (table)
+    new_columns = {}
+    @column_names.each_with_index do |name, i|
+      new_columns[name] = table[nil,i]
+    end
+    return new_columns
+  end
+  
+  def normalize_columns (columns)
+    new_columns = {}
+    columns.each_key do |key|
+      case columns[key]
+      when CArray
+        column = columns[key]
+      when Array
+        column = columns[key].to_ca
+        if column.rank != 1
+          list = columns[key].clone
+          column = CArray.object(list.size).convert { list.shift }
+        end
+      else
+        column = columns[key].to_ca
+      end
+      new_columns[key.to_s] = column
+    end
+    return new_columns
+  end
+
+  public
+
+  #
+  # Attributes
+  #
+
+  attr_reader :columns, :column_names, :row_index, :column_number, :row_number
+
+  def has_column?(name)
+    return @column_names.include?(name)
+  end
+
+  def column_types
+    return @columns_names.map{|name| @columns[name].data_type_name }
+  end
+
+  #
+  # Column, Row Access
+  #
+
+  def column (name_or_index)
+    case name_or_index
+    when Integer
+      return @columns[@column_names[name_or_index]]
+    when String, Symbol
+      return @columns[name_or_index.to_s]
+    end    
+  end
+  
+  alias col column
+
+  def row (idx)
+    if @row_index
+      addr = @row_index.search(idx)
+      return @column_names.map{|name| @columns[name][addr]}.to_ca
+    else
+      return @column_names.map{|name| @columns[name][idx]}.to_ca
+    end
+  end
+  
+  def index 
+    return CArray.int(@row_number).seq
+  end
+  
+  def method (hash)
+    new_hash = {}
+    hash.each do |key, value|
+      new_hash[key.to_s] = value.to_s
+    end
+    @__methods__.update(new_hash)
+  end
+
+  def method_missing (name, *args)
+    if args.size == 0 
+      name = name.to_s
+      if has_column?(name) 
+        return @columns[name]
+      elsif has_column?(name.gsub(/_/,'.')) ### For R
+        return @columns[name.gsub(/_/,'.')]
+      elsif @__methods__.include?(name)
+        return @columns[@__methods__[name]]
+      end
+    end
+    raise "no method '#{name}' for CADataFrame"
+  end
+
+
+  #
+  # Iterators
+  #
+
+  def each_column (&block)
+    return @columns.each(&block)
+  end
+
+  def each_column_name (&block)
+    return @column_names.each(&block)
+  end
+
+  def each_row_index (&block)
+    if @row_index
+      @row_index.each(&block)
+    else
+      @row_number.times(&block)
+    end
+  end
+
+  def each_row (with: Array, &block)
+    if with == Array
+      @row_number.times do |i|
+        yield @columns.map{|n,c| c[i] }
+      end
+    elsif with == Hash
+      row = {}
+      @row_number.times do |i|
+        @column_names.each do |c|
+          row[c] = @columns[c][i]
+        end
+        yield row
+      end      
+    else
+      raise "invalid data type for loop variable"
+    end
+  end
+
+  def each_row_with_row_index (with: Array, &block)
+    if with == Array
+      if @row_index
+        @row_index.each_with_index do |idx, i|
+          yield @columns.map{|n,c| c[i] }, idx
+        end
+      else
+        @row_number.times do |i|
+          yield @columns.map{|n,c| c[i] }, i
+        end
+      end      
+    elsif with == Hash
+      row = {}
+      if @row_index
+        @row_index.each_with_index do |idx, i|
+          @column_names.each do |c|
+            row[c] = @columns[c][i]
+          end
+          yield row, @row_index[i]
+        end
+      else
+        @row_number.times do |idx, i|
+          @column_names.each do |c|
+            row[c] = @columns[c][i]
+          end
+          yield row, @row_index[i]
+        end
+      end
+    else
+      raise "invalid  data type for loop variable"
+    end
+  end
+
+  #
+  # Referencing
+  #
+
+  def [] (*argv)
+    row, col = *argv    
+    new_columns = {}
+    if col.is_a?(NilClass)
+      case row
+      when CADataFrame
+        each_column_name do |key|
+          if row.has_column?(key)
+            new_columns[key] = column(key).maskout(row.column(key))
+          else
+            new_columns[key] = column(key).to_ca
+          end
+        end
+        return CADataFrame.new(new_columns, row_index: row.row_index ? row.row_index : nil)      
+      when String
+        return self[nil,row]
+      when Array
+        if row.all?{|s| s.is_a?(String) }
+          return self[nil,row]
+        else
+          @column_names.each do |key|
+            new_columns[key] = @columns[key][row]
+          end
+        end
+        return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[row] : nil)      
+      else
+        if row.is_a?(Integer)
+          row = [row]
+        end
+        @column_names.each do |key|
+          new_columns[key] = @columns[key][row]
+        end
+        return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[row] : nil)      
+      end
+    else
+      if row.is_a?(Integer)
+        row = [row]
+      end
+      case col
+      when String, Symbol
+        key = col.to_s
+        if has_column?(key)
+           return column(key)[row]
+        else
+          raise "unknow column name '#{key}'"          
+        end
+      when Array
+        if col.all?{|s| s.is_a?(String) }
+          col.each do |key|
+            key = key.to_s
+            if has_column?(key)
+              new_columns[key] = column(key)[row]
+            else
+              raise "unknow column name '#{key}'"          
+            end
+          end
+        else
+          keys = @column_names.to_ca[col].to_a
+          keys.each do |key|
+            new_columns[key] = column(key)[row]
+          end
+        end
+        return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[row] : nil)      
+      else
+        if col.is_a?(Integer)
+          col = [col]
+        end
+        keys = @column_names.to_ca[col].to_a
+        keys.each do |key|
+          new_columns[key] = column(key)[row]
+        end
+        return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[row] : nil)      
+      end
+    end
+  end
+
+  #
+  # Setting Values
+  #
+
+  def []= (*argv)
+    value = argv.pop
+    row, col = *argv
+    case col
+    when NilClass
+      case row
+      when CADataFrame
+        each_column_name do |key|
+          if row.has_column?(key)
+            column(key)[row.column(key)] = value
+          end
+        end
+      when String
+        self[nil,row] = value
+      else
+        col = @column_names.to_a
+        self[row,col] = value
+      end
+    when String, Symbol
+      key = col.to_s
+      if has_column?(key)
+        column(key)[row] = value
+      else
+        arrange {
+          append key, value
+        }
+      end
+    when Array
+      col.each do |key|
+        key = key.to_s
+        if has_column?(key)
+          column(key)[row] = value
+        else
+          raise "unknow column name '#{key}'"          
+        end
+      end
+    else
+      if col.is_a?(Integer)
+        col = [col]
+      end
+      keys = @column_names.to_ca[col].to_a
+      keys.each do |key|
+        column(key)[row] = value
+      end      
+    end
+    return value
+  end
+
+  def where (mask, value)
+    mask.column_names.each do |key|
+      if has_column?(key)
+        column(key)[mask.column(key).boolean.not] = value        
+      end
+    end
+    return value
+  end
+
+  def fill (*names, value)
+    names.each do |name|
+      if has_column?(name)
+        column(name).fill(value)
+      end
+    end
+    return self
+  end
+
+  #
+  # Arrange 
+  #
+
+  def arrange (&block)
+    return Arranger.new(self).arrange(&block)    
+  end
+
+  def rename (name1, name2)
+    if idx = @column_names.index(name1.to_s)
+      @column_names[idx] = name2.to_s
+      column = @columns[name1.to_s]
+      @columns.delete(name1.to_s)
+      @columns[name2.to_s] = column
+    else
+      raise "unknown column name #{name1}"
+    end
+  end
+
+  def downcase 
+    new_column_names = []
+    new_columns = {}
+    each_column_name do |name|
+      new_column_names << name.downcase
+      new_columns[name.downcase] = column(name)
+    end
+    @column_names = new_column_names
+    @columns = new_columns
+    return self
+  end
+
+  def append (name, new_column = nil, &block)
+    if new_column
+      # do nothing
+    elsif block
+      new_column = instance_exec(&block)
+    else
+      new_column = @columns.first[1].template(:object)
+    end
+    unless new_column.is_a?(CArray)
+      new_column = new_column.to_ca
+    end
+    if new_column.rank != 1 or new_column.size != @row_number
+      raise "invalid shape of appended column"
+    end
+    @column_names.push(name)
+    @columns[name] = new_column
+    return new_column
+  end
+
+  def lead (name, new_column = nil, &block)
+    if new_column
+      # do nothing
+    elsif block
+      new_column = instance_exec(&block)
+    else
+      new_column = @columns.first[1].template(:object)
+    end
+    unless new_column.is_a?(CArray)
+      new_column = new_column.to_ca
+    end
+    if new_column.rank != 1 or new_column.size != @row_number
+      raise "invalid shape of appended column"
+    end
+    @column_names.unshift(name)
+    @columns[name] = new_column
+    return new_column
+  end
+
+  def vacant_copy
+    new_columns = {}
+    each_column_name do |key|
+      new_columns[key] = CArray.object(0)
+    end
+    return CADataFrame.new(new_columns)    
+  end
+
+  def merge (*args)
+    return CADataFrame.merge(self, *args)
+  end
+
+  def execute (&block)
+    return instance_exec(&block)
+  end
+  
+  def calculate (label, &block)
+    hash = {}
+    each_column_name do |name|
+      begin
+        if block
+          hash[name] = [yield(name, column(name))]
+        else
+          hash[name] = [column(name).send(label.intern)]
+        end
+      rescue
+        hash[name] = [UNDEF]
+      end
+    end
+    return CADataFrame.new(hash, row_index: [label])
+  end
+
+  def resample (&block)
+    new_columns = {}
+    each_column_name do |name|
+      begin
+        new_columns[name] = yield(name, column(name))
+      rescue
+      end
+    end
+    return CADataFrame.new(new_columns)
+  end
+
+  def select (*names, &block)
+    if names.empty?
+      names = @column_names
+    end
+    if block
+      row = instance_exec(&block)
+    else
+      row = nil
+    end
+    new_columns = {}
+    names.map(&:to_s).each do |name|
+      new_columns[name] = column(name)[row]
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[row] : nil)
+  end
+
+  #
+  # Maintenance
+  #
+
+  def unmask! (value = nil)
+    each_column_name do |name|
+      column(name).unmask(value)
+    end    
+    return self
+  end
+
+  def unmask (value = nil)
+    return to_df.unmask!(value)
+  end
+
+  def detouch!
+    @columns = @columns.clone
+    each_column_name do |name|
+      @columns[name] = @columns[name].to_ca
+    end
+    if @row_index
+      @row_index = @row_index.clone
+    end
+    return self
+  end
+
+  #
+  # Transformation
+  #
+
+  def eliminate_columns (*names)
+    if names.empty?
+      return self
+    end
+    names = names.map(&:to_s)
+    new_columns = {}
+    each_column_name do |name|
+      unless names.include?(name)
+        new_columns[name] = column(name)
+      end
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index)    
+  end
+  
+  def reorder (&block)
+    index = instance_exec(&block)
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name)[index]
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index ? @row_index[index] : nil)    
+  end
+
+  def order_by (*names, &block)
+    if names.empty?
+      if block
+        ret = instance_exec(&block)
+        case ret
+        when CArray
+          list = [ret]
+        when Array
+          list = ret
+        end
+      end
+    else
+      list = @columns.values_at(*names.map{|s| s.to_s})
+    end
+    return reorder { CA.sort_addr(*list) }
+  end
+
+  def reverse
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name).reverse
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index ? @row_index.reverse : nil)    
+  end
+
+  def transpose (column_names: nil)
+    if column_names
+      column_names = header.map(&:to_s)
+    else
+      if @row_index
+        column_names = @row_index.convert(:object) {|v| v.to_s }
+      else
+        column_names = CArray.object(@row_number).seq("a",:succ)
+      end
+    end
+    return CADataFrame.new(ca.transpose, row_index: @column_names.to_ca, column_names: column_names)
+  end
+
+  def add_suffix (suf)
+    new_columns = {}
+    each_column_name do |name|
+      new_name = (name.to_s + suf).to_s
+      new_columns[new_name] = column(name)
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index)        
+  end
+
+  #
+  # Conversions
+  #
+
+  def to_df 
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name)
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index).detouch!
+  end
+
+  def objectify
+    new_columns = {}
+    each_column_name do |name|
+      new_columns[name] = column(name).object
+    end
+    return CADataFrame.new(new_columns, row_index: @row_index)
+  end
+
+  def ca (*names)
+    if names.empty?
+      return CADFArray.new(@column_names, @columns)
+    else
+      return CADFArray.new(names.map(&:to_s), @columns)
+    end
+  end
+  
+  def to_ca (*names)  
+    return ca(*names).to_ca
+  end
+  
+  def to_hash 
+    hash = {}
+    @columns.each do |k,v|
+      hash[k] = v.to_a
+    end
+    return hash
+  end
+
+  def columns_to_hash (key_name, value_names) 
+    hash = {}
+    unless @column_names.include?(key_name)
+      raise ArgumentError, "include invalid key column name #{key_name}"
+    end
+    case value_names
+    when String
+      unless @column_names.include?(value_names)
+        raise ArgumentError, "invalid key column name #{value_names}"
+      end
+      key_columns = @columns[key_name]
+      value_columns = @columns[value_names]
+      @row_number.times do |i|
+        hash[key_columns[i]] = value_columns[i]
+      end
+    when Array
+      unless value_names.all?{|s| @column_names.include?(s) }
+        raise ArgumentError, "include invalid column name in #{value_names.join(' ')}"
+      end
+      key_columns = @columns[key_name]
+      value_columns = @columns.values_at(*value_names)
+      @row_number.times do |i|
+        hash[key_columns[i]] = value_columns.map{|c| c[i]}
+      end
+    else
+      raise ArgumentError, "invalud argument"
+    end
+    return hash
+  end
+
+  private
+
+  def __obj_to_string__ (obj)
+    case obj
+    when Float
+      "%.6g" % obj
+    else
+      obj.to_s
+    end
+  end
+
+  def __strwidth__ (string)
+    if string.ascii_only?
+      return string.length
+    else
+      return string.each_char.inject(0){|s,c| s += c.bytesize > 1 ? 2 : 1 }
+    end
+  end
+
+  public
+
+  def ascii_table (rowmax = :full)
+    if @row_index
+      namelist = ["    "] + @column_names
+      tbl = CADFArray.new(namelist, @columns.clone.update("    " => @row_index))
+    else
+      namelist = ["    "] + @column_names
+      tbl = CADFArray.new(namelist, @columns.clone.update("    " => CArray.int(@row_number).seq))
+    end
+    if rowmax.is_a?(Integer) and @row_number > rowmax
+      list = tbl[0..(rowmax/2),nil].to_a
+      list.push namelist.map { "..." }
+      list.push *(tbl[-rowmax/2+1..-1,nil].to_a)
+      tbl = list.to_ca
+    end
+    datastr = tbl.convert {|c| __obj_to_string__(c) }.unmask("")
+    datamb  = datastr.convert(:boolean, &:"ascii_only?").not.sum(0).ne(0)
+    namemb  = namelist.to_ca.convert(:boolean) {|c| c.to_s.ascii_only? }.eq(0)
+    mb      = datamb.or(namemb)
+    namelen = namelist.map(&:length).to_ca
+    datalen = datastr.convert(&:length)
+    if mb.max == 0
+      if datalen.size == 0
+        lengths  = namelen.to_a
+      else
+        lengths  = datalen.max(0).pmax(namelen).to_a
+      end
+      hrule  = "-" + lengths.map {|len| "-"*len}.join("--") + "-"
+      header = " " + 
+               [namelist, lengths].transpose.map{|name, len| 
+                                            "#{name.to_s.ljust(len)}" }.join("  ") + " "
+      ary = [hrule, header, hrule]
+			if datalen.size > 0
+	      datastr[:i,nil].each_with_index do |blk, i|
+	        list = blk.flatten.to_a
+	        ary << " " + [list, lengths].transpose.map{|value, len| 
+	                                           "#{value.ljust(len)}"}.join("  ") + " "
+	      end
+			end
+      ary << hrule
+      return "DataFrame: rows#=#{@row_number}: \n" + ary.join("\n")
+    else
+      namewidth  = namelist.to_ca.convert{|c| __strwidth__(c.to_s) }
+      if datalen.size == 0
+        maxwidth   = namewidth
+      else
+        datawidth  = datastr.convert{|c| __strwidth__(c.to_s) }
+        maxwidth   = datawidth.max(0).pmax(namewidth)
+      end
+      len = maxwidth[:*,nil] - datawidth + datalen 
+      hrule  = "-" + maxwidth.map {|len| "-"*len}.join("--") + "-"
+      header = " " + 
+               [namelist, maxwidth.to_a].transpose.map{|name, len| 
+                                            "#{name.to_s.ljust(len-__strwidth__(name.to_s)+name.to_s.length)}" }.join("  ") + " "
+      ary = [hrule, header, hrule]
+			if datalen.size > 0
+	      datastr[:i,nil].each_with_addr do |blk, i|
+	        list = blk.flatten.to_a
+	        ary << " " + list.map.with_index {|value, j|
+	                  "#{value.ljust(len[i,j])}"}.join("  ") + " "
+	      end
+			end
+      ary << hrule
+      return "DataFrame: row#=#{@row_number}: \n" + ary.join("\n")
+    end
+  end
+
+
+  def inspect
+    return ascii_table(10)
+  end
+
+  def to_s
+    return ascii_table
+  end
+  
+  def to_ary
+    return [to_s]
+  end
+
+
+end
+
+#############################################################
+# 
+# ARRANGER
+#
+#############################################################
+
+
+class CADataFrame
+
+  class Arranger
+    
+    def initialize (dataframe)
+      @dataframe = dataframe
+    end
+    
+    def arrange (&block)
+      instance_exec(&block)
+      return @dataframe      
+    end
+
+    private
+    
+    def column_names
+      return @dataframe.column_names
+    end
+
+    def row_number
+      return @dataframe.row_number
+    end
+
+    def method (hash)
+      @dataframe.method(hash)
+    end
+
+    def timeseries (name, fmt = "%Y-%m-%d %H:%M:%S")
+      @dataframe.columns[name.to_s] = @dataframe.columns[name.to_s].strptime(fmt)
+    end
+
+    def type (type, name, mask = :novalue)
+      @dataframe.columns[name.to_s] = @dataframe.columns[name.to_s].to_type(type)
+      if mask != :novalue
+        @dataframe.columns[name.to_s].maskout!(options[:maskout])
+      end
+    end
+
+    def eliminate (*names)
+      if names.empty?
+        return self
+      end
+      names = names.map(&:to_s)
+      @dataframe.column_names.each do |name|
+        if names.include?(name)
+          @dataframe.columns.delete(name)
+          @dataframe.column_names.delete(name)
+        end
+      end
+    end
+
+    def template (*args, &block)
+      return @dataframe.columns.first[1].template(*args, &block)
+    end
+  
+    def double (*names)
+      names.flatten.map(&:to_s).each do |name|
+        if @dataframe.column_names.include?(name)
+          type(:double, name)
+        else
+          raise "Unknown column name '#{name}'"
+        end
+      end
+    end
+
+    def int (*names)
+      names.flatten.map(&:to_s).each do |name|
+        if @dataframe.column_names.include?(name)
+          type(:int, name)
+        else
+          raise "Unknown column name '#{name}'"
+        end
+      end
+    end
+
+    def maskout (value, *names)
+      names.flatten.map(&:to_s).each do |name|
+        @dataframe.columns[name].maskout!(value)
+      end
+    end
+
+    def unmask (value, *names)
+      names.flatten.map(&:to_s).each do |name|
+        @dataframe.columns[name].unmask(value)
+      end
+    end
+  
+    def col (name)
+      return @dataframe.col(name)
+    end
+    
+    def append (name, new_column)
+      if new_column
+        # do nothing
+      else
+        new_column = @dataframe.columns.first[1].template(:object)
+      end
+      unless new_column.is_a?(CArray)
+        new_column = new_column.to_ca
+      end
+      @dataframe.columns[name.to_s] = new_column
+      @dataframe.column_names.push(name.to_s)
+    end
+
+    def lead (name, new_column)
+      if new_column
+        # do nothing
+      else
+        new_column = @dataframe.columns.first[1].template(:object)
+      end
+      unless new_column.is_a?(CArray)
+        new_column = new_column.to_ca
+      end
+      @dataframe.columns[name.to_s] = new_column
+      @dataframe.column_names.unshift(name.to_s)
+    end
+    
+    def rename (name1, name2)
+      if idx = @dataframe.column_names.index(name1.to_s)
+        @dataframe.column_names[idx] = name2.to_s
+        column = @dataframe.columns[name1.to_s]
+        @dataframe.columns.delete(name1.to_s)
+        @dataframe.columns[name2.to_s] = column
+      else
+        raise "unknown column name #{name1}"
+      end
+    end
+
+    def downcase 
+      @dataframe.downcase
+    end
+
+    def classify (name, scale, opt = {})
+      return @dataframe.classify(name, scale, opt)
+    end
+    
+    def map (mapper, name_or_column)
+      case name_or_column
+      when String, Symbol
+        name = name_or_column
+        column = @dataframe.columns[name.to_s]
+      when CArray
+        column = name_or_column
+      when Array
+        column = name_or_column.to_ca
+      else
+        raise "invalid argument"
+      end
+      case mapper
+      when Hash
+        return column.convert(:object) {|v| hash[v] }
+      when CArray
+        return mapper.project(column)
+      when Array
+        return mapper.to_ca.project(column)        
+      end
+    end
+    
+    def method_missing (name, *args)
+      if args.size == 0 
+        if @dataframe.column_names.include?(name.to_s) 
+          return @dataframe.columns[name.to_s]
+        elsif @dataframe.__methods__.include?(name.to_s)
+          return @dataframe.columns[@dataframe.__methods__[name.to_s]]
+        end
+      end
+      super
+    end
+    
+  end
+  
+end
+
+#############################################################
+# 
+# Class methods
+#
+#############################################################
+
+class CADataFrame
+
+  def self.merge (*args)
+    ref = args.first
+    new_columns = {}
+    args.each do |table|
+      table.column_names.each do |name|
+        new_columns[name] = table.col(name)
+      end
+    end
+    return CADataFrame.new(new_columns, row_index: ref.row_index)
+  end
+
+  def self.concat (*args)
+    ref = args.first
+    column_names = ref.column_names
+    new_columns = {}
+    column_names.each do |name|
+      list = args.map{|t| t.col(name) }
+      data_type = list.first.data_type
+      new_columns[name] = CArray.bind(data_type, list, 0)   
+    end
+    if args.map(&:row_index).all?
+      new_row_index = CArray.join(*args.map(&:row_index))
+    else
+      new_row_index = nil
+    end
+    return CADataFrame.new(new_columns, row_index: new_row_index)
+  end
+
+
+end
+
+#############################################################
+# 
+# CADFArray
+#
+#############################################################
+
+class CADFArray < CAObject # :nodoc:
+
+  def initialize (column_names, columns)
+    @column_names = column_names
+    @columns      = columns
+    dim = [@columns[@column_names.first].size, @column_names.size]
+    extend CA::TableMethods
+    super(:object, dim, :read_only=>true)
+    __create_mask__
+  end
+  
+  attr_reader :column_names
+
+  def fetch_index (idx)
+    r, c = *idx
+    name = @column_names[c]
+    return @columns[name][r]
+  end
+
+  def copy_data (data)
+    @column_names.each_with_index do |name, i|
+      data[nil,i] = @columns[name].value
+    end
+  end
+
+  def create_mask
+  end
+
+  def mask_fetch_index (idx)
+    r, c = *idx
+    name = @column_names[c]
+    if @columns[name].has_mask?
+      return @columns[name].mask[r]
+    else
+      return 0
+    end
+  end
+
+  def mask_copy_data (data)
+    @column_names.each_with_index do |name, i|
+      if @columns[name].has_mask?
+        data[nil,i] = @columns[name].mask
+      end
+    end
+  end
+  
+  def to_ca
+    obj = super
+    obj.extend CA::TableMethods
+    obj.column_names = @column_names
+    return obj
+  end
+
+end
+
+#############################################################
+# 
+# BASIC Comparison
+#
+#############################################################
+
+
+class CADataFrame
+  
+  def -@
+    return cmp(:-@)    
+  end
+    
+  def < (other)
+    return cmp(:<, other)
+  end
+
+  def <= (other)
+    return cmp(:<=, other)
+  end
+
+  def > (other)
+    return cmp(:>, other)
+  end
+
+  def >= (other)
+    return cmp(:>=, other)
+  end
+
+  def is_masked
+    return cmp(:is_masked)
+  end
+
+  def is_finite
+    return cmp(:is_finite)
+  end
+  
+  private
+
+  def cmp (method, *argv)
+    return CADataFrame.new(ca.send(method,*argv), column_names: @column_names)    
+  end
+
+end
+
+#############################################################
+# 
+# BASIC Manipulations
+#
+#############################################################
+
+class CADataFrame
+
+  def matchup (keyname, reference)
+    key = column(keyname.to_s)
+    idx = reference.matchup(key)
+    new_columns = {}
+    each_column_name do |name|
+      if name == keyname
+        new_columns[name] = reference
+      else
+        new_columns[name] = column(name).project(idx)
+      end
+    end
+    if @row_index
+      new_row_index = @row_index.project(idx).unmask(nil)
+    else
+      new_row_index = nil
+    end
+    return CADataFrame.new(new_columns, row_index: new_row_index) {
+      self.send(keyname)[] = reference
+    }
+  end
+
+  def join (table, on: nil)
+  end
+
+  def histogram (name, scale = nil, options = nil)
+    if scale.nil?
+      return group_by(name).table{ { :count => col(name).count_valid } }
+    else
+      if options
+        hist = CAHistogram.int(scale, options)
+      else
+        hist = CAHistogram.int(scale)      
+      end
+      hist.increment(@columns[name.to_s])
+      hash = {
+        name.to_s => hist.midpoints[0],
+        "#{name}_L".to_s => scale[0..-2],
+        "#{name}_R".to_s => scale.shift(-1)[0..-2],
+        :count => hist[0..-2].to_ca,
+      }
+      return CADataFrame.new(hash)
+    end
+  end
+
+  def classify (name, scale = nil, opt = {})
+    if not scale
+      column = @columns[name.to_s]
+      mids   = column.uniq
+      mapper = {}
+      mids.each_with_index do |v,i|
+        mapper[v] = i
+      end
+      cls = columns.convert(:int32) {|v| mapper[v] }
+      hash = {
+        "#{name}_M" => mids,
+        "#{name}_L" => mids,
+        "#{name}_R" => mids,
+        "#{name}_CLASS" => cls        
+      }
+    else
+      option = {
+        :include_upper  => false,
+        :include_lowest => true,
+        :offset => 0,
+      }.update(opt)
+      column = @columns[name.to_s]
+      cls = scale.bin(column, 
+                      option[:include_upper],
+                      option[:include_lowest], 
+                      option[:offset])
+      mids = ((scale + scale.shifted(-1))/2)[0..-2].to_ca
+      left = scale[0..-2]
+      right = scale.shift(-1)[0..-2]
+      hash = {
+        "#{name}_M" => mids.project(cls).to_ca,
+        "#{name}_L" => left.project(cls).to_ca,
+        "#{name}_R" => right.project(cls).to_ca,
+        "#{name}_CLASS" => cls
+      }
+    end
+    return CADataFrame.new(hash)
+  end
+
+  def cross (name1, name2)
+    col1 = column(name1)
+    col2 = column(name2)
+    var1 = col1.uniq.sort
+    var2 = col2.uniq.sort
+    hash = {}
+    count = Hash.new {0}
+    var1.each do |v1|
+      var2.each do |v2|
+        hash[[v1,v2]] = 0
+      end
+    end
+    list = CArray.join([col1, col2]).to_a
+    list.each do |item|
+      hash[item] += 1 
+    end
+    out = CArray.object(var1.size, var2.size) { 0 }
+    var1.each_with_index do |v1, i|
+      var2.each_with_index do |v2, j|
+        out[i,j] = hash[[v1,v2]]
+      end
+    end
+    return CADataFrame.new(out, row_index: var1, column_names: var2)
+  end
+
+end
+
+
+#############################################################
+# 
+# GROUPING
+#
+#############################################################
+
+class CADataFrame
+  
+  def group_by (*names)
+    if names.size == 1
+      return CADataFrameGroup.new(self, names[0])
+    else
+      return CADataFrameGroupMulti.new(self, *names)
+    end
+  end
+  
+end
+
+class CADataFrameGroup
+
+  def initialize (dataframe, name)
+    @dataframe = dataframe
+    case name
+    when Hash
+      name, list = name.first
+      @column = @dataframe.col(name)
+      @keys = list.to_ca
+    else
+      @column = @dataframe.col(name)
+      @keys = @column.uniq.sort
+    end
+    @name = name.to_s
+    @addrs = {}
+    @keys.each do |k|
+      @addrs[k] = @column.eq(k).where
+    end
+  end
+
+  def table (&block)
+    hashpool = []
+    @keys.each do |k|
+      hashpool << @dataframe[@addrs[k]].execute(&block)
+    end
+    columns = {@name=>@keys}
+    hashpool.each_with_index do |hash, i|
+      hash.each do |key, value|
+        columns[key] ||= []
+        columns[key][i] = value
+      end
+    end
+    return CADataFrame.new(columns)
+  end
+
+	def calculate (label, &block)
+    new_columns = {@name=>@keys}
+		@dataframe.each_column do |name, clmn|
+      if name == @name
+        next
+      end
+      new_columns[name] = CArray.object(@keys.size) { UNDEF }
+      @keys.each_with_index do |k, i|
+        begin
+          if block
+            new_columns[name][i] = yield(name, clmn[@addrs[k]])
+          else
+            new_columns[name][i] = clmn[@addrs[k]].send(label.intern)
+          end
+        rescue
+        end
+      end
+    end
+    return CADataFrame.new(new_columns)
+	end
+
+  def [] (group_value)
+    if map = @addrs[group_value]
+      return @dataframe[map]
+    else
+      return @dataframe.vacant_copy
+    end
+  end
+
+  def each 
+    @addrs.each do |key, map|
+      yield @dataframe[map]
+    end
+  end
+  
+  def each_with_index
+    @addrs.each do |key, map|
+      yield @dataframe[map], key
+    end    
+  end
+
+  include Enumerable
+
+end
+
+class CADataFrameGroupMulti
+  
+  def initialize (dataframe, *names)
+    @rank = names.size
+    @dataframe = dataframe
+    @names = []
+    @column = []
+    @keys = []
+    names.each_with_index do |name, i|    
+      case name
+      when Hash
+        name, list = name.first
+        @column[i] = @dataframe.col(name)
+        @keys[i] = list.to_ca
+      else
+        @column[i] = @dataframe.col(name)
+        @keys[i] = @column[i].to_ca.uniq.sort
+      end
+      @names[i] = name
+    end
+    @addrs = {}
+    each_with_keys do |list|
+      flag = @column[0].eq(list[0])
+      (1...@rank).each do |i|
+        flag &= @column[i].eq(list[i])
+      end
+      @addrs[list] = flag.where
+    end
+  end
+
+  def each_with_keys (&block)
+    @keys[0].to_a.product(*@keys[1..-1].map(&:to_a)).each(&block)    
+  end
+  
+  def table (&block)
+    hashpool = []
+    each_with_keys do |list|
+      hashpool << @dataframe[@addrs[list]].execute(&block) 
+    end
+    columns = {}
+    @names.each do |name|
+      columns[name] = []
+    end
+    each_with_keys.with_index do |list,j|
+      @names.each_with_index do |name,i|
+        columns[name][j] = list[i]
+      end
+    end
+    hashpool.each_with_index do |hash, i|
+      hash.each do |key, value|
+        columns[key] ||= []
+        columns[key][i] = value
+      end
+    end
+    return CADataFrame.new(columns)
+  end
+
+  def [] (group_value)
+    if map = @addrs[group_value]
+      return @dataframe[map]
+    else
+      return @dataframe.vacant_copy
+    end
+  end
+  
+  def each 
+    each_with_keys do |key|
+      yield key, @dataframe[@addrs[key]]
+    end
+  end
+  
+end
+
+#############################################################
+# 
+# PIVOT TABLE
+#
+#############################################################
+
+class CADataFrame
+  
+  def pivot (name1, name2)
+    return CADataFramePivot.new(self, name1, name2)
+  end
+  
+end
+
+class CADataFramePivot
+  
+  def initialize (dataframe, name1, name2)
+    @dataframe = dataframe
+    case name1
+    when Hash
+      name1, list = name1.first
+      @column1 = @dataframe.col(name1)
+      @keys1 = list.to_ca
+    else
+      @column1 = @dataframe.col(name1)
+      @keys1 = @column1.uniq.sort  
+    end
+    case name2
+    when Hash
+      name2, list = name2.first
+      @column2 = @dataframe.col(name2)
+      @keys2 = list
+    else
+      @column2 = @dataframe.col(name2)
+      @keys2 = @column2.uniq.sort
+    end
+    @addrs = {}
+    @keys1.each do |k1|
+      @keys2.each do |k2|
+        @addrs[[k1,k2]] = (@column1.eq(k1) & @column2.eq(k2)).where
+      end
+    end
+  end
+  
+  def table (&block)
+    columns = {}
+    @keys2.each do |k2|
+      columns[k2] = CArray.object(@keys1.size) { UNDEF }
+    end
+    @keys1.each_with_index do |k1, i|
+      @keys2.each do |k2|
+        columns[k2][i] = @dataframe[@addrs[[k1,k2]]].execute(&block)
+      end
+    end
+    return CADataFrame.new(columns, row_index: @keys1)
+  end
+  
+end
+
+
+#############################################################
+# 
+# CArray
+#
+#############################################################
+
+
+class CArray
+  
+  def value_counts
+    hash = {}
+    values = uniq
+    values.each do |value|
+      hash[value] = 0
+    end
+    each do |value|
+      hash[value] += 1      
+    end
+    counts = values.convert{|value| hash[value]}
+    return CADataFrame.new({'value' => values, 'count' => counts})
+  end
+  
+end
+
+
+
+class CADataFrame
+
+  def to_sqlite3 (*args)
+    self.ca.to_sqlite3(*args)
+  end
+
+  def to_sql (tablename)
+    if @column_names.any?{ |s| s =~ /[\. \-]/ }
+      columns = {}
+      each_column_name do |name|
+        name2 = name.gsub(/[\. \-]/, '_')
+        columns[name2] = column(name)
+      end
+      df = CADataFrame.new(columns)
+      return df.to_sqlite3(database: ":memory:", table: tablename)
+    else
+      return to_sqlite3(database: ":memory:", table: tablename)
+    end
+  end
+
+end
+
+module SQLite3
+
+  class Database
+
+    def to_df (expr)
+      return CADataFrame.load_sqlite3 self, expr 
+    end
+
+  end
+
+end
+
+######################################
+#
+#  IO methods
+#
+######################################
+
+require "spreadsheet"
+
+class CArray
+  
+  def save_excel (filename, &block)
+    if self.rank >= 3
+      raise "too large rank (>2) to write excel file"
+    end
+    book = Spreadsheet::Workbook.new
+    worksheet = book.create_worksheet
+    self.dim0.times do |i|
+      worksheet.row(i).push *self[i,nil]
+    end
+    if block
+      block.call(worksheet)
+    end
+    book.write(filename)    
+  end
+  
+  def self.load_excel (filename, sheet=0)
+    book = Spreadsheet.open(filename)
+    sheet = book.worksheet(sheet)
+    return sheet.map(&:to_a).to_ca
+  end
+  
+end
+
+class CADataFrame
+  
+  def self.load_sqlite3 (*args)
+    return CArray.load_sqlite3(*args).to_dataframe.arrange{ maskout nil, *column_names }
+  end
+
+
+  def self.load_csv (*args, &block)
+    return CArray.load_csv(*args, &block).to_dataframe.arrange{ maskout nil, *column_names }
+  end
+
+  def self.from_csv (*args, &block)
+    return CArray.from_csv(*args, &block).to_dataframe.arrange{ maskout nil, *column_names }
+  end
+
+  def to_csv (io = "", option = {}, rs: $/, sep: ",", fill: "", with_row_index: true, &block)
+    if @row_index and with_row_index
+      namelist = [""] + @column_names
+      tbl = CADFArray.new(namelist, @columns.clone.update("" => @row_index))
+    else
+      tbl = ca.to_ca
+    end
+    return tbl.to_csv(io, option, rs: rs, sep: sep, fill: fill, &block)
+  end
+
+  def to_daru
+    require "daru"
+    columns = {}
+    each_column_name do |name|
+      columns[name] = column(name).to_a
+    end
+    if @row_index
+      return Daru::DataFrame.new(columns, index: @row_index.to_a, order: @column_names)
+    else
+      return Daru::DataFrame.new(columns, order: @column_names)      
+    end
+  end
+
+  def to_xlsx (filename, sheet_name: 'Sheet1', with_row_index: false, &block)
+    require "axlsx"
+    xl = Axlsx::Package.new
+    xl.use_shared_strings = true
+    sheet = xl.workbook.add_worksheet(name: sheet_name)
+    df = self.to_df.objectify.unmask("=NA()")
+    if with_row_index
+      sheet.add_row([""] + column_names)
+      df.each_row_with_row_index(with: Array) do |list, i|
+        sheet.add_row([i] + list)
+      end
+    else
+      sheet.add_row(column_names)
+      df.each_row(with: Array) do |list|
+        sheet.add_row(list)
+      end
+    end
+    if block_given?
+      yield sheet
+    end
+    xl.serialize(filename)
+  end
+
+end
+
+
+
+
+
+
+