carray-dataframe 1.0.0 → 1.1.1

data/lib/carray-dataframe/group.rb

@@ -0,0 +1,199 @@
+#############################################################
+#
+# GROUPING
+#
+#############################################################
+
+class CADataFrame
+
+  def group_by (*names)
+    if names.size == 1
+      return CADataFrameGroup.new(self, names[0])
+    else
+      return CADataFrameGroupMulti.new(self, *names)
+    end
+  end
+end
+
+class CADataFrameGroup
+
+  def initialize (dataframe, name)
+    @dataframe = dataframe
+    case name
+    when Hash
+      name, list = name.first
+      @column = @dataframe.col(name)
+      @keys = list.to_ca
+    else
+      @column = @dataframe.col(name)
+      @keys = @column.uniq.sort
+    end
+    if @column.is_a?(CATimeIndex)
+      @keys = CATimeIndex.from_index_array(@keys, @column.timestep)
+    end
+    @name = name.to_s
+    @addrs = {}
+    @keys.each do |k|
+      @addrs[k] = @column.eq(k).where
+    end
+  end
+
+  def table (&block)
+    hashpool = []
+    @keys.each do |k|
+      hashpool << @dataframe[@addrs[k]].execute(&block)
+    end
+    columns = { @name => @keys }
+    hashpool.each_with_index do |hash, i|
+      hash.each do |key, value|
+        columns[key] ||= []
+        columns[key][i] = value
+      end
+    end
+    return CADataFrame.new(columns)
+  end
+
+	def calculate (label = nil, columns: nil, &block)
+    new_columns = { @name => @keys }
+		@dataframe.each_column do |name, clmn|
+      if name == @name or ( columns && ( not columns.include?(name) ) )
+        next
+      end
+      new_columns[name] = CArray.object(@keys.size) { UNDEF }
+      @keys.each_with_index do |k, i|
+        begin
+          if block
+            new_columns[name][i] = yield(name, clmn[@addrs[k]])
+          else
+            new_columns[name][i] = clmn[@addrs[k]].send(label.intern)
+          end
+        rescue
+        end
+      end
+    end
+    return CADataFrame.new(new_columns)
+	end
+
+  def [] (group_value)
+    if @column.is_a?(CATimeIndex) and group_value.is_a?(String)
+      group_value = @column.timestep.index_at(group_value)
+    end
+    if map = @addrs[group_value]
+      return @dataframe[map]
+    else
+      return @dataframe.vacant_copy
+    end
+  end
+
+  def each
+    @addrs.each do |key, map|
+      yield @dataframe[map]
+    end
+  end
+
+  def each_with_index
+    if @column.is_a?(CATimeIndex)
+      ts = @column.timestep
+      @addrs.each do |key, map|
+        yield @dataframe[map], ts.time_at(key)
+      end
+    else
+      @addrs.each do |key, map|
+        yield @dataframe[map], key
+      end
+    end
+  end
+  include Enumerable
+end
+class CADataFrameGroupMulti
+
+  def initialize (dataframe, *names)
+    @rank = names.size
+    @dataframe = dataframe
+    @names = []
+    @column = []
+    @keys = []
+    names.each_with_index do |name, i|
+      case name
+      when Hash
+        name, list = name.first
+        @column[i] = @dataframe.col(name)
+        @keys[i] = list.to_ca
+      else
+        @column[i] = @dataframe.col(name)
+        @keys[i] = @column[i].to_ca.uniq.sort
+      end
+      @names[i] = name
+    end
+    @addrs = {}
+    each_with_keys do |list|
+      flag = @column[0].eq(list[0])
+      (1...@rank).each do |i|
+        flag &= @column[i].eq(list[i])
+      end
+      @addrs[list] = flag.where
+    end
+  end
+
+  def each_with_keys (&block)
+    @keys[0].to_a.product(*@keys[1..-1].map(&:to_a)).each(&block)
+  end
+
+  def table (&block)
+    hashpool = []
+    each_with_keys do |list|
+      hashpool << @dataframe[@addrs[list]].execute(&block)
+    end
+    columns = {}
+    @names.each do |name|
+      columns[name] = []
+    end
+    each_with_keys.with_index do |list,j|
+      @names.each_with_index do |name,i|
+        columns[name][j] = list[i]
+      end
+    end
+    hashpool.each_with_index do |hash, i|
+      hash.each do |key, value|
+        columns[key] ||= []
+        columns[key][i] = value
+      end
+    end
+    return CADataFrame.new(columns)
+  end
+
+	def calculate (label, &block)
+    new_columns = {@name=>@keys}
+		@dataframe.each_column do |name, clmn|
+      if name == @name
+        next
+      end
+      new_columns[name] = CArray.object(@keys.size) { UNDEF }
+      @keys.each_with_index do |k, i|
+        begin
+          if block
+            new_columns[name][i] = yield(name, clmn[@addrs[k]])
+          else
+            new_columns[name][i] = clmn[@addrs[k]].send(label.intern)
+          end
+        rescue
+        end
+      end
+    end
+    return CADataFrame.new(new_columns)
+	end
+
+  def [] (group_value)
+    if map = @addrs[group_value]
+      return @dataframe[map]
+    else
+      return @dataframe.vacant_copy
+    end
+  end
+
+  def each
+    each_with_keys do |key|
+      yield key, @dataframe[@addrs[key]]
+    end
+  end
+end

data/lib/carray-dataframe/iloc_accessor.rb

@@ -0,0 +1,62 @@
+class CADataFrame
+  
+  class ILocAccessor
+
+    def initialize (dataframe)
+      @dataframe = dataframe
+    end
+
+    def [] (*argv)
+      @dataframe.instance_eval {
+        index = argv.first
+        column_selector = select_columns(argv[1])
+        columns = {}
+        column_selector.each do |name|
+          columns[name] = @column_data[name][index] ### df[...]
+        end
+        return CADataFrame.new(columns, index: @row_index ? @row_index[index] : nil)
+      }
+    end
+
+    def []= (*argv)
+      value = argv.pop
+      @dataframe.instance_eval {
+        index = argv.first
+        column_selector = select_columns(argv[1])
+        case value
+        when Hash          ### value = {"a"=> [1,2,3], ... }
+          value = value.map{|k,v| [k.to_s, v]}.to_h
+          column_selector.each do |name|
+            @column_data[name][index] = value[name]
+          end
+        when Array
+          case value.first
+          when Hash        ### value = [{"a"=>1,"b"=>11}, {"a"=>2,""=>12} ...]
+            table = {}
+            column_selector.each do |name|
+              table[name] = []
+            end
+            value.each do |hash|
+              hash = hash.map{|k,v| [k.to_s, v]}.to_h
+              column_selector.each do |name|
+                table[name] << hash[name]
+              end
+            end
+            column_selector.each do |name|
+              @column_data[name][index] = table[name]
+            end
+          else             ### value = [[1,11],[2,12],...]
+            value = value.transpose
+            column_selector.each_with_index do |name, k|
+              @column_data[name][index] = value[k]
+            end
+          end
+        else               ### value = any value
+          column_selector.each_with_index do |name, k|
+            @column_data[name][index] = value
+          end
+        end
+      }
+    end
+  end
+end

data/lib/carray-dataframe/io.rb

@@ -0,0 +1,96 @@
+######################################
+#
+#  IO methods
+#
+######################################
+require "spreadsheet"
+class CArray
+
+  def save_excel (filename, &block)
+    if self.rank >= 3
+      raise "too large rank (>2) to write excel file"
+    end
+    book = Spreadsheet::Workbook.new
+    worksheet = book.create_worksheet
+    self.dim0.times do |i|
+      worksheet.row(i).push *self[i,nil]
+    end
+    if block
+      block.call(worksheet)
+    end
+    book.write(filename)
+  end
+
+  def self.load_excel (filename, sheet=0)
+    book = Spreadsheet.open(filename)
+    sheet = book.worksheet(sheet)
+    return sheet.map(&:to_a).to_ca
+  end
+end
+class CADataFrame
+
+  def self.load_sqlite3 (*args)
+    df = CArray.load_sqlite3(*args).to_dataframe
+    if df
+      return df.arrange{ 
+        column_names.each do |name|
+          mask name, nil
+        end
+      }
+    else
+      return nil
+    end
+  end
+
+  def self.load_csv (file, sep: ",", rs: $/, encoding: nil, index: nil, &block)
+    df = CArray.load_csv(file, sep: sep, rs: rs, encoding: encoding, &block).to_dataframe(index: index)
+    if df
+      return df.arrange{ 
+        column_names.each do |name|
+          mask name, nil
+        end
+      }
+    else
+      return nil
+    end
+  end
+
+  def self.from_csv (file, sep: ",", rs: $/, index: nil, &block)
+    df = CArray.from_csv(file, sep: sep, rs: rs, &block).to_dataframe(index: index)
+    if df
+      return df.arrange{ 
+        column_names.each do |name|
+          mask name, nil
+        end
+      }
+    else
+      return nil
+    end
+  end
+
+  def to_sqlite3 (*args)
+    self.to_ca.to_sqlite3(*args)
+  end
+
+  def to_sql (tablename)
+    if @column_names.any?{ |s| s =~ /[\. \-]/ }
+      columns = {}
+      each_column_name do |name|
+        name2 = name.gsub(/[\. \-]/, '_')
+        columns[name2] = column(name)
+      end
+      df = CADataFrame.new(columns)
+      return df.to_sqlite3(database: ":memory:", table: tablename)
+    else
+      return to_sqlite3(database: ":memory:", table: tablename)
+    end
+  end
+end
+module SQLite3
+  class Database
+
+    def to_df (expr)
+      return CADataFrame.load_sqlite3 self, expr
+    end
+  end
+end

data/lib/carray-dataframe/join.rb

@@ -0,0 +1,283 @@
+# Copyright (c) 2014, Sameer Deshmukh
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+class CADataFrame
+
+  def join(other_df,opts={})
+    CADataFrame::Merge.join(self, other_df, opts)
+  end
+end
+class CADataFrame
+    class MergeFrame
+      class NilSorter
+        include Comparable
+
+        def nil?
+          true
+        end
+
+        def ==(_other)
+          false
+        end
+
+        def <=>(other)
+          other.nil? ? 0 : -1
+        end
+      end
+
+      def initialize left_df, right_df, opts={} # rubocop:disable Metrics/AbcSize -- quick-fix for issue #171
+        init_opts(opts)
+        validate_on!(left_df, right_df)
+        key_sanitizer = ->(h) { sanitize_merge_keys(h.values_at(*on)) }
+        @left = df_to_a(left_df)
+        @left.sort! { |a, b| safe_compare(a.values_at(*on), b.values_at(*on)) }
+        @left_key_values = @left.map(&key_sanitizer)
+        @right = df_to_a(right_df)
+        @right.sort! { |a, b| safe_compare(a.values_at(*on), b.values_at(*on)) }
+        @right_key_values = @right.map(&key_sanitizer)
+        @left_keys, @right_keys = merge_keys(left_df, right_df, on)
+      end
+
+      def join
+        res = []
+        until left.empty? && right.empty?
+          lkey = first_left_key
+          rkey = first_right_key
+          row(lkey, rkey).tap { |r| res << r if r }
+        end
+        CADataFrame.new(res, order: dataframe_vector_names)
+      end
+      private
+      attr_reader :on, :indicator,
+        :left, :left_key_values, :keep_left, :left_keys,
+        :right, :right_key_values, :keep_right, :right_keys
+      attr_accessor :merge_key
+      LEFT_RIGHT_COMBINATIONS = {
+        #       left   right
+        inner: [false, false],
+        left:  [true, false],
+        right: [false, true],
+        outer: [true, true]
+      }.freeze
+
+      def init_opts(opts)
+        @on = opts[:on].map(&:to_s)
+        @keep_left, @keep_right = extract_left_right(opts[:how])
+        @indicator = opts[:indicator]
+      end
+
+      def dataframe_vector_names
+        left_keys.values + on + right_keys.values + Array(indicator)
+      end
+
+      def extract_left_right(how)
+        LEFT_RIGHT_COMBINATIONS[how] or
+          raise ArgumentError, "Unrecognized join option: #{how}"
+      end
+
+      def sanitize_merge_keys(merge_keys)
+        merge_keys.map { |v| v.nil? ? NilSorter.new : v }
+      end
+
+      def df_to_a df
+        # FIXME: much faster than "native" DataFrame#to_a. Should not be
+        h = df.to_h
+        keys = h.keys
+        h.values.map(&:to_a).transpose.map { |r| keys.zip(r).to_h }
+      end
+
+      def merge_keys(df1, df2, on)
+        duplicates =
+          (df1.column_names + df2.column_names - on)
+          .group_by(&:itself)
+          .select { |_, g| g.count == 2 }.map(&:first)
+        [
+          guard_keys(df1.column_names - on, duplicates, 1),
+          guard_keys(df2.column_names - on, duplicates, 2)
+        ]
+      end
+
+      def guard_keys keys, duplicates, num
+        keys.map { |v| [v, guard_duplicate(v, duplicates, num)] }.to_h
+      end
+
+      def guard_duplicate val, duplicates, num
+        duplicates.include?(val) ? "#{val}_" : val
+      end
+
+      def row(lkey, rkey)
+        case
+        when !lkey && !rkey
+          # :nocov:
+          # It's just an impossibility handler, can't be covered :)
+          raise 'Unexpected condition met during merge'
+          # :nocov:
+        when lkey == rkey
+          self.merge_key = lkey
+          add_indicator(merge_matching_rows, :both)
+        when !rkey || lt(lkey, rkey)
+          add_indicator(left_row_missing_right, :left_only)
+        else # !lkey || lt(rkey, lkey)
+          add_indicator(right_row_missing_left, :right_only)
+        end
+      end
+
+      def add_indicator(row, indicator_value)
+        return row unless indicator
+        row[indicator] = indicator_value
+        row
+      end
+
+      def merge_matching_rows
+        if one_to_one_merge?
+          merge_rows(one_to_one_left_row, one_to_one_right_row)
+        elsif one_to_many_merge?
+          result = merge_rows(left.first, right.first)
+          one_to_many_shift
+          result
+        else
+          result = cartesian_product.shift
+          end_cartesian_product if cartesian_product.empty?
+          result
+        end
+      end
+
+      def one_to_many_shift
+        shift_left = first_right_key != next_right_key
+        shift_right = first_left_key != next_left_key
+        one_to_one_left_row if shift_left
+        one_to_one_right_row if shift_right
+      end
+
+      def one_to_one_merge?
+        merge_key != next_left_key && merge_key != next_right_key
+      end
+
+      def one_to_many_merge?
+        !(merge_key == next_left_key && merge_key == next_right_key)
+      end
+
+      def one_to_one_left_row
+        left_key_values.shift
+        left.shift
+      end
+
+      def one_to_one_right_row
+        right_key_values.shift
+        right.shift
+      end
+
+      def left_row_missing_right
+        val = one_to_one_left_row
+        expand_row(val, left_keys) if keep_left
+      end
+
+      def right_row_missing_left
+        val = one_to_one_right_row
+        expand_row(val, right_keys) if keep_right
+      end
+
+      def lt(k1, k2)
+        (k1 <=> k2) == -1
+      end
+
+      def merge_rows lrow, rrow
+        left_keys
+          .map { |from, to| [to, lrow[from]] }.to_h
+          .merge(on.map { |col| [col, lrow[col]] }.to_h)
+          .merge(indicator ? {indicator => nil} : {})
+          .merge(right_keys.map { |from, to| [to, rrow[from]] }.to_h)
+      end
+
+      def expand_row row, renamings
+        renamings
+          .map { |from, to| [to, row[from]] }.to_h
+          .merge(on.map { |col| [col, row[col]] }.to_h)
+          .merge(indicator ? {indicator => nil} : {})
+      end
+
+      def first_right_key
+        right_key_values.empty? ? nil : right_key_values.first
+      end
+
+      def next_right_key
+        right_key_values[1]
+      end
+
+      def first_left_key
+        left_key_values.empty? ? nil : left_key_values.first
+      end
+
+      def next_left_key
+        left_key_values[1]
+      end
+
+      def left_rows_at_merge_key
+        left.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
+      end
+
+      def right_rows_at_merge_key
+        right.take_while { |arr| sanitize_merge_keys(arr.values_at(*on)) == merge_key }
+      end
+
+      def cartesian_product
+        @cartesian_product ||= left_rows_at_merge_key.product(right_rows_at_merge_key).map do |left_row, right_row|
+          merge_rows(left_row, right_row)
+        end
+      end
+
+      def end_cartesian_product
+        left_size = left_rows_at_merge_key.size
+        left_key_values.shift(left_size)
+        left.shift(left_size)
+        right_size = right_rows_at_merge_key.size
+        right_key_values.shift(right_size)
+        right.shift(right_size)
+        @cartesian_product = nil
+      end
+
+      def validate_on!(left_df, right_df)
+        @on.each do |on|
+          left_df.has_column?(on) && right_df.has_column?(on) or
+            raise ArgumentError, "Both dataframes expected to have #{on.inspect} field"
+        end
+      end
+
+      def safe_compare(left_array, right_array)
+        left_array.zip(right_array).map { |l, r|
+          next 0 if l.nil? && r.nil?
+          next 1 if r.nil?
+          next -1 if l.nil?
+          l <=> r
+        }.reject(&:zero?).first || 0
+      end
+    end
+    module Merge
+      class << self
+
+        def join df1, df2, opts={}
+          MergeFrame.new(df1, df2, opts).join
+        end
+      end
+    end
+  end