fat_core 1.5.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6d9999db399e323a385070ca59d15deb692d65d
-  data.tar.gz: 5bb8ade666435e0c67afeaa293e236082cb19906
+  metadata.gz: 4617dd301a8a4f6ea796c27cfff91364f72b79f2
+  data.tar.gz: e6eb1b4057994c9672712b28189066af2ebc56d8
 SHA512:
-  metadata.gz: 806ba3e817c5899bd5cb3ffa7a13069d8710c2a29406ae9b6060cf2d30e5d28cc3fc1b983d8b8b3aa9beafc6ceab505695a218c39e8e605ec811bc0c671a3bf7
-  data.tar.gz: e11602a00d01b81bd201092b4bf3cbe8780435d2da3bd3f708f83b2e94677e43a2993b0fc4066fdf8d5d45b258e2516c928c72790e93e84dc77f64714b202f59
+  metadata.gz: 478b2f046e8cfc211eaed97bcd20ddf80b9cbb7c38e022931ceaeade08ba91d39f94e4ad085145426b7789a3795648112149cc0e53256bce17c0cc6241fbee2c
+  data.tar.gz: 32a4dbf0406779b323a76f61319f9553b9442d8a5ebce0c8567720b3134c9894087b6f22d38720d08f0070884f11eb74d8655ec771b51c4ac8f8e47f6442975a

data/lib/fat_core/column.rb

@@ -62,6 +62,10 @@ module FatCore
       items.compact.sum
     end
 
+    def count
+      items.compact.count
+    end
+
     def min
       only_with('min', 'NilClass', 'Numeric', 'String', 'DateTime')
       items.compact.min

data/lib/fat_core/table.rb

@@ -46,6 +46,7 @@ module FatCore
     def initialize(input = nil, ext = '.csv')
       @columns = []
       @footers = {}
+      @boundaries = []
       return self if input.nil?
       case input
       when IO, StringIO
@@ -121,6 +122,11 @@ module FatCore
       columns.first.size
     end
 
+    # Return whether this table is empty.
+    def empty?
+      size.zero?
+    end
+
     # Return the rows of the table as an array of hashes, keyed by the headers.
     def rows
       rows = []
@@ -136,8 +142,121 @@ module FatCore
       rows
     end
 
-    def empty?
-      rows.empty?
+    protected
+
+    # Return the rows from first to last.  We could just index #rows, but in a
+    # large table, that would require that we construct all the rows for a range
+    # of any size.
+    def rows_range(first = 0, last = size - 1)
+      raise ArgumentError, 'first must be <= last' unless first <= last
+      rows = []
+      unless columns.empty?
+        first.upto(last) do |rnum|
+          row = {}
+          columns.each do |col|
+            row[col.header] = col[rnum]
+          end
+          rows << row
+        end
+      end
+      rows
+    end
+
+    ## ###########################################################################
+    ##  Group Boundaries
+    ##
+    ##  Boundaries mark the last row in each "group" within the table. The last
+    ##  row of the table is always an implicit boundary, and having the last row
+    ##  as the sole boundary is the default for new tables unless mentioned
+    ##  otherwise. Resetting the boundaries means to put it back in that default
+    ##  state.
+    ##
+    ##  Note that tables are for the most part, immutable. That is, the data
+    ##  rows of the table, once set, are never changed by methods on the
+    ##  table. Any transformation of a table results in a new table. Boundaries
+    ##  and footers are exceptions to immutability, but even they only affect
+    ##  the boundary and footer attributes of the table, not the data rows.
+    ##
+    ##  Boundaries can be added when a table is read in, for example, from the
+    ##  text of an org table in which each hline (other than the one separating
+    ##  the headers from the body) marks a boundary for the row immediately
+    ##  preceding the hline.
+    ##
+    ##  The #order_by method resets the boundaries then adds boundaries at the
+    ##  last row of each group as a boundary.
+    ##
+    ##  The #union_all (but not #union since it deletes duplicates) method adds
+    ##  a boundary between the constituent tables. #union_all also preserves any
+    ##  boundary markers within the constituent tables. In doing so, the
+    ##  boundaries of the second table in the #union_all are increased by the
+    ##  size of the first table so that they refer to rows in the new table.
+    ##
+    ##  The #select method preserves any boundaries from the parent table
+    ##  without change, since it only selects columns for the output and deletes
+    ##  no rows.
+    ##
+    ##  All the other table-transforming methods reset the boundaries in the new
+    ##  table. For example, #order_by and #where re-arrange and delete rows, so
+    ##  the old boundaries would make no sense anyway. Likewise, #union,
+    ##  #intersection, #except, and #join reset the boundaries to their default.
+    ##  ###########################################################################
+
+    public
+
+    # Return an array of an array of row hashes for the groups in this Table.
+    def groups
+      normalize_boundaries
+      groups = []
+      (0..boundaries.size - 1).each do |k|
+        groups << group_rows(k)
+      end
+      groups
+    end
+
+    protected
+
+    # Reader for boundaries, but not public.
+    def boundaries
+      @boundaries
+    end
+
+    # Writer for boundaries, but not public.
+    def boundaries=(bounds)
+      @boundaries = bounds
+    end
+
+    # Make sure size - 1 is last boundary and that they are unique and sorted.
+    def normalize_boundaries
+      unless empty?
+        boundaries.push(size - 1) unless boundaries.include?(size - 1)
+        self.boundaries = boundaries.uniq.sort
+      end
+      boundaries
+    end
+
+    # Mark a boundary at k, and if k is nil, the last row in the table
+    # as a group boundary.
+    def mark_boundary(k = nil)
+      if k
+        boundaries.push(k)
+      else
+        boundaries.push(size - 1)
+      end
+    end
+
+    # Concatenate the array of argument bounds to this table's boundaries, but
+    # increase each of the indexes in bounds by shift. This is used in the
+    # #union_all method.
+    def append_boundaries(bounds, shift: 0)
+      @boundaries += bounds.map { |k| k + shift }
+    end
+
+    def group_rows(k)
+      normalize_boundaries
+      return [] unless k < boundaries.size
+      first = k.zero? ? 0 : boundaries[k - 1] + 1
+      last = boundaries[k]
+      rows_range(first, last)
     end
 
     ############################################################################
@@ -145,9 +264,11 @@ module FatCore
     # all return a new Table object rather than modifying the table in place.
     ############################################################################
 
+    public
+
     # Return a new Table sorted on the rows of this Table on the possibly
     # multiple keys given in the array of syms in headers. Append a ! to the
-    # symbol name to indicate reverse sorting on that column.
+    # symbol name to indicate reverse sorting on that column.  Resets groups.
     def order_by(*sort_heads)
       sort_heads = [sort_heads].flatten
       rev_heads = sort_heads.select { |h| h.to_s.ends_with?('!') }
@@ -158,65 +279,62 @@ module FatCore
         key2 = sort_heads.map { |h| rev_heads.include?(h) ? r1[h] : r2[h] }
         key1 <=> key2
       end
+      # Add the new rows to the table, but mark a group boundary at the points
+      # where the sort key changes value.
       new_tab = Table.new
-      new_rows.each do |nrow|
+      last_key = nil
+      new_rows.each_with_index do |nrow, k|
         new_tab << nrow
+        key = nrow.fetch_values(*sort_heads)
+        new_tab.mark_boundary(k - 1) if last_key && key != last_key
+        last_key = key
       end
       new_tab
     end
 
     # Return a Table having the selected column expressions. Each expression can
-    # be either a (1) symbol, (2) a hash of symbol => symbol, or (3) a hash of
-    # symbol => 'string', though the bare symbol arguments (1) must precede any
-    # hash arguments. Each expression results in a column in the resulting Table
-    # in the order given. The expressions are evaluated in order as well.
-    def select(*exps)
+    # be either a (1) symbol, :old_col, representing a column in the current
+    # table, (2) a hash of new_col: :old_col to rename an existing :old_col
+    # column as :new_col, or (3) a hash of new_col: 'expression', to add a new
+    # column that is computed as an arbitrary ruby expression of the existing
+    # columns (whether selected for the output table or not) or any new_col
+    # defined earlier in the argument list.  The expression string can also
+    # access the instance variable @row as the row number of the row being
+    # evaluated.  The bare symbol arguments (1) must precede any hash arguments
+    # (2) or (3). Each expression results in a column in the resulting Table in
+    # the order given. The expressions are evaluated in left-to-right order as
+    # well.  The output table preserves any groups present in the input table.
+    def select(*cols, **new_cols)
       result = Table.new
-      new_cols = {}
       ev = Evaluator.new(vars: { row: 0 }, before: '@row += 1')
       rows.each do |old_row|
-        new_heads = []
-        new_row ||= {}
-        exps.each do |exp|
-          case exp
-          when Symbol, String
-            h = exp.as_sym
-            raise "Column '#{h}' in select does not exist" unless column?(h)
-            new_row[h] = old_row[h]
-          when Hash
-            # Note that when one of the exps is a Hash, it will contain an
-            # output expression for each member of the Hash, so we have to loop
-            # through them here.
-            exp.each_pair do |key, val|
-              # Gather the new values computed so far for this row
-              vars = old_row.merge(new_row)
-              case val
-              when Symbol
-                h = val.as_sym
-                raise "Column '#{h}' in select does not exist" unless vars.keys.include?(h)
-                new_row[key] = vars[h]
-              when String
-                # Now we have a hash, vars, of all local variables we want to be
-                # defined while evaluating expression xp as the value of column
-                # key in the new column.
-                h = key.as_sym
-                new_row[h] = ev.evaluate(val, vars: vars)
-                # Don't add this column to new_heads until after the eval so it
-                # does not shadow the existing value of row[h].
-              else
-                raise 'Hash parameters to select must be a symbol or string'
-              end
-            end
+        new_row = {}
+        cols.each do |k|
+          h = k.as_sym
+          raise "Column '#{h}' in select does not exist" unless column?(h)
+          new_row[h] = old_row[h]
+        end
+        new_cols.each_pair do |key, val|
+          key = key.as_sym
+          vars = old_row.merge(new_row)
+          case val
+          when Symbol
+            raise "Column '#{val}' in select does not exist" unless vars.keys.include?(val)
+            new_row[key] = vars[val]
+          when String
+            new_row[key] = ev.evaluate(val, vars: vars)
           else
-            raise 'Parameters to select must be a symbol, string, or hash'
+            raise 'Hash parameters to select must be a symbol or string'
           end
         end
         result << new_row
       end
+      result.boundaries = boundaries
       result
     end
 
-    # Return a Table containing only rows matching the where expression.
+    # Return a Table containing only rows matching the where expression.  Resets
+    # groups.
     def where(expr)
       expr = expr.to_s
       result = Table.new
@@ -227,7 +345,7 @@ module FatCore
       result
     end
 
-    # Return this table with all duplicate rows eliminated.
+    # Return this table with all duplicate rows eliminated. Resets groups.
     def distinct
       result = Table.new
       uniq_rows = rows.uniq
@@ -237,6 +355,7 @@ module FatCore
       result
     end
 
+    # Return this table with all duplicate rows eliminated. Resets groups.
     def uniq
       distinct
     end
@@ -247,23 +366,31 @@ module FatCore
     # the same type in the two tables, or an exception will be thrown.
     # Duplicates are eliminated from the result.
     def union(other)
-      set_operation(other, :+, true)
+      set_operation(other, :+,
+                    distinct: true,
+                    add_boundaries: true)
     end
 
     # Return a Table that combines this table with another table. In other
     # words, return the union of this table with the other. The headers of this
     # table are used in the result. There must be the same number of columns of
     # the same type in the two tables, or an exception will be thrown.
-    # Duplicates are not eliminated from the result.
+    # Duplicates are not eliminated from the result.  Adds group boundaries at
+    # boundaries of the constituent tables. Preserves and adjusts the group
+    # boundaries of the constituent table.
     def union_all(other)
-      set_operation(other, :+, false)
+      set_operation(other, :+,
+                    distinct: false,
+                    add_boundaries: true,
+                    inherit_boundaries: true)
     end
 
     # Return a Table that includes the rows that appear in this table and in
     # another table. In other words, return the intersection of this table with
     # the other. The headers of this table are used in the result. There must be
     # the same number of columns of the same type in the two tables, or an
-    # exception will be thrown. Duplicates are eliminated from the result.
+    # exception will be thrown. Duplicates are eliminated from the
+    # result. Resets groups.
     def intersect(other)
       set_operation(other, :intersect, true)
     end
@@ -272,7 +399,8 @@ module FatCore
     # another table. In other words, return the intersection of this table with
     # the other. The headers of this table are used in the result. There must be
     # the same number of columns of the same type in the two tables, or an
-    # exception will be thrown. Duplicates are not eliminated from the result.
+    # exception will be thrown. Duplicates are not eliminated from the
+    # result. Resets groups.
     def intersect_all(other)
       set_operation(other, :intersect, false)
     end
@@ -282,7 +410,7 @@ module FatCore
     # set difference between this table an the other. The headers of this table
     # are used in the result. There must be the same number of columns of the
     # same type in the two tables, or an exception will be thrown. Duplicates
-    # are eliminated from the result.
+    # are eliminated from the result. Resets groups.
     def except(other)
       set_operation(other, :difference, true)
     end
@@ -292,7 +420,7 @@ module FatCore
     # set difference between this table an the other. The headers of this table
     # are used in the result. There must be the same number of columns of the
     # same type in the two tables, or an exception will be thrown. Duplicates
-    # are not eliminated from the result.
+    # are not eliminated from the result. Resets groups.
     def except_all(other)
       set_operation(other, :difference, false)
     end
@@ -302,7 +430,10 @@ module FatCore
     # Apply the set operation given by op between this table and the other table
     # given in the first argument.  If distinct is true, eliminate duplicates
     # from the result.
-    def set_operation(other, op = :+, distinct = true)
+    def set_operation(other, op = :+,
+                      distinct: true,
+                      add_boundaries: false,
+                      inherit_boundaries: false)
       unless columns.size == other.columns.size
         raise 'Cannot apply a set operation to tables with a different number of columns.'
       end
@@ -312,8 +443,14 @@ module FatCore
       other_rows = other.rows.map { |r| r.replace_keys(headers) }
       result = Table.new
       new_rows = rows.send(op, other_rows)
-      new_rows.each do |row|
+      new_rows.each_with_index do |row, k|
         result << row
+        result.mark_boundary if k == size - 1 && add_boundaries
+      end
+      if inherit_boundaries
+        result.boundaries = normalize_boundaries
+        other.normalize_boundaries
+        result.append_boundaries(other.boundaries, shift: size)
       end
       distinct ? result.distinct : result
     end
@@ -381,7 +518,7 @@ module FatCore
     #      of all columns in T1 followed by all columns in T2. If the tables
     #      have N and M rows respectively, the joined table will have N * M
     #      rows.
-    #
+    # Resets groups.
     JOIN_TYPES = [:inner, :left, :right, :full, :cross]
 
     def join(other, *exps, join_type: :inner)
@@ -449,6 +586,73 @@ module FatCore
       join(other, join_type: :cross)
     end
 
+    # Return a Table with a single row for each group of rows in the input table
+    # where the value of all columns named as simple symbols are equal. All
+    # other columns are set to the result of aggregating the values of that
+    # column within the group according to a aggregate function (:count, :sum,
+    # :min, :max, etc.), which defaults to the :first function, giving the value
+    # of that column for the first row in the group.  You can specify a
+    # different aggregate function for a column by adding a hash parameter with
+    # the column as the key and a symbol for the aggregate function as the
+    # value.  For example, consider the following call:
+    #
+    # tab.group_by(:date, :code, :price, shares: :sum, ).
+    #
+    # The first three parameters are simple symbols, so the table is divided
+    # into groups of rows in which the value of :date, :code, and :price are
+    # equal. The shares: hash parameter is set to the aggregate function :sum,
+    # so it will appear in the result as the sum of all the :shares values in
+    # each group. Any non-aggregate columns that have no aggregate function set
+    # default to using the aggregate function :first. Because of the way Ruby
+    # parses parameters to a method call, all the grouping symbols must appear
+    # first in the parameter list before any hash parameters.
+    def group_by(*group_cols, **agg_cols)
+      default_agg_func = :first
+      default_cols = headers - group_cols - agg_cols.keys
+      default_cols.each do |h|
+        agg_cols[h] = default_agg_func
+      end
+
+      sorted_tab = order_by(group_cols)
+      groups = sorted_tab.rows.group_by do |r|
+        group_cols.map { |k| r[k] }
+      end
+      result = Table.new
+      groups.each_pair do |_vals, grp_rows|
+        result << row_from_group(grp_rows, group_cols, agg_cols)
+      end
+      result
+    end
+
+    ############################################################################
+    # Footer methods
+    ############################################################################
+    def add_footer(label: 'Total', aggregate: :sum, heads: [])
+      foot = {}
+      heads.each do |h|
+        raise "No #{h} column in table to #{aggregate}" unless headers.include?(h)
+        foot[h] = column(h).send(aggregate)
+      end
+      @footers[label.as_sym] = foot
+      self
+    end
+
+    def add_sum_footer(cols, label = 'Total')
+      add_footer(heads: cols)
+    end
+
+    def add_avg_footer(cols, label = 'Average')
+      add_footer(label: label, aggregate: :avg, heads: cols)
+    end
+
+    def add_min_footer(cols, label = 'Minimum')
+      add_footer(label: label, aggregate: :min, heads: cols)
+    end
+
+    def add_max_footer(cols, label = 'Maximum')
+      add_footer(label: label, aggregate: :max, heads: cols)
+    end
+
     private
 
     # Return an output row appropriate to the given join type, including all the
@@ -585,64 +789,6 @@ module FatCore
       self
     end
 
-    public
-
-    # Return a Table in which all rows of the table are divided into groups
-    # where the value of all columns named as simple symbols are equal. All
-    # other columns are set to the result of aggregating the values of that
-    # column within the group according to the Column aggregate function (:sum,
-    # :min, :max, etc.) set in a hash parameter with the non-aggregate column
-    # name as a key and the symbol for the aggregate function as a value. For
-    # example, consider the following call:
-    #
-    # #+BEGIN_EXAMPLE
-    # tab.group_by(:date, :code, :price, shares: :sum, ).
-    # #+END_EXAMPLE
-    #
-    # The first three parameters are simple symbols, so the table is divided
-    # into groups of rows in which the value of :date, :code, and :price are
-    # equal. The :shares parameter is set to the aggregate function :sum, so it
-    # will appear in the result as the sum of all the :shares values in each
-    # group. Any non-aggregate columns that have no aggregate function set
-    # default to using the aggregate function :first. Note that because of the
-    # way Ruby parses parameters to a method call, all the grouping symbols must
-    # appear first in the parameter list.
-    def group_by(*exprs)
-      group_cols = []
-      agg_cols = {}
-      exprs.each do |xp|
-        case xp
-        when Symbol
-          group_cols << xp
-        when Hash
-          agg_cols = xp
-        else
-          raise "Cannot group by parameter '#{xp}'"
-        end
-      end
-      default_agg_func = :first
-      default_cols = headers - group_cols - agg_cols.keys
-      default_cols.each do |h|
-        agg_cols[h] = default_agg_func
-      end
-
-      sorted_tab = order_by(group_cols)
-      groups = sorted_tab.rows.group_by do |r|
-        group_cols.map { |k| r[k] }
-      end
-      result_rows = []
-      groups.each_pair do |_vals, grp_rows|
-        result_rows << row_from_group(grp_rows, group_cols, agg_cols)
-      end
-      result = Table.new
-      result_rows.each do |row|
-        result << row
-      end
-      result
-    end
-
-    private
-
     def row_from_group(rows, grp_cols, agg_cols)
       new_row = {}
       grp_cols.each do |h|
@@ -663,38 +809,14 @@ module FatCore
 
     public
 
-    def add_footer(label: 'Total', aggregate: :sum, heads: [])
-      foot = {}
-      heads.each do |h|
-        raise "No #{h} column in table to #{aggregate}" unless headers.include?(h)
-        foot[h] = column(h).send(aggregate)
-      end
-      @footers[label.as_sym] = foot
-      self
-    end
-
-    def add_sum_footer(cols, label = 'Total')
-      add_footer(heads: cols)
-    end
-
-    def add_avg_footer(cols, label = 'Average')
-      add_footer(label: label, aggregate: :avg, heads: cols)
-    end
-
-    def add_min_footer(cols, label = 'Minimum')
-      add_footer(label: label, aggregate: :min, heads: cols)
-    end
-
-    def add_max_footer(cols, label = 'Maximum')
-      add_footer(label: label, aggregate: :max, heads: cols)
-    end
-
     # This returns the table as an Array of Arrays with formatting applied.
     # This would normally called after all calculations on the table are done
     # and you want to return the results.  The Array of Arrays structure is
     # what org-mode src blocks will render as an org table in the buffer.
     def to_org(formats: {})
       result = []
+
+      # Headers
       header_row = []
       headers.each do |hdr|
         header_row << hdr.entitle
@@ -703,6 +825,7 @@ module FatCore
       # This causes org to place an hline under the header row
       result << nil unless header_row.empty?
 
+      # Body
       rows.each do |row|
         out_row = []
         headers.each do |hdr|
@@ -710,6 +833,8 @@ module FatCore
         end
         result << out_row
       end
+
+      # Footers
       footers.each_pair do |label, footer|
         foot_row = []
         columns.each do |col|
@@ -726,17 +851,20 @@ module FatCore
     # Table construction methods.
     ############################################################################
 
-    # Add a row represented by a Hash having the headers as keys. All tables
-    # should be built ultimately using this method as a primitive.
-    def add_row(row)
+    # Add a row represented by a Hash having the headers as keys. If mark is
+    # true, mark this row as a boundary. All tables should be built ultimately
+    # using this method as a primitive.
+    def add_row(row, mark: false)
       row.each_pair do |k, v|
         key = k.as_sym
         columns << Column.new(header: k) unless column?(k)
         column(key) << v
       end
+      @boundaries << (size - 1) if mark
       self
     end
 
+    # Add a row without marking.
     def <<(row)
       add_row(row)
     end
@@ -753,24 +881,41 @@ module FatCore
     # respond to #to_hash.
     def from_array_of_hashes(rows)
       rows.each do |row|
+        if row.nil?
+          mark_boundary
+          next
+        end
         add_row(row.to_hash)
       end
       self
     end
 
+    # Construct a new table from an array of arrays. If the second element of
+    # the array is a nil, a string that looks like an hrule, or an array whose
+    # first element is a string that looks like an hrule, interpret the first
+    # element of the array as a row of headers. Otherwise, synthesize headers of
+    # the form "col1", "col2", ... and so forth. The remaining elements are
+    # taken as the body of the table, except that if an element of the outer
+    # array is a nil or a string that looks like an hrule, mark the preceding
+    # row as a boundary.
     def from_array_of_arrays(rows)
+      hrule_re = /\A\s*\|[-+]+/
       headers = []
-      if rows[0].any? { |itm| itm.to_s.number? }
-        headers = (1..rows[0].size).to_a.map { |k| "col#{k}".as_sym }
-        first_data_row = 0
-      else
+      if rows[1].nil? || rows[1] =~ hrule_re || rows[1].first =~ hrule_re
+        # Take the first row as headers
         # Use first row 0 as headers
         headers = rows[0].map(&:as_sym)
-        first_data_row = 1
+        first_data_row = 2
+      else
+        # Synthesize headers
+        headers = (1..rows[0].size).to_a.map { |k| "col#{k}".as_sym }
+        first_data_row = 0
       end
-      hrule_re = /\A\s*\|[-+]+/
       rows[first_data_row..-1].each do |row|
-        next if row[0] =~ hrule_re
+        if row.nil? || row[0] =~ hrule_re
+          mark_boundary
+          next
+        end
         row = row.map { |s| s.to_s.strip }
         hash_row = Hash[headers.zip(row)]
         add_row(hash_row)
@@ -797,18 +942,27 @@ module FatCore
         unless table_found
           # Skip through the file until a table is found
           next unless line =~ table_re
+          unless line =~ hrule_re
+            line = line.sub(/\A\s*\|/, '').sub(/\|\s*\z/, '')
+            rows << line.split('|').map(&:clean)
+          end
           table_found = true
+          next
         end
         break unless line =~ table_re
         if !header_found && line =~ hrule_re
+          rows << nil
           header_found = true
           next
         elsif header_found && line =~ hrule_re
+          # Mark the boundary with a nil
+          rows << nil
+        elsif line !~ table_re
           # Stop reading at the second hline
           break
         else
           line = line.sub(/\A\s*\|/, '').sub(/\|\s*\z/, '')
-          rows << line.split('|')
+          rows << line.split('|').map(&:clean)
         end
       end
       from_array_of_arrays(rows)

data/lib/fat_core/version.rb

@@ -1,7 +1,7 @@
 module FatCore
   MAJOR = 1
   MINOR = 5
-  PATCH = 1
+  PATCH = 2
 
   VERSION = [MAJOR, MINOR, PATCH].compact.join('.')
 end

data/spec/lib/table_spec.rb

@@ -124,6 +124,33 @@ EOS
 |  42 | 2013-05-30 | S    |   6,679 |     18 | 25.04710 | ZMEAC  |
 
 * Another Heading
+EOS
+
+      @org_file_body_with_groups = <<EOS
+
+#+TBLNAME: morgan_tab
+|-----+------------+------+---------+--------+----------+--------|
+| Ref |       Date | Code |     Raw | Shares |    Price | Info   |
+|-----+------------+------+---------+--------+----------+--------|
+|  29 | 2013-05-02 | P    | 795,546 |  2,609 |  1.18500 | ZMPEF1 |
+|-----+------------+------+---------+--------+----------+--------|
+|  30 | 2013-05-02 | P    | 118,186 |    388 | 11.85000 | ZMPEF1 |
+|  31 | 2013-05-02 | P    | 340,948 |  1,926 |  1.18500 | ZMPEF2 |
+|  32 | 2013-05-02 | P    |  50,651 |    286 | 11.85000 | ZMPEF2 |
+|-----+------------+------+---------+--------+----------+--------|
+|  33 | 2013-05-20 | S    |  12,000 |     32 | 28.28040 | ZMEAC  |
+|  34 | 2013-05-20 | S    |  85,000 |    226 | 28.32240 | ZMEAC  |
+|  35 | 2013-05-20 | S    |  33,302 |     88 | 28.63830 | ZMEAC  |
+|  36 | 2013-05-23 | S    |   8,000 |     21 | 27.10830 | ZMEAC  |
+|  37 | 2013-05-23 | S    |  23,054 |     61 | 26.80150 | ZMEAC  |
+|  38 | 2013-05-23 | S    |  39,906 |    106 | 25.17490 | ZMEAC  |
+|  39 | 2013-05-29 | S    |  13,459 |     36 | 24.74640 | ZMEAC  |
+|-----+------------+------+---------+--------+----------+--------|
+|  40 | 2013-05-29 | S    |  15,700 |     42 | 24.77900 | ZMEAC  |
+|  41 | 2013-05-29 | S    |  15,900 |     42 | 24.58020 | ZMEAC  |
+|  42 | 2013-05-30 | S    |   6,679 |     18 | 25.04710 | ZMEAC  |
+|-----+------------+------+---------+--------+----------+--------|
+
 EOS
     end
 
@@ -171,6 +198,28 @@ EOS
         end
       end
 
+      it 'should be create-able from an Org IO object with groups' do
+        tab = Table.new(StringIO.new(@org_file_body), '.org')
+        expect(tab.class).to eq(Table)
+        expect(tab.rows.size).to be > 10
+        expect(tab.headers.sort)
+          .to eq [:code, :date, :info, :price, :raw, :ref, :shares]
+        tab.rows.each do |row|
+          row.each_pair do |k, _v|
+            expect(k.class).to eq Symbol
+          end
+          expect(row[:code].class).to eq String
+          expect(row[:date].class).to eq Date
+          expect(row[:shares].is_a?(Numeric)).to be true
+          unless row[:rawshares].nil?
+            expect(row[:rawshares].is_a?(Numeric)).to be true
+          end
+          expect(row[:price].is_a?(BigDecimal)).to be true
+          expect([Numeric, String].any? { |t| row[:ref].is_a?(t) }).to be true
+          expect(row[:info].class).to eq String
+        end
+      end
+
       it 'should be create-able from a CSV file' do
         File.open('/tmp/junk.csv', 'w') { |f| f.write(@csv_file_body) }
         tab = Table.new('/tmp/junk.csv')
@@ -241,9 +290,10 @@ EOS
         end
       end
 
-      it 'should be create-able from an Array of Arrays with header no hrule' do
+      it 'should be create-able from an Array of Arrays with nil-marked header' do
         aoa = [
           ['First', 'Second', 'Third'],
+          nil,
           ['1', '2', '3.2'],
           ['4', '5', '6.4'],
           ['7', '8', '9.0'],
@@ -633,6 +683,7 @@ EOS
       it 'should select by boolean columns' do
         tab =
           [['Ref', 'Date', 'Code', 'Raw', 'Shares', 'Price', 'Info', 'Bool'],
+           nil,
            [1, '2013-05-02', 'P', 795_546.20, 795_546.2, 1.1850, 'ZMPEF1', 'T'],
            [2, '2013-05-02', 'P', 118_186.40, 118_186.4, 11.8500, 'ZMPEF1', 'T'],
            [7, '2013-05-20', 'S', 12_000.00, 5046.00, 28.2804, 'ZMEAC', 'F'],
@@ -770,6 +821,168 @@ EOS
       end
     end
 
+    describe 'group boundaries' do
+      before :all do
+        @tab_a = Table.new([
+          { id: 1, name: 'Paul', age: 32, address: 'California', salary: 20000, join_date: '2001-07-13' },
+          { id: 3, name: 'Teddy', age: 23, address: 'Norway', salary: 20000},
+          { id: 4, name: 'Mark', age: 25, address: 'Rich-Mond', salary: 65000, join_date: '2007-12-13' },
+          { id: 5, name: 'David', age: 27, address: 'Texas', salary: 85000, join_date: '2007-12-13' },
+          { id: 2, name: 'Allen', age: 25, address: 'Texas', salary: nil, join_date: '2007-12-13' },
+          { id: 8, name: 'Paul', age: 24, address: 'Houston', salary: 20000, join_date: '2005-07-13' },
+          { id: 9, name: 'James', age: 44, address: 'Norway', salary: 5000, join_date: '2005-07-13' },
+          { id: 10, name: 'James', age: 45, address: 'Texas', salary: 5000, join_date: '2005-07-13' }
+        ])
+        # Union compatible with tab_a
+        @tab_a1 = Table.new([
+          { id: 21, name: 'Paula', age: 23, address: 'Kansas', salary: 20000, join_date: '2001-07-13' },
+          { id: 23, name: 'Jenny', age: 32, address: 'Missouri', salary: 20000},
+          { id: 24, name: 'Forrest', age: 52, address: 'Richmond', salary: 65000, join_date: '2007-12-13' },
+          { id: 25, name: 'Syrano', age: 72, address: 'Nebraska', salary: 85000, join_date: '2007-12-13' },
+          # Next four are the same as row as in @tab_a
+          { id: 2, name: 'Allen', age: 25, address: 'Texas', salary: nil, join_date: '2007-12-13' },
+          { id: 8, name: 'Paul', age: 24, address: 'Houston', salary: 20000, join_date: '2005-07-13' },
+          { id: 9, name: 'James', age: 44, address: 'Norway', salary: 5000, join_date: '2005-07-13' },
+          { id: 10, name: 'James', age: 45, address: 'Texas', salary: 5000, join_date: '2005-07-13' },
+          { id: 22, name: 'Paula', age: 52, address: 'Iowa', salary: nil, join_date: '2007-12-13' },
+          { id: 28, name: 'Paula', age: 42, address: 'Oklahoma', salary: 20000, join_date: '2005-07-13' },
+          { id: 29, name: 'Patrick', age: 44, address: 'Lindsbourg', salary: 5000, join_date: '2005-07-13' },
+          { id: 30, name: 'James', age: 54, address: 'Ottawa', salary: 5000, join_date: '2005-07-13' }
+        ])
+        @tab_b = Table.new([
+          { id: 1, dept: 'IT Billing', emp_id: 1 },
+          { id: 2, dept: 'Engineering', emp_id: 2 },
+          { id: 3, dept: 'Finance', emp_id: 7 }
+        ])
+        @aoa =
+          [['Ref', 'Date', 'Code', 'Raw', 'Shares', 'Price', 'Info', 'Bool'],
+           nil,
+           [1, '2013-05-02', 'P', 795_546.20, 795_546.2, 1.1850, 'ZMPEF1', 'T'],
+           nil,
+           [2, '2013-05-02', 'P', 118_186.40, 118_186.4, 11.8500, 'ZMPEF1', 'T'],
+           [7, '2013-05-20', 'S', 12_000.00, 5046.00, 28.2804, 'ZMEAC', 'F'],
+           [8, '2013-05-20', 'S', 85_000.00, 35_742.50, 28.3224, 'ZMEAC', 'T'],
+           nil,
+           [9, '2013-05-20', 'S', 33_302.00, 14_003.49, 28.6383, 'ZMEAC', 'T'],
+           [10, '2013-05-23', 'S', 8000.00, 3364.00, 27.1083, 'ZMEAC', 'T'],
+           [11, '2013-05-23', 'S', 23_054.00, 9694.21, 26.8015, 'ZMEAC', 'F'],
+           [12, '2013-05-23', 'S', 39_906.00, 16_780.47, 25.1749, 'ZMEAC', 'T'],
+           [13, '2013-05-29', 'S', 13_459.00, 5659.51, 24.7464, 'ZMEAC', 'T'],
+           [14, '2013-05-29', 'S', 15_700.00, 6601.85, 24.7790, 'ZMEAC', 'F'],
+           [15, '2013-05-29', 'S', 15_900.00, 6685.95, 24.5802, 'ZMEAC', 'T'],
+           nil,
+           [16, '2013-05-30', 'S', 6_679.00, 2808.52, 25.0471, 'ZMEAC', 'T']]
+        @aoh = [
+          { id: 1, name: 'Paul', age: 32, address: 'California', salary: 20000, join_date: '2001-07-13' },
+          nil,
+          { id: 3, name: 'Teddy', age: 23, address: 'Norway', salary: 20000},
+          { id: 4, name: 'Mark', age: 25, address: 'Rich-Mond', salary: 65000, join_date: '2007-12-13' },
+          { id: 5, name: 'David', age: 27, address: 'Texas', salary: 85000, join_date: '2007-12-13' },
+          nil,
+          { id: 2, name: 'Allen', age: 25, address: 'Texas', salary: nil, join_date: '2007-12-13' },
+          { id: 8, name: 'Paul', age: 24, address: 'Houston', salary: 20000, join_date: '2005-07-13' },
+          { id: 9, name: 'James', age: 44, address: 'Norway', salary: 5000, join_date: '2005-07-13' },
+          nil,
+          { id: 10, name: 'James', age: 45, address: 'Texas', salary: 5000, join_date: '2005-07-13' }
+        ]
+      end
+
+      it 'an empty table should have no groups' do
+        expect(Table.new.groups.size).to eq(0)
+      end
+
+      it 'default group boundaries of whole table' do
+        expect(@tab_a.groups.size).to eq(1)
+      end
+
+      it 'add group boundaries on reading from org text' do
+        tab = Table.new(StringIO.new(@org_file_body_with_groups), '.org')
+        expect(tab.groups.size).to eq(4)
+        expect(tab.groups[0].size).to eq(1)
+        expect(tab.groups[1].size).to eq(3)
+        expect(tab.groups[2].size).to eq(7)
+        expect(tab.groups[3].size).to eq(3)
+      end
+
+      it 'add group boundaries on reading from aoa' do
+        tab = Table.new(@aoa)
+        expect(tab.groups.size).to eq(4)
+        expect(tab.groups[0].size).to eq(1)
+        expect(tab.groups[1].size).to eq(3)
+        expect(tab.groups[2].size).to eq(7)
+        expect(tab.groups[3].size).to eq(1)
+      end
+
+      it 'add group boundaries on reading from aoh' do
+        tab = Table.new(@aoh)
+        expect(tab.groups.size).to eq(4)
+        expect(tab.groups[0].size).to eq(1)
+        expect(tab.groups[1].size).to eq(3)
+        expect(tab.groups[2].size).to eq(3)
+        expect(tab.groups[3].size).to eq(1)
+      end
+
+      it 'add group boundaries on order_by' do
+        tab = @tab_a.order_by(:name)
+        # Now the table is ordered by name, and the names are: Allen, David,
+        # James, James, Mark, Paul, Paul, Teddy. So there are groups of size 1,
+        # 1, 2, 1, 2, and 1.  Six groups in all.
+        expect(tab.groups.size).to eq(6)
+        expect(tab.groups[0].size).to eq(1)
+        expect(tab.groups[1].size).to eq(1)
+        expect(tab.groups[2].size).to eq(2)
+        tab.groups[2].each do |row|
+          expect(row[:name]).to eq('James')
+        end
+        expect(tab.groups[3].size).to eq(1)
+        expect(tab.groups[4].size).to eq(2)
+        tab.groups[4].each do |row|
+          expect(row[:name]).to eq('Paul')
+        end
+        expect(tab.groups[5].size).to eq(1)
+      end
+
+      it 'add group boundaries on union_all' do
+        tab = @tab_a.union_all(@tab_a1)
+        expect(tab.size).to eq(20)
+        expect(tab.groups.size).to eq(2)
+        expect(tab.groups[0].size).to eq(8)
+        expect(tab.groups[1].size).to eq(12)
+      end
+
+      it 'inherit group boundaries on union_all' do
+        tab1 = @tab_a.order_by(:name)
+        tab2 = @tab_a1.order_by(:name)
+        tab = tab1.union_all(tab2)
+        expect(tab.size).to eq(20)
+        expect(tab.groups.size).to eq(tab1.groups.size + tab2.groups.size)
+        tab.groups.each do |grp|
+          names = grp.map {|r| r[:name]}
+          expect(names.uniq.size).to eq(1)
+        end
+      end
+
+      it 'inherit group boundaries on select' do
+        tab = @tab_a.order_by(:name).select(:name, :age, :join_date)
+        # Now the table is ordered by name, and the names are: Allen, David,
+        # James, James, Mark, Paul, Paul, Teddy. So there are groups of size 1,
+        # 1, 2, 1, 2, and 1.  Six groups in all.
+        expect(tab.groups.size).to eq(6)
+        expect(tab.groups[0].size).to eq(1)
+        expect(tab.groups[1].size).to eq(1)
+        expect(tab.groups[2].size).to eq(2)
+        tab.groups[2].each do |row|
+          expect(row[:name]).to eq('James')
+        end
+        expect(tab.groups[3].size).to eq(1)
+        expect(tab.groups[4].size).to eq(2)
+        tab.groups[4].each do |row|
+          expect(row[:name]).to eq('Paul')
+        end
+        expect(tab.groups[5].size).to eq(1)
+      end
+    end
+
     describe 'output' do
       it 'should be able to return itself as an array of arrays' do
         aoh = [
@@ -789,6 +1002,7 @@ EOS
         # blocks.
         tab =
           [['Ref', 'Date', 'Code', 'Raw', 'Shares', 'Price', 'Info', 'Bool'],
+           nil,
            [1, '2013-05-02', 'P', 795_546.20, 795_546.2, 1.1850, 'ZMPEF1', 'T'],
            [2, '2013-05-02', 'P', 118_186.40, 118_186.4, 11.8500, 'ZMPEF1', 'T'],
            [7, '2013-05-20', 'S', 12_000.00, 5046.00, 28.2804, 'ZMEAC', 'F'],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fat_core
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors:
 - Daniel E. Doherty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-03 00:00:00.000000000 Z
+date: 2017-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov