fat_table 0.3.4 → 0.5.2

data/lib/fat_table/formatters/latex_formatter.rb

@@ -91,7 +91,7 @@ module FatTable
         else
           ''
         end
-      result += "\\usepackage[pdftex,x11names]{xcolor}\n"
+      result += "\\usepackage[pdftex,table,x11names]{xcolor}\n"
       result
     end
 
@@ -113,12 +113,14 @@ module FatTable
     # default.
     def decorate_string(str, istruct)
       str = quote(str)
-      result = ''
-      result += '\\bfseries{}' if istruct.bold
-      result += '\\itshape{}' if istruct.italic
-      result += "\\color{#{istruct.color}}" if istruct.color &&
-                                               istruct.color != 'none'
-      result = "#{result}#{str}"
+      result = istruct.italic ? "\\itshape{#{str}}" : str
+      result = istruct.bold ? "\\bfseries{#{result}}" : result
+      if istruct.color && istruct.color != 'none'
+        result = "{\\textcolor{#{istruct.color}}{#{result}}}"
+      end
+      if istruct.bgcolor && istruct.bgcolor != 'none'
+        result = "\\cellcolor{#{istruct.bgcolor}}#{result}"
+      end
       unless istruct.alignment == format_at[:body][istruct._h].alignment
         ac = alignment_code(istruct.alignment)
         result = "\\multicolumn{1}{#{ac}}{#{result}}"

data/lib/fat_table/table.rb

@@ -53,7 +53,14 @@ module FatTable
   class Table
     # An Array of FatTable::Columns that constitute the table.
     attr_reader :columns
-    attr_accessor :boundaries
+
+    # Record boundaries set explicitly with mark_boundaries or from reading
+    # hlines from input.  When we want to access boundaries, however, we want
+    # to add an implict boundary at the last row of the table.  Since, as the
+    # table grows, the implict boundary changes index, we synthesize the
+    # boundaries by dynamically adding the final boundary with the #boundaries
+    # method call.
+    attr_accessor :explicit_boundaries
 
     ###########################################################################
     # Constructors
@@ -62,9 +69,30 @@ module FatTable
     # :category: Constructors
 
     # Return an empty FatTable::Table object.
-    def initialize
+    def initialize(*heads)
       @columns = []
-      @boundaries = []
+      @explicit_boundaries = []
+      unless heads.empty?
+        heads.each do |h|
+          @columns << Column.new(header: h)
+        end
+      end
+    end
+
+    # :category: Constructors
+
+    # Return an empty duplicate of self.  This allows the library to create an
+    # empty table that preserves all the instance variables from self.  Even
+    # though FatTable::Table objects have no instance variables, a class that
+    # inherits from it might.
+    def empty_dup
+      self.dup.__empty!
+    end
+
+    def __empty!
+      @columns = []
+      @explicit_boundaries = []
+      self
     end
 
     # :category: Constructors
@@ -188,6 +216,7 @@ module FatTable
           end
           result << hsh.to_h
         end
+        result.normalize_boundaries
         result
       end
 
@@ -236,6 +265,7 @@ module FatTable
           hash_row = Hash[headers.zip(row)]
           result << hash_row
         end
+        result.normalize_boundaries
         result
       end
 
@@ -245,6 +275,7 @@ module FatTable
                   skip_blanks: true).each do |row|
           result << row.to_h
         end
+        result.normalize_boundaries
         result
       end
 
@@ -317,8 +348,11 @@ module FatTable
     # Set the column type for Column with the given +key+ as a String type,
     # but only if empty.  Otherwise, we would have to worry about converting
     # existing items in the column to String.  Perhaps that's a TODO.
-    def set_column_to_string_type(key)
-      column(key).force_to_string_type
+    def force_string!(*keys)
+      keys.each do |h|
+        column(h).force_string!
+      end
+      self
     end
 
     # :category: Attributes
@@ -426,8 +460,6 @@ module FatTable
     # large table, that would require that we construct all the rows for a range
     # of any size.
     def rows_range(first = 0, last = nil) # :nodoc:
-      last ||= size - 1
-      last = [last, 0].max
       raise UserError, 'first must be <= last' unless first <= last
 
       rows = []
@@ -473,6 +505,8 @@ module FatTable
     # the headers from the body) marks a boundary for the row immediately
     # preceding the hline.
     #
+    # Boundaries can also be added manually with the +mark_boundary+ method.
+    #
     # The #order_by method resets the boundaries then adds boundaries at the
     # last row of each group of rows on which the sort keys were equal as a
     # boundary.
@@ -506,6 +540,42 @@ module FatTable
       groups
     end
 
+    # Return the number of groups in the table.
+    def number_of_groups
+      empty? ? 0 : boundaries.size
+    end
+
+    # Return the range of row indexes for boundary number +k+
+    def group_row_range(k)
+      last_k = boundaries.size - 1
+      if k < 0 || k > last_k
+        raise ArgumentError, "boundary number '#{k}' out of range in boundary_row_range"
+      end
+
+      if boundaries.size == 1
+        (0..boundaries.first)
+      elsif k.zero?
+        # Keep index at or above zero
+        (0..boundaries[k])
+      else
+        ((boundaries[k - 1] + 1)..boundaries[k])
+      end
+    end
+
+    # Return an Array of Column objects for header +col+ representing a
+    # sub-column for each group in the table under that header.
+    def group_cols(col)
+      normalize_boundaries
+      cols = []
+      (0..boundaries.size - 1).each do |k|
+        range = group_row_range(k)
+        tab_col = column(col)
+        gitems = tab_col.items[range]
+        cols << Column.new(header: col, items: gitems, type: tab_col.type)
+      end
+      cols
+    end
+
     # :category: Operators
 
     # Return this table mutated with all groups removed. Useful after something
@@ -513,56 +583,99 @@ module FatTable
     # the groups displayed in the output. This modifies the input table, so is a
     # departure from the otherwise immutability of Tables.
     def degroup!
-      @boundaries = []
+      self.explicit_boundaries = []
       self
     end
 
     # Mark a group boundary at row +row+, and if +row+ is +nil+, mark the last
-    # row in the table as a group boundary. This is mainly used for internal
-    # purposes.
-    def mark_boundary(row = nil) # :nodoc:
-      if row
-        boundaries.push(row)
-      else
-        boundaries.push(size - 1)
+    # row in the table as a group boundary.  An attempt to add a boundary to
+    # an empty table has no effect.  We adopt the convention that the last row
+    # of the table always marks an implicit boundary even if it is not in the
+    # @explicit_boundaries array.  When we "mark" a boundary, we intend it to
+    # be an explicit boundary, even if it marks the last row of the table.
+    def mark_boundary(row_num = nil)
+      return self if empty?
+
+      if row_num
+        unless row_num < size
+          raise ArgumentError, "can't mark boundary at row #{row_num}, last row is #{size - 1}"
+        end
+        unless row_num >= 0
+          raise ArgumentError, "can't mark boundary at non-positive row #{row_num}"
+        end
+        explicit_boundaries.push(row_num)
+      elsif size > 0
+        explicit_boundaries.push(size - 1)
       end
+      normalize_boundaries
+      self
     end
 
-    protected
-
     # :stopdoc:
 
     # Make sure size - 1 is last boundary and that they are unique and sorted.
     def normalize_boundaries
       unless empty?
-        boundaries.push(size - 1) unless boundaries.include?(size - 1)
-        self.boundaries = boundaries.uniq.sort
+        self.explicit_boundaries = explicit_boundaries.uniq.sort
       end
-      boundaries
+      explicit_boundaries
     end
 
+    # Return the explicit_boundaries, augmented by an implicit boundary for
+    # the end of the table, unless it's already an implicit boundary.
+    def boundaries
+      return [] if empty?
+
+      if explicit_boundaries.last == size - 1
+        explicit_boundaries
+      else
+        explicit_boundaries + [size - 1]
+      end
+    end
+
+    protected
+
     # Concatenate the array of argument bounds to this table's boundaries, but
     # increase each of the indexes in bounds by shift. This is used in the
     # #union_all method.
     def append_boundaries(bounds, shift: 0)
-      @boundaries += bounds.map { |k| k + shift }
+      @explicit_boundaries += bounds.map { |k| k + shift }
     end
 
-    # Return the group number to which row ~row~ belongs. Groups, from the
-    # user's point of view are indexed starting at 1.
-    def row_index_to_group_index(row)
+    # Return the group number to which row ~row_num~ belongs. Groups, from the
+    # user's point of view are indexed starting at 0.
+    def row_index_to_group_index(row_num)
       boundaries.each_with_index do |b_last, g_num|
-        return (g_num + 1) if row <= b_last
+        return (g_num + 1) if row_num <= b_last
+      end
+      0
+    end
+
+    # Return the index of the first row in group number +grp_num+
+    def first_row_num_in_group(grp_num)
+      if grp_num >= boundaries.size || grp_num < 0
+        raise ArgumentError, "group number #{grp_num} out of bounds"
+      end
+
+      grp_num.zero? ? 0 : boundaries[grp_num - 1] + 1
+    end
+
+    # Return the index of the last row in group number +grp_num+
+    def last_row_num_in_group(grp_num)
+      if grp_num > boundaries.size || grp_num < 0
+        raise ArgumentError, "group number #{grp_num} out of bounds"
+      else
+        boundaries[grp_num]
       end
-      1
     end
 
-    def group_rows(row) # :nodoc:
+    # Return the rows for group number +grp_num+.
+    def group_rows(grp_num) # :nodoc:
       normalize_boundaries
-      return [] unless row < boundaries.size
+      return [] unless grp_num < boundaries.size
 
-      first = row.zero? ? 0 : boundaries[row - 1] + 1
-      last = boundaries[row]
+      first = first_row_num_in_group(grp_num)
+      last = last_row_num_in_group(grp_num)
       rows_range(first, last)
     end
 
@@ -587,22 +700,43 @@ module FatTable
     # After sorting, the output Table will have group boundaries added after
     # each row where the sort key changes.
     def order_by(*sort_heads)
-      sort_heads = [sort_heads].flatten
-      rev_heads = sort_heads.select { |h| h.to_s.ends_with?('!') }
-      sort_heads = sort_heads.map { |h| h.to_s.sub(/\!\z/, '').to_sym }
-      rev_heads = rev_heads.map { |h| h.to_s.sub(/\!\z/, '').to_sym }
+      # Sort the rows in order and add to new_rows.
+      key_hash = partition_sort_keys(sort_heads)
       new_rows = rows.sort do |r1, r2|
-        key1 = sort_heads.map { |h| rev_heads.include?(h) ? r2[h] : r1[h] }
-        key2 = sort_heads.map { |h| rev_heads.include?(h) ? r1[h] : r2[h] }
-        key1 <=> key2
+        # Set the sort keys based on direction
+        key1 = []
+        key2 = []
+        key_hash.each_pair do |h, dir|
+          if dir == :forward
+            key1 << r1[h]
+            key2 << r2[h]
+          else
+            key1 << r2[h]
+            key2 << r1[h]
+          end
+        end
+        # Make any booleans comparable with <=>
+        key1 = key1.map_booleans
+        key2 = key2.map_booleans
+
+        # If there are any nils, <=> will return nil, and we have to use the
+        # special comparison method, compare_with_nils, instead.
+        result = (key1 <=> key2)
+        result.nil? ? compare_with_nils(key1, key2) : result
       end
-      # Add the new rows to the table, but mark a group boundary at the points
-      # where the sort key changes value.
-      new_tab = Table.new
+
+      # Add the new_rows to the table, but mark a group boundary at the points
+      # where the sort key changes value.  NB: I use self.class.new here
+      # rather than Table.new because if this class is inherited, I want the
+      # new_tab to be an instance of the subclass.  With Table.new, this
+      # method's result will be an instance of FatTable::Table rather than of
+      # the subclass.
+      new_tab = empty_dup
       last_key = nil
       new_rows.each_with_index do |nrow, k|
         new_tab << nrow
-        key = nrow.fetch_values(*sort_heads)
+        # key = nrow.fetch_values(*sort_heads)
+        key = nrow.fetch_values(*key_hash.keys)
         new_tab.mark_boundary(k - 1) if last_key && key != last_key
         last_key = key
       end
@@ -610,6 +744,33 @@ module FatTable
       new_tab
     end
 
+    # :category: Operators
+
+    # Return a new Table sorting the rows of this Table on an any expression
+    # +expr+ that is valid with the +select+ method, except that they
+    # expression may end with an exclamation mark +!+ to indicate a reverse
+    # sort.  The new table will have an additional column called +sort_key+
+    # populated with the result of evaluating the given expression and will be
+    # sorted (or reverse sorted) on that column.
+    #
+    #   tab.order_with('date.year') => table sorted by date's year
+    #   tab.order_with('date.year!') => table reverse sorted by date's year
+    #
+    # After sorting, the output Table will have group boundaries added after
+    # each row where the sort key changes.
+    def order_with(expr)
+      unless expr.is_a?(String)
+        raise "must call FatTable::Table\#order_with with a single string expression"
+      end
+      rev = false
+      if expr.match?(/\s*!\s*\z/)
+        rev = true
+        expr = expr.sub(/\s*!\s*\z/, '')
+      end
+      sort_sym = rev ? :sort_key! : :sort_key
+      dup.select(*headers, sort_key: expr).order_by(sort_sym)
+    end
+
     # :category: Operators
     #
     # Return a Table having the selected column expressions. Each expression can
@@ -713,7 +874,7 @@ module FatTable
                          before: before_hook,
                          after: after_hook)
       # Compute the new Table from this Table
-      result = Table.new
+      result = empty_dup
       normalize_boundaries
       rows.each_with_index do |old_row, old_k|
         # Set the group number in the before hook and run the hook with the
@@ -723,7 +884,15 @@ module FatTable
         ev.eval_before_hook(locals: old_row)
         # Compute the new row.
         new_row = {}
-        cols.each do |k|
+        # Allow the :omni col to stand for all columns if it is alone and
+        # first.
+        cols_to_include =
+          if cols.size == 1 && cols.first.as_sym == :omni
+            headers
+          else
+            cols
+          end
+        cols_to_include.each do |k|
           h = k.as_sym
           msg = "Column '#{h}' in select does not exist"
           raise UserError, msg unless column?(h)
@@ -752,7 +921,7 @@ module FatTable
         ev.eval_after_hook(locals: new_row)
         result << new_row
       end
-      result.boundaries = boundaries
+      result.explicit_boundaries = explicit_boundaries
       result.normalize_boundaries
       result
     end
@@ -770,7 +939,7 @@ module FatTable
     #   tab.where('@row.even? && shares > 500') => even rows with lots of shares
     def where(expr)
       expr = expr.to_s
-      result = Table.new
+      result = empty_dup
       headers.each do |h|
         col = Column.new(header: h)
         result.add_column(col)
@@ -792,7 +961,7 @@ module FatTable
     # Return a new table with all duplicate rows eliminated. Resets groups. Same
     # as #uniq.
     def distinct
-      result = Table.new
+      result = empty_dup
       uniq_rows = rows.uniq
       uniq_rows.each do |row|
         result << row
@@ -889,38 +1058,6 @@ module FatTable
       set_operation(other, :difference, distinct: false)
     end
 
-    private
-
-    # Apply the set operation given by ~oper~ between this table and the other
-    # table given in the first argument.  If distinct is true, eliminate
-    # duplicates from the result.
-    def set_operation(other, oper = :+, distinct: true, add_boundaries: true, inherit_boundaries: false)
-      unless columns.size == other.columns.size
-        msg = "can't apply set ops to tables with a different number of columns"
-        raise UserError, msg
-      end
-      unless columns.map(&:type) == other.columns.map(&:type)
-        msg = "can't apply a set ops to tables with different column types."
-        raise UserError, msg
-      end
-      other_rows = other.rows.map { |r| r.replace_keys(headers) }
-      result = Table.new
-      new_rows = rows.send(oper, other_rows)
-      new_rows.each_with_index do |row, k|
-        result << row
-        result.mark_boundary if k == size - 1 && add_boundaries
-      end
-      if inherit_boundaries
-        result.boundaries = normalize_boundaries
-        other.normalize_boundaries
-        result.append_boundaries(other.boundaries, shift: size)
-      end
-      result.normalize_boundaries
-      distinct ? result.distinct : result
-    end
-
-    public
-
     # An Array of symbols for the valid join types.
     JOIN_TYPES = %i[inner left right full cross].freeze
 
@@ -1011,7 +1148,7 @@ module FatTable
       join_exp, other_common_heads =
         build_join_expression(exps, other, join_type)
       ev = Evaluator.new
-      result = Table.new
+      result = empty_dup
       other_rows = other.rows
       other_row_matches = Array.new(other_rows.size, false)
       rows.each do |self_row|
@@ -1029,14 +1166,14 @@ module FatTable
                                   type: join_type)
           result << out_row
         end
-        next unless %i[left full].include?(join_type)
+        next unless [:left, :full].include?(join_type)
         next if self_row_matched
 
         result << build_out_row(row_a: self_row,
                                 row_b: other_row_nils,
                                 type: join_type)
       end
-      if %i[right full].include?(join_type)
+      if [:right, :full].include?(join_type)
         other_rows.each_with_index do |other_row, k|
           next if other_row_matches[k]
 
@@ -1165,7 +1302,7 @@ module FatTable
                 partial_result = nil
               else
                 # First of a pair of _a or _b
-                partial_result = String.new("(#{a_head}_a == ")
+                partial_result = +"(#{a_head}_a == "
               end
               last_sym = a_head
             when /\A(?<sy>.*)_b\z/
@@ -1184,7 +1321,7 @@ module FatTable
                 partial_result = nil
               else
                 # First of a pair of _a or _b
-                partial_result = String.new("(#{b_head}_b == ")
+                partial_result = +"(#{b_head}_b == "
               end
               b_common_heads << b_head
               last_sym = b_head
@@ -1259,7 +1396,7 @@ module FatTable
       groups = sorted_tab.rows.group_by do |r|
         group_cols.map { |k| r[k] }
       end
-      result = Table.new
+      result = empty_dup
       groups.each_pair do |_vals, grp_rows|
         result << row_from_group(grp_rows, group_cols, agg_cols)
       end
@@ -1291,25 +1428,24 @@ module FatTable
 
     # :category: Constructors
 
-    # Add a group boundary mark at the given row, or at the end of the table
-    # by default.
-    def add_boundary(at_row = nil)
-      row = at_row || (size - 1)
-      @boundaries << row
-    end
-
-    # :category: Constructors
-
     # Add a +row+ represented by a Hash having the headers as keys. If +mark:+
     # is set true, mark this row as a boundary. All tables should be built
     # ultimately using this method as a primitive.
     def add_row(row, mark: false)
-      row.each_pair do |k, v|
-        key = k.as_sym
-        columns << Column.new(header: k) unless column?(k)
-        column(key) << v
+      row.transform_keys!(&:as_sym)
+      # Make sure there is a column for each known header and each new key
+      # present in row.
+      new_heads = row.keys - headers
+      new_heads.each do |h|
+        # This column is new, so it needs nil items for all prior rows lest
+        # the value be added to a prior row.
+        items = Array.new(size, nil)
+        columns << Column.new(header: h, items: items)
+      end
+      headers.each do |h|
+        # NB: This adds a nil if h is not in row.
+        column(h) << row[h]
       end
-      add_boundary if mark
       self
     end
 
@@ -1478,5 +1614,74 @@ module FatTable
       yield fmt if block_given?
       fmt.output
     end
+
+    private
+
+    # Apply the set operation given by ~oper~ between this table and the other
+    # table given in the first argument.  If distinct is true, eliminate
+    # duplicates from the result.
+    def set_operation(other, oper = :+, distinct: true, add_boundaries: true, inherit_boundaries: false)
+      unless columns.size == other.columns.size
+        msg = "can't apply set ops to tables with a different number of columns"
+        raise UserError, msg
+      end
+      unless columns.map(&:type) == other.columns.map(&:type)
+        msg = "can't apply a set ops to tables with different column types."
+        raise UserError, msg
+      end
+      other_rows = other.rows.map { |r| r.replace_keys(headers) }
+      result = empty_dup
+      new_rows = rows.send(oper, other_rows)
+      new_rows.each_with_index do |row, k|
+        result << row
+        result.mark_boundary if k == size - 1 && add_boundaries
+      end
+      if inherit_boundaries
+        result.explicit_boundaries = boundaries
+        result.append_boundaries(other.boundaries, shift: size)
+      end
+      result.normalize_boundaries
+      distinct ? result.distinct : result
+    end
+
+    # Return a hash with the key being the header to sort on and the value
+    # being either :forward or :reverse to indicate the sort order on that
+    # key.
+    def partition_sort_keys(keys)
+      result = {}
+      [keys].flatten.each do |h|
+        if h.to_s.match?(/\s*!\s*\z/)
+          result[h.to_s.sub(/\s*!\s*\z/, '').to_sym] = :reverse
+        else
+          result[h] = :forward
+        end
+      end
+      result
+    end
+
+    # The <=> operator cannot handle nils without some help.  Treat a nil as
+    # smaller than any other value, but equal to other nils.  The two keys are assumed to be arrays of values to be
+    # compared with <=>.
+    def compare_with_nils(key1, key2)
+      result = nil
+      key1.zip(key2) do |k1, k2|
+        if k1.nil? && k2.nil?
+          result = 0
+          next
+        elsif k1.nil?
+          result = -1
+          break
+        elsif k2.nil?
+          result = 1
+          break
+        elsif (k1 <=> k2) == 0
+          next
+        else
+          result = (k1 <=> k2)
+          break
+        end
+      end
+      result
+    end
   end
 end

data/lib/fat_table/version.rb

@@ -2,5 +2,5 @@
 
 module FatTable
   # The current version of FatTable
-  VERSION = '0.3.4'
+  VERSION = '0.5.2'
 end

data/lib/fat_table.rb

@@ -19,9 +19,12 @@ module FatTable
 
   require 'fat_table/version'
   require 'fat_table/patches'
+  require 'ext/array'
   require 'fat_table/evaluator'
+  require 'fat_table/convert'
   require 'fat_table/column'
   require 'fat_table/table'
+  require 'fat_table/footer'
   require 'fat_table/formatters'
   require 'fat_table/db_handle'
   require 'fat_table/errors'
@@ -58,8 +61,8 @@ module FatTable
 
   # Return an empty FatTable::Table object. You can use FatTable::Table#add_row
   # or FatTable::Table#add_column to populate the table with data.
-  def self.new
-    Table.new
+  def self.new(*args)
+    Table.new(*args)
   end
 
   # Construct a FatTable::Table from the contents of a CSV file given by the

data/md/README.md

@@ -1109,8 +1109,7 @@ will raise an exception.
 
 -   **`first`:** the first non-nil item in the column,
 -   **`last`:** the last non-nil item in the column,
--   **`rng`:** form a string of the form `"#{first}..#{last}"` to show the range of
-    values in the column,
+-   **`range`:** form a Range ~~{min}..{max}~ to show the range of values in the column,
 -   **`sum`:** for `Numeric` and `String` columns, apply ’+’ to all the non-nil
     values,
 -   **`count`:** the number of non-nil values in the column,