fat_table 0.4.0 → 0.5.3

data/lib/fat_table/table.rb

@@ -53,7 +53,17 @@ module FatTable
   class Table
     # An Array of FatTable::Columns that constitute the table.
     attr_reader :columns
-    attr_accessor :boundaries
+
+    # Record boundaries set explicitly with mark_boundaries or from reading
+    # hlines from input.  When we want to access boundaries, however, we want
+    # to add an implict boundary at the last row of the table.  Since, as the
+    # table grows, the implict boundary changes index, we synthesize the
+    # boundaries by dynamically adding the final boundary with the #boundaries
+    # method call.
+    attr_accessor :explicit_boundaries
+
+    # An Array of FatTable::Columns that should be tolerant.
+    attr_reader :tolerant_columns
 
     ###########################################################################
     # Constructors
@@ -61,19 +71,71 @@ module FatTable
 
     # :category: Constructors
 
-    # Return an empty FatTable::Table object.
-    def initialize
+    # Return an empty FatTable::Table object.  Specifying headers is optional.
+    # Any headers ending with a ! are marked as tolerant, in that, if an
+    # incompatible type is added to it, the column is re-typed as a String
+    # column, and construction proceeds.  The ! is stripped from the header to
+    # form the column key, though.  You can also provide the names of columns
+    # that should be tolerant by using the +tolerant_columns key-word to
+    # provide an array of headers that should be tolerant.  The special string
+    # '*' or the symbol :* indicates that all columns should be created
+    # tolerant.
+    def initialize(*heads, tolerant_columns: [])
+      @columns = []
+      @explicit_boundaries = []
+      @tolerant_columns =
+        case tolerant_columns
+        when Array
+          tolerant_columns.map { |h| h.to_s.as_sym }
+        when String
+          if tolerant_columns.strip == '*'
+            ['*'.to_sym]
+          else
+            [tolerant_columns.as_sym]
+          end
+        when Symbol
+          if tolerant_columns.to_s.strip == '*'
+            ['*'.to_sym]
+          else
+            [tolerant_columns.to_s.as_sym]
+          end
+        else
+          raise ArgumentError, "set tolerant_columns to String, Symbol, or an Array of either"
+        end
+      unless heads.empty?
+        heads.each do |h|
+          if h.to_s.end_with?('!') || @tolerant_columns.include?(h)
+            @columns << Column.new(header: h.to_s.sub(/!\s*\z/, ''), tolerant: true)
+          else
+            @columns << Column.new(header: h)
+          end
+        end
+      end
+    end
+
+    # :category: Constructors
+
+    # Return an empty duplicate of self.  This allows the library to create an
+    # empty table that preserves all the instance variables from self.  Even
+    # though FatTable::Table objects have no instance variables, a class that
+    # inherits from it might.
+    def empty_dup
+      self.dup.__empty!
+    end
+
+    def __empty!
       @columns = []
-      @boundaries = []
+      @explicit_boundaries = []
+      self
     end
 
     # :category: Constructors
 
     # Construct a Table from the contents of a CSV file named +fname+. Headers
     # will be taken from the first CSV row and converted to symbols.
-    def self.from_csv_file(fname)
+    def self.from_csv_file(fname, tolerant_columns: [])
       File.open(fname, 'r') do |io|
-        from_csv_io(io)
+        from_csv_io(io, tolerant_columns: tolerant_columns)
       end
     end
 
@@ -81,8 +143,8 @@ module FatTable
 
     # Construct a Table from a CSV string +str+, treated in the same manner as
     # the input from a CSV file in ::from_org_file.
-    def self.from_csv_string(str)
-      from_csv_io(StringIO.new(str))
+    def self.from_csv_string(str, tolerant_columns: [])
+      from_csv_io(StringIO.new(str), tolerant_columns: tolerant_columns)
     end
 
     # :category: Constructors
@@ -91,9 +153,9 @@ module FatTable
     # file named +fname+. Headers are taken from the first row if the second row
     # is an hrule. Otherwise, synthetic headers of the form +:col_1+, +:col_2+,
     # etc. are created.
-    def self.from_org_file(fname)
+    def self.from_org_file(fname, tolerant_columns: [])
       File.open(fname, 'r') do |io|
-        from_org_io(io)
+        from_org_io(io, tolerant_columns: tolerant_columns)
       end
     end
 
@@ -101,8 +163,8 @@ module FatTable
 
     # Construct a Table from a string +str+, treated in the same manner as the
     # contents of an org-mode file in ::from_org_file.
-    def self.from_org_string(str)
-      from_org_io(StringIO.new(str))
+    def self.from_org_string(str, tolerant_columns: [])
+      from_org_io(StringIO.new(str), tolerant_columns: tolerant_columns)
     end
 
     # :category: Constructors
@@ -121,8 +183,8 @@ module FatTable
     # :hlines no +) org-mode strips all hrules from the table; otherwise (+
     # HEADER: :hlines yes +) they are indicated with nil elements in the outer
     # array.
-    def self.from_aoa(aoa, hlines: false)
-      from_array_of_arrays(aoa, hlines: hlines)
+    def self.from_aoa(aoa, hlines: false, tolerant_columns: [])
+      from_array_of_arrays(aoa, hlines: hlines, tolerant_columns: tolerant_columns)
     end
 
     # :category: Constructors
@@ -132,9 +194,9 @@ module FatTable
     # keys, which, when converted to symbols will become the headers for the
     # Table. If hlines is set true, mark a group boundary whenever a nil, rather
     # than a hash appears in the outer array.
-    def self.from_aoh(aoh, hlines: false)
+    def self.from_aoh(aoh, hlines: false, tolerant_columns: [])
       if aoh.first.respond_to?(:to_h)
-        from_array_of_hashes(aoh, hlines: hlines)
+        from_array_of_hashes(aoh, hlines: hlines, tolerant_columns: tolerant_columns)
       else
         raise UserError,
               "Cannot initialize Table with an array of #{input[0].class}"
@@ -153,7 +215,7 @@ module FatTable
 
     # Construct a Table by running a SQL +query+ against the database set up
     # with FatTable.connect, with the rows of the query result as rows.
-    def self.from_sql(query)
+    def self.from_sql(query, tolerant_columns: [])
       msg = 'FatTable.db must be set with FatTable.connect'
       raise UserError, msg if FatTable.db.nil?
 
@@ -175,8 +237,8 @@ module FatTable
       # Construct table from an array of hashes or an array of any object that
       # can respond to #to_h. If an array element is a nil, mark it as a group
       # boundary in the Table.
-      def from_array_of_hashes(hashes, hlines: false)
-        result = new
+      def from_array_of_hashes(hashes, hlines: false, tolerant_columns: [])
+        result = new(tolerant_columns: tolerant_columns)
         hashes.each do |hsh|
           if hsh.nil?
             unless hlines
@@ -188,6 +250,7 @@ module FatTable
           end
           result << hsh.to_h
         end
+        result.normalize_boundaries
         result
       end
 
@@ -203,8 +266,8 @@ module FatTable
       # hlines are stripped from the table, otherwise (:hlines yes) they are
       # indicated with nil elements in the outer array as expected by this
       # method when hlines is set true.
-      def from_array_of_arrays(rows, hlines: false)
-        result = new
+      def from_array_of_arrays(rows, hlines: false, tolerant_columns: [])
+        result = new(tolerant_columns: tolerant_columns)
         headers = []
         if !hlines
           # Take the first row as headers
@@ -236,15 +299,17 @@ module FatTable
           hash_row = Hash[headers.zip(row)]
           result << hash_row
         end
+        result.normalize_boundaries
         result
       end
 
-      def from_csv_io(io)
-        result = new
+      def from_csv_io(io, tolerant_columns: [])
+        result = new(tolerant_columns: tolerant_columns)
         ::CSV.new(io, headers: true, header_converters: :symbol,
                   skip_blanks: true).each do |row|
           result << row.to_h
         end
+        result.normalize_boundaries
         result
       end
 
@@ -252,7 +317,7 @@ module FatTable
       # header row must be marked with an hline (i.e, a row that looks like
       # '|---+--...--|') and groups of rows may be marked with hlines to
       # indicate group boundaries.
-      def from_org_io(io)
+      def from_org_io(io, tolerant_columns: [])
         table_re = /\A\s*\|/
         hrule_re = /\A\s*\|[-+]+/
         rows = []
@@ -287,7 +352,7 @@ module FatTable
             rows << line.split('|').map(&:clean)
           end
         end
-        from_array_of_arrays(rows, hlines: true)
+        from_array_of_arrays(rows, hlines: true, tolerant_columns: tolerant_columns)
       end
     end
 
@@ -317,8 +382,11 @@ module FatTable
     # Set the column type for Column with the given +key+ as a String type,
     # but only if empty.  Otherwise, we would have to worry about converting
     # existing items in the column to String.  Perhaps that's a TODO.
-    def set_column_to_string_type(key)
-      column(key).force_to_string_type
+    def force_string!(*keys)
+      keys.each do |h|
+        column(h).force_string!
+      end
+      self
     end
 
     # :category: Attributes
@@ -378,6 +446,15 @@ module FatTable
 
     # :category: Attributes
 
+    # Return whether the column with the given head should be made tolerant.
+    def tolerant_col?(h)
+      return true if tolerant_columns.include?(:'*')
+
+      tolerant_columns.include?(h)
+    end
+
+    # :category: Attributes
+
     # Return the number of rows in the Table.
     def size
       return 0 if columns.empty?
@@ -426,8 +503,6 @@ module FatTable
     # large table, that would require that we construct all the rows for a range
     # of any size.
     def rows_range(first = 0, last = nil) # :nodoc:
-      last ||= size - 1
-      last = [last, 0].max
       raise UserError, 'first must be <= last' unless first <= last
 
       rows = []
@@ -473,6 +548,8 @@ module FatTable
     # the headers from the body) marks a boundary for the row immediately
     # preceding the hline.
     #
+    # Boundaries can also be added manually with the +mark_boundary+ method.
+    #
     # The #order_by method resets the boundaries then adds boundaries at the
     # last row of each group of rows on which the sort keys were equal as a
     # boundary.
@@ -506,6 +583,43 @@ module FatTable
       groups
     end
 
+    # Return the number of groups in the table.
+    def number_of_groups
+      empty? ? 0 : boundaries.size
+    end
+
+    # Return the range of row indexes for boundary number +k+
+    def group_row_range(k)
+      last_k = boundaries.size - 1
+      if k < 0 || k > last_k
+        raise ArgumentError, "boundary number '#{k}' out of range in boundary_row_range"
+      end
+
+      if boundaries.size == 1
+        (0..boundaries.first)
+      elsif k.zero?
+        # Keep index at or above zero
+        (0..boundaries[k])
+      else
+        ((boundaries[k - 1] + 1)..boundaries[k])
+      end
+    end
+
+    # Return an Array of Column objects for header +col+ representing a
+    # sub-column for each group in the table under that header.
+    def group_cols(col)
+      normalize_boundaries
+      cols = []
+      (0..boundaries.size - 1).each do |k|
+        range = group_row_range(k)
+        tab_col = column(col)
+        gitems = tab_col.items[range]
+        cols << Column.new(header: col, items: gitems,
+                           type: tab_col.type, tolerant: tab_col.tolerant?)
+      end
+      cols
+    end
+
     # :category: Operators
 
     # Return this table mutated with all groups removed. Useful after something
@@ -513,56 +627,99 @@ module FatTable
     # the groups displayed in the output. This modifies the input table, so is a
     # departure from the otherwise immutability of Tables.
     def degroup!
-      @boundaries = []
+      self.explicit_boundaries = []
       self
     end
 
     # Mark a group boundary at row +row+, and if +row+ is +nil+, mark the last
-    # row in the table as a group boundary. This is mainly used for internal
-    # purposes.
-    def mark_boundary(row = nil) # :nodoc:
-      if row
-        boundaries.push(row)
-      else
-        boundaries.push(size - 1)
+    # row in the table as a group boundary.  An attempt to add a boundary to
+    # an empty table has no effect.  We adopt the convention that the last row
+    # of the table always marks an implicit boundary even if it is not in the
+    # @explicit_boundaries array.  When we "mark" a boundary, we intend it to
+    # be an explicit boundary, even if it marks the last row of the table.
+    def mark_boundary(row_num = nil)
+      return self if empty?
+
+      if row_num
+        unless row_num < size
+          raise ArgumentError, "can't mark boundary at row #{row_num}, last row is #{size - 1}"
+        end
+        unless row_num >= 0
+          raise ArgumentError, "can't mark boundary at non-positive row #{row_num}"
+        end
+        explicit_boundaries.push(row_num)
+      elsif size > 0
+        explicit_boundaries.push(size - 1)
       end
+      normalize_boundaries
+      self
     end
 
-    protected
-
     # :stopdoc:
 
     # Make sure size - 1 is last boundary and that they are unique and sorted.
     def normalize_boundaries
       unless empty?
-        boundaries.push(size - 1) unless boundaries.include?(size - 1)
-        self.boundaries = boundaries.uniq.sort
+        self.explicit_boundaries = explicit_boundaries.uniq.sort
+      end
+      explicit_boundaries
+    end
+
+    # Return the explicit_boundaries, augmented by an implicit boundary for
+    # the end of the table, unless it's already an implicit boundary.
+    def boundaries
+      return [] if empty?
+
+      if explicit_boundaries.last == size - 1
+        explicit_boundaries
+      else
+        explicit_boundaries + [size - 1]
       end
-      boundaries
     end
 
+    protected
+
     # Concatenate the array of argument bounds to this table's boundaries, but
     # increase each of the indexes in bounds by shift. This is used in the
     # #union_all method.
     def append_boundaries(bounds, shift: 0)
-      @boundaries += bounds.map { |k| k + shift }
+      @explicit_boundaries += bounds.map { |k| k + shift }
     end
 
-    # Return the group number to which row ~row~ belongs. Groups, from the
-    # user's point of view are indexed starting at 1.
-    def row_index_to_group_index(row)
+    # Return the group number to which row ~row_num~ belongs. Groups, from the
+    # user's point of view are indexed starting at 0.
+    def row_index_to_group_index(row_num)
       boundaries.each_with_index do |b_last, g_num|
-        return (g_num + 1) if row <= b_last
+        return (g_num + 1) if row_num <= b_last
+      end
+      0
+    end
+
+    # Return the index of the first row in group number +grp_num+
+    def first_row_num_in_group(grp_num)
+      if grp_num >= boundaries.size || grp_num < 0
+        raise ArgumentError, "group number #{grp_num} out of bounds"
+      end
+
+      grp_num.zero? ? 0 : boundaries[grp_num - 1] + 1
+    end
+
+    # Return the index of the last row in group number +grp_num+
+    def last_row_num_in_group(grp_num)
+      if grp_num > boundaries.size || grp_num < 0
+        raise ArgumentError, "group number #{grp_num} out of bounds"
+      else
+        boundaries[grp_num]
       end
-      1
     end
 
-    def group_rows(row) # :nodoc:
+    # Return the rows for group number +grp_num+.
+    def group_rows(grp_num) # :nodoc:
       normalize_boundaries
-      return [] unless row < boundaries.size
+      return [] unless grp_num < boundaries.size
 
-      first = row.zero? ? 0 : boundaries[row - 1] + 1
-      last = boundaries[row]
+      first = first_row_num_in_group(grp_num)
+      last = last_row_num_in_group(grp_num)
       rows_range(first, last)
     end
 
@@ -587,22 +744,43 @@ module FatTable
     # After sorting, the output Table will have group boundaries added after
     # each row where the sort key changes.
     def order_by(*sort_heads)
-      sort_heads = [sort_heads].flatten
-      rev_heads = sort_heads.select { |h| h.to_s.ends_with?('!') }
-      sort_heads = sort_heads.map { |h| h.to_s.sub(/\!\z/, '').to_sym }
-      rev_heads = rev_heads.map { |h| h.to_s.sub(/\!\z/, '').to_sym }
+      # Sort the rows in order and add to new_rows.
+      key_hash = partition_sort_keys(sort_heads)
       new_rows = rows.sort do |r1, r2|
-        key1 = sort_heads.map { |h| rev_heads.include?(h) ? r2[h] : r1[h] }
-        key2 = sort_heads.map { |h| rev_heads.include?(h) ? r1[h] : r2[h] }
-        key1 <=> key2
+        # Set the sort keys based on direction
+        key1 = []
+        key2 = []
+        key_hash.each_pair do |h, dir|
+          if dir == :forward
+            key1 << r1[h]
+            key2 << r2[h]
+          else
+            key1 << r2[h]
+            key2 << r1[h]
+          end
+        end
+        # Make any booleans comparable with <=>
+        key1 = key1.map_booleans
+        key2 = key2.map_booleans
+
+        # If there are any nils, <=> will return nil, and we have to use the
+        # special comparison method, compare_with_nils, instead.
+        result = (key1 <=> key2)
+        result.nil? ? compare_with_nils(key1, key2) : result
       end
-      # Add the new rows to the table, but mark a group boundary at the points
-      # where the sort key changes value.
-      new_tab = Table.new
+
+      # Add the new_rows to the table, but mark a group boundary at the points
+      # where the sort key changes value.  NB: I use self.class.new here
+      # rather than Table.new because if this class is inherited, I want the
+      # new_tab to be an instance of the subclass.  With Table.new, this
+      # method's result will be an instance of FatTable::Table rather than of
+      # the subclass.
+      new_tab = empty_dup
       last_key = nil
       new_rows.each_with_index do |nrow, k|
         new_tab << nrow
-        key = nrow.fetch_values(*sort_heads)
+        # key = nrow.fetch_values(*sort_heads)
+        key = nrow.fetch_values(*key_hash.keys)
         new_tab.mark_boundary(k - 1) if last_key && key != last_key
         last_key = key
       end
@@ -610,6 +788,33 @@ module FatTable
       new_tab
     end
 
+    # :category: Operators
+
+    # Return a new Table sorting the rows of this Table on an any expression
+    # +expr+ that is valid with the +select+ method, except that they
+    # expression may end with an exclamation mark +!+ to indicate a reverse
+    # sort.  The new table will have an additional column called +sort_key+
+    # populated with the result of evaluating the given expression and will be
+    # sorted (or reverse sorted) on that column.
+    #
+    #   tab.order_with('date.year') => table sorted by date's year
+    #   tab.order_with('date.year!') => table reverse sorted by date's year
+    #
+    # After sorting, the output Table will have group boundaries added after
+    # each row where the sort key changes.
+    def order_with(expr)
+      unless expr.is_a?(String)
+        raise "must call FatTable::Table\#order_with with a single string expression"
+      end
+      rev = false
+      if expr.match?(/\s*!\s*\z/)
+        rev = true
+        expr = expr.sub(/\s*!\s*\z/, '')
+      end
+      sort_sym = rev ? :sort_key! : :sort_key
+      dup.select(*headers, sort_key: expr).order_by(sort_sym)
+    end
+
     # :category: Operators
     #
     # Return a Table having the selected column expressions. Each expression can
@@ -713,7 +918,7 @@ module FatTable
                          before: before_hook,
                          after: after_hook)
       # Compute the new Table from this Table
-      result = Table.new
+      result = empty_dup
       normalize_boundaries
       rows.each_with_index do |old_row, old_k|
         # Set the group number in the before hook and run the hook with the
@@ -723,7 +928,15 @@ module FatTable
         ev.eval_before_hook(locals: old_row)
         # Compute the new row.
         new_row = {}
-        cols.each do |k|
+        # Allow the :omni col to stand for all columns if it is alone and
+        # first.
+        cols_to_include =
+          if cols.size == 1 && cols.first.as_sym == :omni
+            headers
+          else
+            cols
+          end
+        cols_to_include.each do |k|
           h = k.as_sym
           msg = "Column '#{h}' in select does not exist"
           raise UserError, msg unless column?(h)
@@ -752,7 +965,7 @@ module FatTable
         ev.eval_after_hook(locals: new_row)
         result << new_row
       end
-      result.boundaries = boundaries
+      result.explicit_boundaries = explicit_boundaries
       result.normalize_boundaries
       result
     end
@@ -770,9 +983,14 @@ module FatTable
     #   tab.where('@row.even? && shares > 500') => even rows with lots of shares
     def where(expr)
       expr = expr.to_s
-      result = Table.new
+      result = empty_dup
       headers.each do |h|
-        col = Column.new(header: h)
+        col =
+        if tolerant_col?(h)
+          Column.new(header: h, tolerant: true)
+        else
+          Column.new(header: h)
+        end
         result.add_column(col)
       end
       ev = Evaluator.new(ivars: { row: 0, group: 0 })
@@ -792,7 +1010,7 @@ module FatTable
     # Return a new table with all duplicate rows eliminated. Resets groups. Same
     # as #uniq.
     def distinct
-      result = Table.new
+      result = empty_dup
       uniq_rows = rows.uniq
       uniq_rows.each do |row|
         result << row
@@ -889,38 +1107,6 @@ module FatTable
       set_operation(other, :difference, distinct: false)
     end
 
-    private
-
-    # Apply the set operation given by ~oper~ between this table and the other
-    # table given in the first argument.  If distinct is true, eliminate
-    # duplicates from the result.
-    def set_operation(other, oper = :+, distinct: true, add_boundaries: true, inherit_boundaries: false)
-      unless columns.size == other.columns.size
-        msg = "can't apply set ops to tables with a different number of columns"
-        raise UserError, msg
-      end
-      unless columns.map(&:type) == other.columns.map(&:type)
-        msg = "can't apply a set ops to tables with different column types."
-        raise UserError, msg
-      end
-      other_rows = other.rows.map { |r| r.replace_keys(headers) }
-      result = Table.new
-      new_rows = rows.send(oper, other_rows)
-      new_rows.each_with_index do |row, k|
-        result << row
-        result.mark_boundary if k == size - 1 && add_boundaries
-      end
-      if inherit_boundaries
-        result.boundaries = normalize_boundaries
-        other.normalize_boundaries
-        result.append_boundaries(other.boundaries, shift: size)
-      end
-      result.normalize_boundaries
-      distinct ? result.distinct : result
-    end
-
-    public
-
     # An Array of symbols for the valid join types.
     JOIN_TYPES = %i[inner left right full cross].freeze
 
@@ -1011,7 +1197,7 @@ module FatTable
       join_exp, other_common_heads =
         build_join_expression(exps, other, join_type)
       ev = Evaluator.new
-      result = Table.new
+      result = empty_dup
       other_rows = other.rows
       other_row_matches = Array.new(other_rows.size, false)
       rows.each do |self_row|
@@ -1029,14 +1215,14 @@ module FatTable
                                   type: join_type)
           result << out_row
         end
-        next unless %i[left full].include?(join_type)
+        next unless [:left, :full].include?(join_type)
         next if self_row_matched
 
         result << build_out_row(row_a: self_row,
                                 row_b: other_row_nils,
                                 type: join_type)
       end
-      if %i[right full].include?(join_type)
+      if [:right, :full].include?(join_type)
         other_rows.each_with_index do |other_row, k|
           next if other_row_matches[k]
 
@@ -1165,7 +1351,7 @@ module FatTable
                 partial_result = nil
               else
                 # First of a pair of _a or _b
-                partial_result = String.new("(#{a_head}_a == ")
+                partial_result = +"(#{a_head}_a == "
               end
               last_sym = a_head
             when /\A(?<sy>.*)_b\z/
@@ -1184,7 +1370,7 @@ module FatTable
                 partial_result = nil
               else
                 # First of a pair of _a or _b
-                partial_result = String.new("(#{b_head}_b == ")
+                partial_result = +"(#{b_head}_b == "
               end
               b_common_heads << b_head
               last_sym = b_head
@@ -1259,7 +1445,7 @@ module FatTable
       groups = sorted_tab.rows.group_by do |r|
         group_cols.map { |k| r[k] }
       end
-      result = Table.new
+      result = empty_dup
       groups.each_pair do |_vals, grp_rows|
         result << row_from_group(grp_rows, group_cols, agg_cols)
       end
@@ -1269,6 +1455,9 @@ module FatTable
 
     private
 
+    # Collapse a group of rows to a single row by applying the aggregator from
+    # the +agg_cols+ to the items in that column and the presumably identical
+    # value in the +grp_cols to those columns.
     def row_from_group(rows, grp_cols, agg_cols)
       new_row = {}
       grp_cols.each do |h|
@@ -1291,15 +1480,6 @@ module FatTable
 
     # :category: Constructors
 
-    # Add a group boundary mark at the given row, or at the end of the table
-    # by default.
-    def add_boundary(at_row = nil)
-      row = at_row || (size - 1)
-      @boundaries << row
-    end
-
-    # :category: Constructors
-
     # Add a +row+ represented by a Hash having the headers as keys. If +mark:+
     # is set true, mark this row as a boundary. All tables should be built
     # ultimately using this method as a primitive.
@@ -1312,7 +1492,7 @@ module FatTable
         # This column is new, so it needs nil items for all prior rows lest
         # the value be added to a prior row.
         items = Array.new(size, nil)
-        columns << Column.new(header: h, items: items)
+        columns << Column.new(header: h, items: items, tolerant: tolerant_col?(h))
       end
       headers.each do |h|
         # NB: This adds a nil if h is not in row.
@@ -1486,5 +1666,74 @@ module FatTable
       yield fmt if block_given?
       fmt.output
     end
+
+    private
+
+    # Apply the set operation given by ~oper~ between this table and the other
+    # table given in the first argument.  If distinct is true, eliminate
+    # duplicates from the result.
+    def set_operation(other, oper = :+, distinct: true, add_boundaries: true, inherit_boundaries: false)
+      unless columns.size == other.columns.size
+        msg = "can't apply set ops to tables with a different number of columns"
+        raise UserError, msg
+      end
+      unless columns.map(&:type) == other.columns.map(&:type)
+        msg = "can't apply a set ops to tables with different column types."
+        raise UserError, msg
+      end
+      other_rows = other.rows.map { |r| r.replace_keys(headers) }
+      result = empty_dup
+      new_rows = rows.send(oper, other_rows)
+      new_rows.each_with_index do |row, k|
+        result << row
+        result.mark_boundary if k == size - 1 && add_boundaries
+      end
+      if inherit_boundaries
+        result.explicit_boundaries = boundaries
+        result.append_boundaries(other.boundaries, shift: size)
+      end
+      result.normalize_boundaries
+      distinct ? result.distinct : result
+    end
+
+    # Return a hash with the key being the header to sort on and the value
+    # being either :forward or :reverse to indicate the sort order on that
+    # key.
+    def partition_sort_keys(keys)
+      result = {}
+      [keys].flatten.each do |h|
+        if h.to_s.match?(/\s*!\s*\z/)
+          result[h.to_s.sub(/\s*!\s*\z/, '').to_sym] = :reverse
+        else
+          result[h] = :forward
+        end
+      end
+      result
+    end
+
+    # The <=> operator cannot handle nils without some help.  Treat a nil as
+    # smaller than any other value, but equal to other nils.  The two keys are assumed to be arrays of values to be
+    # compared with <=>.
+    def compare_with_nils(key1, key2)
+      result = nil
+      key1.zip(key2) do |k1, k2|
+        if k1.nil? && k2.nil?
+          result = 0
+          next
+        elsif k1.nil?
+          result = -1
+          break
+        elsif k2.nil?
+          result = 1
+          break
+        elsif (k1 <=> k2) == 0
+          next
+        else
+          result = (k1 <=> k2)
+          break
+        end
+      end
+      result
+    end
   end
 end