fat_table 0.4.0 → 0.5.3

data/README.rdoc

@@ -888,8 +888,7 @@ will raise an exception.
 
 +last+:: the last non-nil item in the column,
 
-+rng+:: form a string of the form "#{first}..#{last}" to show the range of
-        values in the column,
++range+- :: form a Range ~~{min}..{max}~ to show the range of values in the column,
 
 +sum+:: for Numeric and String columns, apply '+' to all the non-nil values,
 

data/TODO.org

@@ -1,3 +1,20 @@
+* TODO Conversion to Spreadsheets
+- State "TODO"       from              [2017-04-21 Fri 10:36]
+This is a [[https://github.com/westonganger/spreadsheet_architect][gem]] that I can include into the Table model to convert a table into
+a spread-sheet, or even a sheet in a multi-sheet spreadsheet file.
+
+* TODO Add from_yql for fetching from Yahoo
+- State "TODO"       from              [2017-04-21 Fri 10:35]
+Add a constructor to allow fetching stock data from yql.  Perhaps grab all
+available fields, then allow a select of those of interest.
+
+* DONE Allow sorting by expression
+CLOSED: [2022-01-20 Thu 12:47]
+Either by a single string argument as the sole argument to order_by, or use
+another method, such a order_with.  Note that this can be done now by creating
+a new column having the sort expression with select and then just order_by
+that column.  Perhaps that is an easy way to implement it.
+
 * DONE Ensure that columns resulting from aggregates have proper type
 CLOSED: [2017-12-29 Fri 05:34]
 - State "WAIT"       from "TODO"       [2017-12-29 Fri 05:34]
@@ -5,11 +22,6 @@ CLOSED: [2017-12-29 Fri 05:34]
 After applying avg, does the column have the proper Numeric or Date, or DateTime
 type.  How about Boolean aggregates?
 
-* TODO Conversion to Spreadsheets
-- State "TODO"       from              [2017-04-21 Fri 10:36]
-This is a [[https://github.com/westonganger/spreadsheet_architect][gem]] that I can include into the Table model to convert a table into
-a spread-sheet, or even a sheet in a multi-sheet spreadsheet file.
-
 * DONE Formatters
 CLOSED: [2017-04-21 Fri 10:36]
 - State "WAIT"       from "TODO"       [2017-04-21 Fri 10:36]
@@ -31,8 +43,3 @@ CLOSED: [2017-03-02 Thu 15:54]
 - State "TODO"       from              [2017-03-02 Thu 15:54]
 For tables, add a method that eliminates any duplicate rows. Perhaps just apply
 Array#uniq to the columns?
-
-* TODO Add from_yql for fetching from Yahoo
-- State "TODO"       from              [2017-04-21 Fri 10:35]
-Add a constructor to allow fetching stock data from yql.  Perhaps grab all
-available fields, then allow a select of those of interest.

data/examples/create_trans.sql

@@ -0,0 +1,14 @@
+drop table trans;
+create table trans(date text, code text, raw, shares, price, info text, ok text);
+insert into trans values('2013-05-29', 'S', 15700.00, 6601.85, 24.7790, 'ENTITY3', 'F');
+insert into trans values('2013-05-02', 'P', 118186.40, 118186.4, 11.8500, 'ENTITY1', 'T');
+insert into trans values('2013-05-20', 'S', 12000.00, 5046.00, 28.2804, 'ENTITY3', 'F');
+insert into trans values('2013-05-23', 'S', 8000.00, 3364.00, 27.1083, 'ENTITY3', 'T');
+insert into trans values('2013-05-23', 'S', 39906.00, 16780.47, 25.1749, 'ENTITY3', 'T');
+insert into trans values('2013-05-20', 'S', 85000.00, 35742.50, 28.3224, 'ENTITY3', 'T');
+insert into trans values('2013-05-02', 'P', 795546.20, 795546.2, 1.1850, 'ENTITY1', 'T');
+insert into trans values('2013-05-29', 'S', 13459.00, 5659.51, 24.7464, 'ENTITY3', 'T');
+insert into trans values('2013-05-20', 'S', 33302.00, 14003.49, 28.6383, 'ENTITY3', 'T');
+insert into trans values('2013-05-29', 'S', 15900.00, 6685.95, 24.5802, 'ENTITY3', 'T');
+insert into trans values('2013-05-30', 'S', 6679.00, 2808.52, 25.0471, 'ENTITY3', 'T');
+insert into trans values('2013-05-23', 'S', 23054.00, 9694.21, 26.8015, 'ENTITY3', 'F');

data/examples/quick.tex

@@ -0,0 +1,8 @@
+\documentclass{article}
+
+\usepackage{longtable}
+\usepackage[pdftex,table,x11names]{xcolor}
+
+\begin{document}
+\include{quicktable.tex}
+\end{document}

data/examples/quicktable.tex

@@ -0,0 +1,123 @@
+\begin{longtable}{clcrrc}
+\bfseries{Ref}&
+\multicolumn{1}{c}{\bfseries{Date}}&
+\bfseries{Code}&
+\multicolumn{1}{c}{\bfseries{Shares}}&
+\multicolumn{1}{c}{\bfseries{Price}}&
+\bfseries{Ok}\\
+\endhead
+\bfseries{1}&
+2013-05-02&
+P&
+\cellcolor{lightgray}{\textcolor{blue}{118,186.4}}&
+\$11.8500&
+Y\\
+\bfseries{2}&
+2013-05-02&
+P&
+\cellcolor{lightgray}{\textcolor{blue}{795,546.2}}&
+1.1850&
+Y\\
+\bfseries{Avg}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{456,866.3}&
+\bfseries{6.5175}&
+\\
+\bfseries{3}&
+2013-05-20&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{5,046.0}}&
+28.2804&
+N\\
+\bfseries{4}&
+2013-05-20&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{35,742.5}}&
+28.3224&
+Y\\
+\bfseries{5}&
+2013-05-20&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{14,003.5}}&
+28.6383&
+Y\\
+\bfseries{Avg}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{18,264.0}&
+\bfseries{28.4137}&
+\\
+\bfseries{6}&
+2013-05-23&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{3,364.0}}&
+27.1083&
+Y\\
+\bfseries{7}&
+2013-05-23&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{16,780.5}}&
+25.1749&
+Y\\
+\bfseries{8}&
+2013-05-23&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{9,694.2}}&
+26.8015&
+N\\
+\bfseries{Avg}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{9,946.2}&
+\bfseries{26.3616}&
+\\
+\bfseries{9}&
+2013-05-29&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{6,601.9}}&
+24.7790&
+N\\
+\bfseries{10}&
+2013-05-29&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{5,659.5}}&
+24.7464&
+Y\\
+\bfseries{11}&
+2013-05-29&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{6,686.0}}&
+24.5802&
+Y\\
+\bfseries{Avg}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{6,315.8}&
+\bfseries{24.7019}&
+\\
+\bfseries{12}&
+2013-05-30&
+S&
+\cellcolor{lightgray}{\textcolor{blue}{2,808.5}}&
+25.0471&
+Y\\
+\bfseries{Avg}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{2,808.5}&
+\bfseries{25.0471}&
+\\
+\bfseries{Average}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{85,009.9}&
+\bfseries{23.0428}&
+\\
+\bfseries{Total}&
+&
+\multicolumn{1}{l}{}&
+\bfseries{1,020,119.1}&
+\bfseries{}&
+\\
+\end{longtable}

data/examples/trans.csv

@@ -0,0 +1,13 @@
+'Date', 'Code', 'Raw', 'Shares', 'Price', 'Info', 'Ok',
+'2013-05-29', 'S', 15700.00, 6601.85, 24.7790, 'ENTITY3', FALSE,
+'2013-05-02', 'P', 118186.40, 118186.4, 11.8500, 'ENTITY1', TRUE,
+'2013-05-20', 'S', 12000.00, 5046.00, 28.2804, 'ENTITY3', FALSE,
+'2013-05-23', 'S', 8000.00, 3364.00, 27.1083, 'ENTITY3', TRUE,
+'2013-05-23', 'S', 39906.00, 16780.47, 25.1749, 'ENTITY3', TRUE,
+'2013-05-20', 'S', 85000.00, 35742.50, 28.3224, 'ENTITY3', TRUE,
+'2013-05-02', 'P', 795546.20, 795546.2, 1.1850, 'ENTITY1', TRUE,
+'2013-05-29', 'S', 13459.00, 5659.51, 24.7464, 'ENTITY3', TRUE,
+'2013-05-20', 'S', 33302.00, 14003.49, 28.6383, 'ENTITY3', TRUE,
+'2013-05-29', 'S', 15900.00, 6685.95, 24.5802, 'ENTITY3', TRUE,
+'2013-05-30', 'S', 6679.00, 2808.52, 25.0471, 'ENTITY3', TRUE,
+'2013-05-23', 'S', 23054.00, 9694.21, 26.8015, 'ENTITY3', FALSE

data/fat_table.gemspec

@@ -70,6 +70,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'redcarpet'
   spec.add_development_dependency 'pg'
+  spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'rubocop-performance'

data/lib/ext/array.rb

@@ -0,0 +1,15 @@
+class Array
+  # Map booleans true to 1 and false to 0 so they can be compared in a sort
+  # with the <=> operator.
+  def map_booleans
+    map do |v|
+      if v == true
+        1
+      elsif v == false
+        0
+      else
+        v
+      end
+    end
+  end
+end

data/lib/fat_table/column.rb

@@ -83,7 +83,7 @@ module FatTable
     #   col.type #=> 'Numeric'
     #   col.header #=> :prices
     #   col.sum #=> 18376.75
-    def initialize(header:, items: [], type: 'NilClass')
+    def initialize(header:, items: [], type: 'NilClass', tolerant: false)
       @raw_header = header
       @header =
         if @raw_header.is_a?(Symbol)
@@ -92,6 +92,7 @@ module FatTable
           @raw_header.to_s.as_sym
         end
       @type = type
+      @tolerant = tolerant
       msg = "unknown column type '#{type}"
       raise UserError, msg unless TYPES.include?(@type.to_s)
 
@@ -141,9 +142,17 @@ module FatTable
 
     # :category: Attributes
 
+    # Is this column tolerant of type incompatibilities?  If so, the Column
+    # type will be forced to String if an incompatible type is found.
+    def tolerant?
+      @tolerant
+    end
+
+    # :category: Attributes
+
     # Force the column to have String type and then convert all items to
     # strings.
-    def force_to_string_type
+    def force_string!
       # msg = "Can only force an empty column to String type"
       # raise UserError, msg unless empty?
       @type = 'String'
@@ -175,66 +184,92 @@ module FatTable
 
     # The names of the known aggregate operations that can be performed on a
     # Column.
-    VALID_AGGREGATES = %s(first last rng
-                          sum count min max avg var dev
+    VALID_AGGREGATES = %s(first last range
+                          sum count min max
+                          avg var pvar dev pdev
                           any? all? none? one?)
 
     # :category: Aggregates
 
     # Return the first non-nil item in the Column.  Works with any Column type.
     def first
-      items.compact.first
+      if type == 'String'
+        items.reject(&:blank?).first
+      else
+        items.compact.first
+      end
     end
 
     # :category: Aggregates
 
     # Return the last non-nil item in the Column.  Works with any Column type.
     def last
-      items.compact.last
+      if type == 'String'
+        items.reject(&:blank?).last
+      else
+        items.compact.last
+      end
     end
 
     # :category: Aggregates
 
-    # Return a string of the #first and #last non-nil values in the Column.
-    # Works with any Column type.
-    def rng
-      "#{first}..#{last}"
+    # Return a count of the non-nil items in the Column.  Works with any Column
+    # type.
+    def count
+      if type == 'String'
+        items.reject(&:blank?).count.to_d
+      else
+        items.compact.count.to_d
+      end
     end
 
     # :category: Aggregates
 
-    # Return the sum of the non-nil items in the Column.  Works with numeric and
-    # string Columns. For a string Column, it will return the concatenation of
-    # the non-nil items.
-    def sum
-      only_with('sum', 'Numeric', 'String')
-      items.compact.sum
+    # Return the smallest non-nil, non-blank item in the Column.  Works with
+    # numeric, string, and datetime Columns.
+    def min
+      only_with('min', 'NilClass', 'Numeric', 'String', 'DateTime')
+      if type == 'String'
+        items.reject(&:blank?).min
+      else
+        items.compact.min
+      end
     end
 
     # :category: Aggregates
 
-    # Return a count of the non-nil items in the Column.  Works with any Column
-    # type.
-    def count
-      items.compact.count.to_d
+    # Return the largest non-nil, non-blank item in the Column.  Works with
+    # numeric, string, and datetime Columns.
+    def max
+      only_with('max', 'NilClass', 'Numeric', 'String', 'DateTime')
+      if type == 'String'
+        items.reject(&:blank?).max
+      else
+        items.compact.max
+      end
     end
 
     # :category: Aggregates
 
-    # Return the smallest non-nil item in the Column.  Works with numeric,
-    # string, and datetime Columns.
-    def min
-      only_with('min', 'NilClass', 'Numeric', 'String', 'DateTime')
-      items.compact.min
+    # Return a Range object for the smallest to largest value in the column.
+    # Works with numeric, string, and datetime Columns.
+    def range
+      only_with('range', 'NilClass', 'Numeric', 'String', 'DateTime')
+      Range.new(min, max)
     end
 
     # :category: Aggregates
 
-    # Return the largest non-nil item in the Column.  Works with numeric,
-    # string, and datetime Columns.
-    def max
-      only_with('max', 'NilClass', 'Numeric', 'String', 'DateTime')
-      items.compact.max
+    # Return the sum of the non-nil items in the Column.  Works with numeric and
+    # string Columns. For a string Column, it will return the concatenation of
+    # the non-nil items.
+    def sum
+      only_with('sum', 'Numeric', 'String')
+      if type == 'String'
+        items.reject(&:blank?).join(' ')
+      else
+        items.compact.sum
+      end
     end
 
     # :category: Aggregates
@@ -374,9 +409,18 @@ module FatTable
 
     # Append +itm+ to end of the Column after converting it to the Column's
     # type. If the Column's type is still open, i.e. NilClass, attempt to fix
-    # the Column's type based on the type of +itm+ as with Column.new.
+    # the Column's type based on the type of +itm+ as with Column.new.  If its
+    # a tolerant column, respond to type errors by converting the column to a
+    # String type.
     def <<(itm)
       items << convert_to_type(itm)
+    rescue IncompatibleTypeError => ex
+      if tolerant?
+        force_string!
+        retry
+      else
+        raise ex
+      end
     end
 
     # :category: Constructors
@@ -392,187 +436,24 @@ module FatTable
 
     private
 
-    # Convert val to the type of key, a ruby class constant, such as Date,
-    # Numeric, etc. If type is NilClass, the type is open, and a non-blank val
-    # will attempt conversion to one of the allowed types, typing it as a String
-    # if no other type is recognized. If the val is blank, and the type is nil,
-    # the Column type remains open. If the val is nil or a blank and the type is
-    # already determined, the val is set to nil, and should be filtered from any
-    # Column computations. If the val is non-blank and the Column type
-    # determined, raise an error if the val cannot be converted to the Column
-    # type. Otherwise, returns the converted val as an object of the correct
-    # class.
     def convert_to_type(val)
-      case type
-      when 'NilClass'
-        if val != false && val.blank?
-          # Leave the type of the Column open. Unfortunately, false counts as
-          # blank and we don't want it to. It should be classified as a boolean.
-          new_val = nil
-        else
-          # Only non-blank values are allowed to set the type of the Column
-          bool_val = convert_to_boolean(val)
-          new_val =
-            if bool_val.nil?
-              convert_to_date_time(val) ||
-                convert_to_numeric(val) ||
-                convert_to_string(val)
-            else
-              bool_val
-            end
-          @type =
-            if [true, false].include?(new_val)
-              'Boolean'
-            elsif new_val.is_a?(Date) || new_val.is_a?(DateTime)
-              'DateTime'
-            elsif new_val.is_a?(Numeric)
-              'Numeric'
-            elsif new_val.is_a?(String)
-              'String'
-            else
-              msg = "can't add #{val} of type #{new_val.class.name} to a column"
-              raise UserError, msg
-            end
-        end
-        new_val
-      when 'Boolean'
-        if val.is_a?(String) && val.blank? || val.nil?
-          nil
-        else
-          new_val = convert_to_boolean(val)
-          if new_val.nil?
-            msg = "attempt to add '#{val}' to a column already typed as #{type}"
-            raise UserError, msg
-          end
-          new_val
-        end
-      when 'DateTime'
-        if val.blank?
-          nil
-        else
-          new_val = convert_to_date_time(val)
-          if new_val.nil?
-            msg = "attempt to add '#{val}' to a column already typed as #{type}"
-            raise UserError, msg
-          end
-          new_val
-        end
-      when 'Numeric'
-        if val.blank?
-          nil
-        else
-          new_val = convert_to_numeric(val)
-          if new_val.nil?
-            msg = "attempt to add '#{val}' to a column already typed as #{type}"
-            raise UserError, msg
-          end
-          new_val
-        end
-      when 'String'
-        if val.nil?
-          nil
-        else
-          new_val = convert_to_string(val)
-          if new_val.nil?
-            msg = "attempt to add '#{val}' to a column already typed as #{type}"
+      new_val = Convert.convert_to_type(val, type)
+      if new_val && type == 'NilClass'
+        @type =
+          if [true, false].include?(new_val)
+            'Boolean'
+          elsif new_val.is_a?(Date) || new_val.is_a?(DateTime)
+            'DateTime'
+          elsif new_val.is_a?(Numeric)
+            'Numeric'
+          elsif new_val.is_a?(String)
+            'String'
+          else
+            msg = "can't add value '#{val}' of type #{new_val.class.name} to a column"
             raise UserError, msg
           end
-          new_val
-        end
-      else
-        raise UserError, "Mysteriously, column has unknown type '#{type}'"
-      end
-    end
-
-    # Convert the val to a boolean if it looks like one, otherwise return nil.
-    # Any boolean or a string of t, f, true, false, y, n, yes, or no, regardless
-    # of case is assumed to be a boolean.
-    def convert_to_boolean(val)
-      return val if val.is_a?(TrueClass) || val.is_a?(FalseClass)
-      val = val.to_s.clean
-      return nil if val.blank?
-      if val.match?(/\A(false|f|n|no)\z/i)
-        false
-      elsif val.match?(/\A(true|t|y|yes)\z/i)
-        true
-      end
-    end
-
-    ISO_DATE_RE = %r{(?<yr>\d\d\d\d)[-\/]
-                (?<mo>\d\d?)[-\/]
-                (?<dy>\d\d?)\s*
-                (T?\s*\d\d:\d\d(:\d\d)?
-                ([-+](\d\d?)(:\d\d?))?)?}x
-
-    AMR_DATE_RE = %r{(?<dy>\d\d?)[-/](?<mo>\d\d?)[-/](?<yr>\d\d\d\d)\s*
-                     (?<tm>T\d\d:\d\d:\d\d(\+\d\d:\d\d)?)?}x
-
-    # A Date like 'Tue, 01 Nov 2016' or 'Tue 01 Nov 2016' or '01 Nov 2016'.
-    # These are emitted by Postgresql, so it makes from_sql constructor
-    # possible without special formatting of the dates.
-    INV_DATE_RE = %r{((mon|tue|wed|thu|fri|sat|sun)[a-zA-z]*,?)?\s+  # looks like dow
-    (?<dy>\d\d?)\s+  # one or two-digit day
-    (?<mo_name>[jfmasondJFMASOND][A-Za-z]{2,})\s+  # looks like a month name
-    (?<yr>\d\d\d\d) # and a 4-digit year
-    }xi
-
-    # Convert the val to a DateTime if it is either a DateTime, a Date, a Time, or a
-    # String that can be parsed as a DateTime, otherwise return nil. It only
-    # recognizes strings that contain a something like '2016-01-14' or '2/12/1985'
-    # within them, otherwise DateTime.parse would treat many bare numbers as dates,
-    # such as '2841381', which it would recognize as a valid date, but the user
-    # probably does not intend it to be so treated.
-    def convert_to_date_time(val)
-      return val if val.is_a?(DateTime)
-      return val if val.is_a?(Date)
-      return val.to_datetime if val.is_a?(Time)
-      begin
-        str = val.to_s.clean
-        return nil if str.blank?
-
-        if str.match(ISO_DATE_RE)
-          date = DateTime.parse(val)
-        elsif str =~ AMR_DATE_RE
-          date = DateTime.new(Regexp.last_match[:yr].to_i,
-                         Regexp.last_match[:mo].to_i,
-                         Regexp.last_match[:dy].to_i)
-        elsif str =~ INV_DATE_RE
-          mo = Date.mo_name_to_num(last_match[:mo_name])
-          date = DateTime.new(Regexp.last_match[:yr].to_i, mo,
-                              Regexp.last_match[:dy].to_i)
-        else
-          return nil
-        end
-        # val = val.to_date if
-        date.seconds_since_midnight.zero? ? date.to_date : date
-      rescue ArgumentError
-        nil
       end
-    end
-
-    # Convert the val to a Numeric if is already a Numeric or is a String that
-    # looks like one. Any Float is promoted to a BigDecimal. Otherwise return
-    # nil.
-    def convert_to_numeric(val)
-      return BigDecimal(val, Float::DIG) if val.is_a?(Float)
-      return val if val.is_a?(Numeric)
-      # Eliminate any commas, $'s (or other currency symbol), or _'s.
-      cursym = Regexp.quote(FatTable.currency_symbol)
-      clean_re = /[,_#{cursym}]/
-      val = val.to_s.clean.gsub(clean_re, '')
-      return nil if val.blank?
-      case val
-      when /(\A[-+]?\d+\.\d*\z)|(\A[-+]?\d*\.\d+\z)/
-        BigDecimal(val.to_s.clean)
-      when /\A[-+]?[\d]+\z/
-        val.to_i
-      when %r{\A(?<nm>[-+]?\d+)\s*[:/]\s*(?<dn>[-+]?\d+)\z}
-        Rational(Regexp.last_match[:nm], Regexp.last_match[:dn])
-      end
-    end
-
-    def convert_to_string(val)
-      val.to_s
+      new_val
     end
   end
 end