fat_table 0.2.4 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 772c4f7d95d5af94968f650b4bc98ebad4756626
-  data.tar.gz: c15462e9a444de246690482da1a8acd3a46e0c60
+  metadata.gz: fc10b9aadaada3eb303831288c6cc65d9362a5c6
+  data.tar.gz: 1a4c9e66382b2879ca6306f955d7aba5c4b9eef1
 SHA512:
-  metadata.gz: 4f1177b1568ece2027849b102f2e7db84db99846e7291882f943fba5fd400b75032847df303c528fada0683afb2fb5b0bc97a2a91f2e52ed74cbd6960da07ac2
-  data.tar.gz: 813ddb4b57d893ce9bfb0ca17bb6d03d8807fa361b166396d0635fc5d23e70f4c1641fb69b0f3973678faa2e1bb87667de6bca4b8ff3c0033ceba38db0168658
+  metadata.gz: 2f263f677bdd964642a6700c5c139d689184059ee994e551941dba6fac2d1d3bc3830ac63c7b28e526356b0166ee63c255ffac00769a38ee6e71fba090216b9f
+  data.tar.gz: 5d02b955317232d48a78f8ddb73632a4aaea5c49da31f72a808303a362c295203b7f6d508b55f03b14687953b72c7870a840844ff2779bb0cbc1ae16a966bd22

data/.gitignore

@@ -20,3 +20,6 @@
 /lib/GPATH
 /lib/GRTAGS
 /lib/GTAGS
+/GPATH
+/GRTAGS
+/GTAGS

data/.travis.yml

@@ -1,5 +1,17 @@
-sudo: false
 language: ruby
+before_install:
+  - sudo apt-get -qq update
+  - sudo apt-get install -y texlive-latex-base texlive-latex-recommended
+after_failure:
+  - "pwd"
+  - "cat ./spec/tmp/latex.err"
+  - "cat ./spec/tmp/example1.log"
+  - "cat ./spec/tmp/psql.out"
+bundler_args: --without debug
+services:
+  - postgresql
 rvm:
-  - 2.3.0
-before_install: gem install bundler -v 1.14.3
+  - 2.2.2
+  - 2.3
+  - 2.4
+  - ruby-head

data/README.org

@@ -1,6 +1,12 @@
 #+OPTIONS: :toc
 #+LATEX_HEADER: \usepackage[margin=0.75in]{geometry}
 
+#+BEGIN_COMMENT
+[![Build Status](https://travis-ci.org/ddoherty03/fat_table.svg?branch=master)](https://travis-ci.org/ddoherty03/fat_table)
+#+END_COMMENT
+
+[[https://travis-ci.org/ddoherty03/fat_table.svg?branch=master]]
+
 * Introduction
 
 ~FatTable~ is a gem that treats tables as a data type. It provides methods for
@@ -28,9 +34,9 @@ as a foundation for providing reporting functions where flexibility about the
 output medium can be quite useful. Finally ~FatTable~ can be used within Emacs
 ~org-mode~ files in code blocks targeting the Ruby language. Org mode tables are
 presented to a ruby code block as an array of arrays, so ~FatTable~ can read
-them in with its ~.from_aoa~ constructor. A ~FatTable~ table can output as an
-array of arrays with its ~.to_aoa~ output function and will be rendered in an
-org-mode buffer as an org-table, ready for processing by other code blocks.
+them in with its ~.from_aoa~ constructor. A ~FatTable~ table output as an array
+of arrays with its ~.to_aoa~ output function will be rendered in an org-mode
+buffer as an org-table, ready for processing by other code blocks.
 
 * Installation
 
@@ -244,21 +250,24 @@ types as follows:
      regardless of case, are interpreted as ~FalseClass~, in either case
      resulting in a Boolean column. Empty strings in a column already having a
      Boolean type are converted to nil.
-- DateTime :: strings that contain patterns of 'yyyy-mm-dd' or 'yyyy/mm/dd' will
-     be interpreted as a ~DateTime~ or a ~Date~ (if there are no sub-day time
-     components present). The number of digits in the month and day can be one
-     or two, but the year component must be four digits. Any time components are
-     valid if they can be properly interpreted by ~DateTime.parse~. Org mode
-     timestamps, active or inactive, are valid input strings for DateTime
-     columns. Empty strings in a column already having the DateTime type are
-     converted to nil.
-- Numeric :: all commas ',', underscores, '_', and '$' dollar signs are removed
-     from the string and if the remaining string can be interpreted as a
-     ~Numeric~, it will be. It is interpreted as an ~Integer~ if there are no
-     decimal places in the remaining string, as a ~Rational~ if the string has
-     the form '<number>:<number>' or '<number>/<number>', or as a ~BigDecimal~
-     if there is a decimal point in the remaining string. Empty strings in a
-     column already having the Numeric type are converted to nil.
+- DateTime :: strings that contain patterns of 'yyyy-mm-dd' or 'yyyy/mm/dd' or
+     'mm-dd-yyy' or 'mm/dd/yyyy' or any of the foregoing with an added
+     'Thh:mm:ss' or 'Thh:mm' will be interpreted as a ~DateTime~ or a ~Date~ (if
+     there are no sub-day time components present). The number of digits in the
+     month and day can be one or two, but the year component must be four
+     digits. Any time components are valid if they can be properly interpreted
+     by ~DateTime.parse~. Org mode timestamps (any of the foregoing surrounded
+     by square '[]' or pointy '<>' brackets), active or inactive, are valid
+     input strings for DateTime columns. Empty strings in a column already
+     having the DateTime type are converted to nil.
+- Numeric :: all commas ',', underscores, '_', and '$' dollar signs (or other
+     currency symbol as set by ~FatTable.currency_symbol~ are removed from the
+     string and if the remaining string can be interpreted as a ~Numeric~, it
+     will be. It is interpreted as an ~Integer~ if there are no decimal places
+     in the remaining string, as a ~Rational~ if the string has the form
+     '<number>:<number>' or '<number>/<number>', or as a ~BigDecimal~ if there
+     is a decimal point in the remaining string. Empty strings in a column
+     already having the Numeric type are converted to nil.
 - String :: if all else fails, ~FatTable~ applies ~#to_s~ to the input value
      and, treats it as an item of type ~String~.  Empty strings in a column
      already having the String type are kept as empty strings.
@@ -497,10 +506,11 @@ or strings that can parsed into one of the permissible column types.
 *** From SQL queries
 
 Another way to initialize a ~FatTable~ table is with the results of a SQL query.
-~FatTable~ uses the ~dbi~ gem to query databases. You must first set the
+~FatTable~ uses the ~sequel~ gem to query databases. You must first set the
 database parameters to be used for the queries.
 
 #+BEGIN_SRC ruby
+  # This automatically requires sequel.
   require 'fat_table'
   FatTable.set_db(driver: 'Pg',
                   database: 'XXX_development',
@@ -522,6 +532,19 @@ The ~.set_db~ function need only be called once, and the database handle it
 creates will be used for all subsequent ~.from_sql~ calls until ~.set_db~ is
 called again.
 
+Alternatively, you can build the ~Sequel~ connection with ~Sequel.connect~ or
+with adapter-specific ~Sequel~ connection methods and let ~FatTable~ know to use
+that connection:
+
+#+BEGIN_SRC ruby
+  require 'fat_table'
+  FatTable.db = Sequel.connect('postgres://user:password@localhost/dbname')
+  FatTable.db = Sequel.ado(conn_string: 'Provider=Microsoft.ACE.OLEDB.12.0;Data Source=drive:\path\filename.accdb')
+#+END_SRC
+
+Consult ~Sequel's~ documentation for details on its connection methods.
+[[http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html]]
+
 *** Marking Groups in Input
 
 The ~.from_aoa~ and ~.from_aoh~ functions take an optional keyword parameter
@@ -679,6 +702,8 @@ With the ~select~ method, you can select which existing columns should appear in
 the output table and create new columns in the output table that are a function
 of existing and new columns.
 
+**** Selecting Existing Columns
+
 Here we select three existing columns by simply passing header symbols in the
 order we want them to appear in the output. Thus, one use of =select= is to
 filter and permute the order of existing columns. The =select= method preserves
@@ -716,6 +741,8 @@ any group boundaries present in the input table.
 |  8.25 | T016 |    100 |
 #+END_EXAMPLE
 
+**** Adding New Columns
+
 More interesting is that ~select~ can take hash-like keyword arguments following
 all of the symbol arguments to create new columns in the output as functions of
 other columns. For each hash-like parameter, the keyword given must be a symbol,
@@ -810,9 +837,71 @@ second, chained call to ~select~:
 | T016 | 2016-11-02 |  8.25 |    100 |    825.0 |
 #+END_EXAMPLE
 
+**** Custom Instance Variables and Hooks
+
+As the above examples demonstrate, the instance variables ~@row~ and ~@group~
+are available when evaluating expressions that add new columns.  You can set up
+your own instance variables as well for keeping track of things that cross row
+boundaries, such as running sums.
+
+To declare instance variables, you can use the ~ivars:~ hash parameter to
+~select~.  Each key of the hash becomes an instance variable and each value
+becomes its initial value before any rows are evaluated.
+
+In addition, you can provide ~before_hook:~ and ~after_hook:~ parameters as
+strings that are evaluated as ruby expressions before and after each row is
+processed.  You can use these to update instance variables.  The values set in
+the ~before_hook:~ can be used in expressions for adding new columns by
+referencing them with the '@' prefix.
+
+For example, suppose we wanted to not only add a cost column, but a column that
+shows the cumulative cost after each transaction in our example table. The
+following example uses the ~ivars:~ and ~before_hook:~ parameters to keep track
+of the running cost of shares, then formats the table.
+
+#+HEADER: :colnames no :session readme :hlines yes :wrap EXAMPLE :exports both
+#+BEGIN_SRC ruby
+  tab = tab1.select(:ref, :price, :shares, traded_on: :date, \
+              cost: 'price * shares', cumulative: '@total_cost', \
+              ivars: { total_cost: 0 }, \
+              before_hook: '@total_cost += price * shares')
+  FatTable.to_aoa(tab) do |f|
+    f.format(price: '0.4', shares: '0.0,', cost: '0.2,', cumulative: '0.2,')
+  end
+#+END_SRC
+
+#+BEGIN_EXAMPLE
+| Ref  |  Price | Shares |  Traded On |       Cost | Cumulative |
+|------+--------+--------+------------+------------+------------|
+| T001 | 7.7000 |    100 | 2016-11-01 |     770.00 |     770.00 |
+| T002 | 7.7500 |    200 | 2016-11-01 |   1,550.00 |   2,320.00 |
+| T003 | 7.5000 |    800 | 2016-11-01 |   6,000.00 |   8,320.00 |
+| T003 | 7.5000 |    800 | 2016-11-01 |   6,000.00 |  14,320.00 |
+|------+--------+--------+------------+------------+------------|
+| T004 | 7.5500 |  6,811 | 2016-11-01 |  51,423.05 |  65,743.05 |
+| T005 | 7.5000 |  4,000 | 2016-11-01 |  30,000.00 |  95,743.05 |
+| T006 | 7.6000 |  1,000 | 2016-11-01 |   7,600.00 | 103,343.05 |
+| T006 | 7.6000 |  1,000 | 2016-11-01 |   7,600.00 | 110,943.05 |
+| T007 | 7.6500 |    200 | 2016-11-01 |   1,530.00 | 112,473.05 |
+| T008 | 7.6500 |  2,771 | 2016-11-01 |  21,198.15 | 133,671.20 |
+| T009 | 7.6000 |  9,550 | 2016-11-01 |  72,580.00 | 206,251.20 |
+|------+--------+--------+------------+------------+------------|
+| T010 | 7.5500 |  3,175 | 2016-11-01 |  23,971.25 | 230,222.45 |
+| T011 | 7.4250 |    100 | 2016-11-02 |     742.50 | 230,964.95 |
+| T012 | 7.5500 |  4,700 | 2016-11-02 |  35,485.00 | 266,449.95 |
+| T012 | 7.5500 |  4,700 | 2016-11-02 |  35,485.00 | 301,934.95 |
+| T013 | 7.3500 | 53,100 | 2016-11-02 | 390,285.00 | 692,219.95 |
+|------+--------+--------+------------+------------+------------|
+| T014 | 7.4500 |  5,847 | 2016-11-02 |  43,560.15 | 735,780.10 |
+| T015 | 7.7500 |    500 | 2016-11-02 |   3,875.00 | 739,655.10 |
+| T016 | 8.2500 |    100 | 2016-11-02 |     825.00 | 740,480.10 |
+#+END_EXAMPLE
+
+**** Argument Order and Boundaries
 
 Notice that ~select~ can take any number of arguments but all the symbol
-arguments must come first followed by all the hash-like keyword arguments.
+arguments must come first followed by all the hash-like keyword arguments,
+including the special arguments for instance variables and hooks.
 
 As the example illustrates, ~.select~ transmits any group boundaries in its
 input table to the result table.

data/fat_table.gemspec

@@ -1,4 +1,5 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fat_table/version'
@@ -74,8 +75,8 @@ org-mode buffer as an org-table, ready for processing by other code blocks.
   spec.add_runtime_dependency 'fat_core', '~> 4.0', '>= 4.1'
   spec.add_runtime_dependency 'activesupport'
   spec.add_runtime_dependency 'rainbow'
-  spec.add_runtime_dependency 'dbi'
-  spec.add_runtime_dependency 'dbd-pg'
-  spec.add_runtime_dependency 'dbd-mysql'
-  spec.add_runtime_dependency 'dbd-sqlite3'
+  spec.add_runtime_dependency 'sequel'
+  spec.add_runtime_dependency 'pg'
+  spec.add_runtime_dependency 'sqlite3'
+  spec.add_runtime_dependency 'mysql2'
 end

data/lib/fat_table.rb

@@ -10,12 +10,13 @@ module FatTable
   require 'fat_core/hash'
   require 'fat_core/numeric'
   require 'csv'
-  require 'dbi'
+  require 'sequel'
   require 'active_support'
   require 'active_support/core_ext'
   require 'active_support/number_helper'
 
   require 'fat_table/version'
+  require 'fat_table/patches'
   require 'fat_table/evaluator'
   require 'fat_table/column'
   require 'fat_table/table'

data/lib/fat_table/column.rb

@@ -473,6 +473,9 @@ module FatTable
       end
     end
 
+    IS0_DATE_RE = %r{\b(\d\d\d\d)[-/](\d\d?)[-/](\d\d?)\s*(T\d\d:\d\d:\d\d(\+\d\d:\d\d)?)?\b}
+    AMR_DATE_RE = %r{\b(\d\d?)[-/](\d\d?)[-/](\d\d\d\d)\s*(T\d\d:\d\d:\d\d(\+\d\d:\d\d)?)?\b}
+
     # Convert the val to a DateTime if it is either a DateTime, a Date, or a
     # String that can be parsed as a DateTime, otherwise return nil. It only
     # recognizes strings that contain a something like '2016-01-14' or
@@ -485,8 +488,13 @@ module FatTable
       begin
         val = val.to_s.clean
         return nil if val.blank?
-        return nil unless val =~ %r{\b\d\d\d\d[-/]\d\d?[-/]\d\d?\b}
-        val = DateTime.parse(val.to_s.clean)
+        if val =~ IS0_DATE_RE
+          val = DateTime.parse(val)
+        elsif val =~ AMR_DATE_RE
+          val = DateTime.new($3.to_i, $1.to_i, $2.to_i)
+        else
+          return nil
+        end
         val = val.to_date if val.seconds_since_midnight.zero?
         val
       rescue ArgumentError
@@ -506,11 +514,11 @@ module FatTable
       val = val.to_s.clean.gsub(clean_re, '')
       return nil if val.blank?
       case val
-      when /\A(\d+\.\d*)|(\d*\.\d+)\z/
+      when /(\A[-+]?\d+\.\d*\z)|(\A[-+]?\d*\.\d+\z)/
         BigDecimal.new(val.to_s.clean)
-      when /\A[\d]+\z/
+      when /\A[-+]?[\d]+\z/
         val.to_i
-      when %r{\A(\d+)\s*[:/]\s*(\d+)\z}
+      when %r{\A([-+]?\d+)\s*[:/]\s*([-+]?\d+)\z}
         Rational($1, $2)
       end
     end

data/lib/fat_table/db_handle.rb

@@ -1,18 +1,25 @@
 module FatTable
   class << self;
-    # The +DBI+ database handle returned by the last successful call to
-    # FatTable.set_db.
+    # The +Sequel+ database handle to use in calls to +FatTable.from_sql+.
     attr_accessor :handle
   end
 
-  # This method must be called before calling FatTable.from_sql or
-  # FatTable::Table.from_sql in order to specify the database to use.  All of
-  # the keyword parameters have a default except +database:+, which must contain
-  # the name of the database to query.
+  # This method must be called before calling +FatTable.from_sql+ or
+  # +FatTable::Table.from_sql+ in order to specify the database to use.
+  #
+  # You can pass in a +Sequel+ connection with +db+, or have fat_table construct
+  # a uri from given components. In the latter case, all of the keyword
+  # parameters have a default except +database:+, which must contain the name of
+  # the database to query.
+  #
+  # +db+::
+  #    Inject a Sequel connection constructed +Sequel.connect+ or one of
+  #    Sequel's adapter-specific connection methods.
+  #    http://sequel.jeremyevans.net/rdoc/files/doc/opening_databases_rdoc.html
   #
   # +driver+::
-  #    One of 'Pg' (for Postgresql), 'Mysql' (for Mysql), or 'SQLite3' (for
-  #    SQLite3) to specify the +DBI+ driver to use.  You may have to install the
+  #    One of 'pg' (for Postgresql), 'mysql' or 'mysql2' (for Mysql), or 'sqlite' (for
+  #    SQLite3) to specify the +Sequel+ driver to use.  You may have to install the
   #    driver to make this work.  By default use 'Pg'.
   #
   # +database+::
@@ -20,11 +27,11 @@ module FatTable
   #
   # +user+::
   #    The user name to use for accessing the database.  It defaults to nil,
-  #    which may be interpreted as a default user by the DBI driver being used.
+  #    which may be interpreted as a default user by the Sequel driver being used.
   #
   # +password+::
   #    The password to use for accessing the database.  It defaults to nil,
-  #    which may be interpreted as a default password by the DBI driver being used.
+  #    which may be interpreted as a default password by the Sequel driver being used.
   #
   # +host+::
   #    The name of the host on which to look for the database connection,
@@ -38,44 +45,59 @@ module FatTable
   #    The socket to use to access the database if the host is 'localhost'.
   #    Defaults to the standard socket for the Pg driver, '/tmp/.s.PGSQL.5432'.
   #
-  # If successful the database handle for DBI is return.
+  # If successful the database handle for Sequel is return.
   # Once called successfully, this establishes the database handle to use for
   # all subsequent calls to FatTable.from_sql or FatTable::Table.from_sql.  You
   # can then access the handle if needed with FatTable.db.
-  def self.set_db(driver: 'Pg',
+  def self.set_db(db: nil,
+                  driver: 'postgres',
                   database:,
-                  user: nil,
+                  user: ENV['LOGNAME'],
                   password: nil,
                   host: 'localhost',
                   port: '5432',
                   socket: '/tmp/.s.PGSQL.5432')
-    raise UserError, 'must supply database name to set_db' unless database
+    if db
+      self.handle = db
+    else
+      raise UserError, 'must supply database name to set_db' unless database
 
-    valid_drivers = ['Pg', 'Mysql', 'SQLite3']
-    unless valid_drivers.include?(driver)
-      raise UserError, "set_db driver must be one of #{valid_drivers.join(' or ')}"
-    end
-    # In case port is given as an integer
-    port = port.to_s if port
+      valid_drivers = ['postgres', 'mysql', 'mysql2', 'sqlite']
+      unless valid_drivers.include?(driver)
+        raise UserError, "'#{driver}' driver must be one of #{valid_drivers.join(' or ')}"
+      end
+      if database.blank?
+        raise UserError, "must supply database parameter to set_db"
+      end
 
-    # Set the dsn for DBI
-    dsn =
-      if host == 'localhost'
-        "DBI:Pg:database=#{database};host=#{host};socket=#{socket}"
+      if driver == 'sqlite'
+        dsn = "sqlite://#{database}"
       else
-        "DBI:Pg:database=#{database};host=#{host};port=#{port}"
+        pw_part = password ? ":#{password}" : ''
+        hst_part = host ? "@#{host}" : ''
+        prt_part = port ? ":#{port}" : ''
+        dsn = "#{driver}:://#{user}#{pw_part}#{hst_part}#{prt_part}/#{database}"
+      end
+
+      # Set the dsn for Sequel
+      begin
+        # DB = Sequel.connect(dsn)
+        self.handle = Sequel.connect(dsn)
+      rescue => ex
+        raise TransientError, "#{dsn}: #{ex}"
       end
-    begin
-      self.handle = ::DBI.connect(dsn, user, password)
-    rescue DBI::OperationalError => ex
-      raise TransientError, "#{dsn}: #{ex}"
     end
     handle
   end
 
-  # Return the +DBI+ database handle as returned by the last call to
-  # FatTable.set_db.
+  # Return the +Sequel+ database handle.
   def self.db
     handle
   end
+
+  # Directly set the db handle to a Sequel connection formed without
+  # FatTable.set_db.
+  def self.db=(db)
+    self.handle = db
+  end
 end

data/lib/fat_table/evaluator.rb

@@ -17,24 +17,37 @@ module FatTable
     # subsequent calls to Evaluator.evaluate. The strings +before+ and +after+
     # are string expressions that will be evaluated before and after each
     # subsequent call to Evaluator.evaluate.
-    def initialize(vars: {}, before: nil, after: nil)
+    def initialize(ivars: {}, before: nil, after: nil)
       @before = before
       @after = after
-      set_instance_vars(vars)
+      set_instance_vars(ivars)
+    end
+
+    # Run the before hook after setting the @group to grp, passed in as a
+    # parameter.  This is run before any column has been evaluated with the
+    # evaluate method.
+    def eval_before_hook(vars)
+      bdg = binding
+      set_local_vars(vars, bdg)
+      @group = vars[:__group]
+      eval(@before, bdg) if @before
+    end
+
+    # Run the after hook.  This is run after each column in the row has been
+    # evaluated with the evaluate method.
+    def eval_after_hook(vars)
+      bdg = binding
+      set_local_vars(vars, bdg)
+      eval(@after, bdg) if @after
     end
 
     # Return the result of evaluating +expr+ as a Ruby expression in which the
     # instance variables set in Evaluator.new and any local variables set in the
-    # Hash parameter +vars+ are available to the expression. Call any +before+
-    # hook defined in Evaluator.new before evaluating +expr+ and any +after+
-    # hook defined in Evaluator.new after evaluating +expr+.
+    # Hash parameter +vars+ are available to the expression.
     def evaluate(expr = '', vars: {})
       bdg = binding
       set_local_vars(vars, bdg)
-      eval(@before, bdg) if @before
-      result = eval(expr, bdg)
-      eval(@after, bdg) if @after
-      result
+      eval(expr, bdg)
     end
 
     private

data/lib/fat_table/formatters/term_formatter.rb

@@ -60,7 +60,8 @@ module FatTable
     end
 
     def strip_ansi(str)
-      str&.gsub(/\e\[[0-9;]+m/, '')
+      return '' unless str
+      str.gsub(/\e\[[0-9;]+m/, '')
     end
 
     # Add ANSI codes to string to implement the given decorations

data/lib/fat_table/patches.rb

@@ -0,0 +1,16 @@
+unless {a: 1}.respond_to?(:fetch_values)
+  class Hash
+    def fetch_values(*keys)
+      result = []
+      keys.each do |k|
+        result <<
+          if block_given?
+            yield(self[k])
+          else
+            self[k]
+          end
+      end
+      result
+    end
+  end
+end

data/lib/fat_table/table.rb

@@ -153,9 +153,7 @@ module FatTable
     def self.from_sql(query)
       raise UserError, 'FatTable.db must be set with FatTable.set_db' if FatTable.db.nil?
       result = Table.new
-      sth = FatTable.db.prepare(query)
-      sth.execute
-      sth.fetch_hash do |h|
+      FatTable.db[query].each do |h|
         result << h
       end
       result
@@ -595,7 +593,7 @@ module FatTable
     end
 
     # :category: Operators
-
+    #
     # Return a Table having the selected column expressions. Each expression can
     # be either a
     #
@@ -613,6 +611,12 @@ module FatTable
     #    access the instance variable @row, as the row number of the row being
     #    evaluated, and @group, as the group number of the row being evaluated.
     #
+    # 4. a hash in +new_cols+ with one of the special keys, +ivars: {literal
+    #    hash}+, +before_hook: 'ruby-code'+, or +after_hook: 'ruby-code'+ for
+    #    defining custom instance variables to be used during evaluation of
+    #    parameters described in point 3 and hooks of ruby code snippets to be
+    #    evaluated before and after processing each row.
+    #
     # The bare symbol arguments +cols+ (1) must precede any hash arguments
     # +new_cols+ (2 or 3). Each expression results in a column in the resulting
     # Table in the order given in the argument list. The expressions are
@@ -623,12 +627,84 @@ module FatTable
     #   tab.select(:ref, :date, shares: :quantity) => rename :shares->:quantity
     #   tab.select(:ref, :date, :shares, cost: 'price * shares') => new column
     #   tab.select(:ref, :date, :shares, seq: '@row') => add sequential nums
+    #
+    # The instance variables and hooks mentioned in point 4 above allow you to
+    # keep track of things that cross row boundaries, such as running sums or
+    # the values of columns before or after construction of the new row. You can
+    # define instance variables other than the default @row and @group variables
+    # to be available when evaluating normal string expressions for constructing
+    # a new row.
+    #
+    # You define custom instance variables by passing a Hash to the ivars
+    # parameter. The names of the instance variables will be the keys and their
+    # initial values will be the values. For example, you can keep track of a
+    # running sum of the cost of shares and the number of shares in the prior
+    # row by adding two custom instance variables and the appropriate hooks:
+    #
+    #   tab.select(:ref, :date, :shares, :price,
+    #              cost: 'shares * price', cumulative_cost: '@total_cost'
+    #              ivars: { total_cost: 0, prior_shares: 0},
+    #              before_hook: '@total_cost += shares * price,
+    #              after_hook: '@prior_shares = shares')
+    #
+    # Notice that in the +ivars:+ parameter, the '@' is not prefixed to the name
+    # since it is a symbol, but must be prefixed when the instance variable is
+    # referenced in an expression, otherwise it would be interpreted as a column
+    # name.  You could include the '@' if you use a string as a key, e.g., +{
+    # '@total_cost' => 0 }+  The ivars values are evaluated once, before the
+    # first row is processed with the select statement.
+    #
+    # For each row, the +before_hook+ is evaluated, then the +new_cols+
+    # expressions for setting the new value of columns, then the +after_hook+ is
+    # evaluated.
+    #
+    # In the before_hook, the values of all columns are available as local
+    # variables as they were before processing the row. The values of all
+    # instance variables are available as well with the values they had after
+    # processing the prior row of the table.
+    #
+    # In the string expressions for new columns, all the instance variables are
+    # available with the values they have after the before_hook is evaluated.
+    # You could also modify instance variables in the new_cols expression, but
+    # remember, they are evaluated once for each new column expression. Also,
+    # the new column is assigned the value of the entire expression, so you must
+    # ensure that the last expression is the one you want assigned to the new
+    # column. You might want to use a semicolon: +cost: '@total_cost += shares *
+    # price; shares * price'
+    #
+    # In the after_hook, the new, updated values of all columns, old and new are
+    # available as local variables, and the instance variables are available
+    # with the values they had after executing the before_hook.
     def select(*cols, **new_cols)
+      # Set up the Evaluator
+      ivars = { row: 0, group: 0 }
+      if new_cols.key?(:ivars)
+        ivars = ivars.merge(new_cols[:ivars])
+        new_cols.delete(:ivars)
+      end
+      before_hook = '@row += 1'
+      if new_cols.key?(:before_hook)
+        before_hook += "; #{new_cols[:before_hook]}"
+        new_cols.delete(:before_hook)
+      end
+      after_hook = nil
+      if new_cols.key?(:after_hook)
+        after_hook = new_cols[:after_hook].to_s
+        new_cols.delete(:after_hook)
+      end
+      ev = Evaluator.new(ivars: ivars,
+                         before: before_hook,
+                         after: after_hook)
+      # Compute the new Table from this Table
       result = Table.new
       normalize_boundaries
-      ev = Evaluator.new(vars: { row: 0, group: 1 },
-                         before: '@row = __row; @group = __group')
       rows.each_with_index do |old_row, old_k|
+        # Set the group number in the before hook and run the hook with the
+        # local variables set to the row before the new row is evaluated.
+        grp = row_index_to_group_index(old_k)
+        vars = old_row.merge(__group: grp)
+        ev.eval_before_hook(vars)
+        # Compute the new row.
         new_row = {}
         cols.each do |k|
           h = k.as_sym
@@ -638,8 +714,6 @@ module FatTable
         new_cols.each_pair do |key, val|
           key = key.as_sym
           vars = old_row.merge(new_row)
-          vars[:__row] = old_k + 1
-          vars[:__group] = row_index_to_group_index(old_k)
           case val
           when Symbol
             raise UserError, "Column '#{val}' in select does not exist" unless vars.keys.include?(val)
@@ -647,9 +721,13 @@ module FatTable
           when String
             new_row[key] = ev.evaluate(val, vars: vars)
           else
-            raise UserError, 'Hash parameters to select must be a symbol or string'
+            raise UserError, "Hash parameter '#{key}' to select must be a symbol or string"
           end
         end
+        # Set the group number and run the hook with the local variables set to
+        # the row after the new row is evaluated.
+        vars = new_row.merge(__group: grp)
+        ev.eval_after_hook(vars)
         result << new_row
       end
       result.boundaries = boundaries
@@ -675,13 +753,14 @@ module FatTable
         col = Column.new(header: h)
         result.add_column(col)
       end
-      ev = Evaluator.new(vars: { row: 0 },
-                         before: '@row = __row; @group = __group')
+      ev = Evaluator.new(ivars: { row: 0, group: 0 },
+                         before: '@row += 1')
       rows.each_with_index do |row, k|
-        vars = row.dup
-        vars[:__row] = k + 1
-        vars[:__group] = row_index_to_group_index(k)
+        grp = row_index_to_group_index(k)
+        vars = row.merge(__group: grp)
+        ev.eval_before_hook(vars)
         result << row if ev.evaluate(expr, vars: vars)
+        ev.eval_after_hook(vars)
       end
       result.normalize_boundaries
       result

data/lib/fat_table/version.rb

@@ -1,4 +1,4 @@
 module FatTable
   # The current version of FatTable
-  VERSION = '0.2.4'.freeze
+  VERSION = '0.2.6'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fat_table
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.6
 platform: ruby
 authors:
 - Daniel E. Doherty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-23 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: simplecov
@@ -185,7 +185,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dbi
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -199,7 +199,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dbd-pg
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -213,7 +213,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dbd-mysql
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -227,7 +227,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dbd-sqlite3
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -303,6 +303,7 @@ files:
 - lib/fat_table/formatters/org_formatter.rb
 - lib/fat_table/formatters/term_formatter.rb
 - lib/fat_table/formatters/text_formatter.rb
+- lib/fat_table/patches.rb
 - lib/fat_table/table.rb
 - lib/fat_table/version.rb
 homepage: https://github.com/ddoherty03/fat_table
@@ -326,7 +327,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Provides tools for working with tables as a data type.