reckon 0.6.0 → 0.7.2

data/lib/reckon.rb

@@ -16,4 +16,5 @@ require_relative 'reckon/date_column'
 require_relative 'reckon/money'
 require_relative 'reckon/ledger_parser'
 require_relative 'reckon/csv_parser'
+require_relative 'reckon/options'
 require_relative 'reckon/app'

data/lib/reckon/app.rb

@@ -8,9 +8,10 @@ module Reckon
     attr_accessor :options, :seen, :csv_parser, :regexps, :matcher
     @@cli = HighLine.new
 
-    def initialize(options = {})
+    def initialize(opts = {})
+      self.options = opts
       LOGGER.level = Logger::INFO if options[:verbose]
-      self.options = options
+
       self.regexps = {}
       self.seen = Set.new
       self.options[:currency] ||= '$'
@@ -19,10 +20,10 @@ module Reckon
       learn!
     end
 
-    def interactive_output(str)
+    def interactive_output(str, fh = $stdout)
       return if options[:unattended]
 
-      puts str
+      fh.puts str
     end
 
     def learn!
@@ -157,10 +158,10 @@ module Reckon
                   :money => @csv_parser.money_for(index),
                   :description => @csv_parser.description_for(index) }
       end
-      rows.sort_by { |n| n[:date] }.each { |row| yield row }
+      rows.sort_by { |n| [n[:date], -n[:money], n[:description]] }.each { |row| yield row }
     end
 
-    def print_transaction(rows)
+    def print_transaction(rows, fh = $stdout)
       str = "\n"
       header = %w[Date Amount Description Note]
       maxes = header.map(&:length)
@@ -184,7 +185,7 @@ module Reckon
         str += "\n"
       end
 
-      interactive_output str
+      interactive_output str, fh
     end
 
     def ask_account_question(msg, row)
@@ -192,11 +193,13 @@ module Reckon
       LOGGER.info "possible_answers===> #{possible_answers.inspect}"
 
       if options[:unattended]
-        default = if row[:pretty_money][0] == '-'
-                    options[:default_into_account] || 'Expenses:Unknown'
-                  else
-                    options[:default_outof_account] || 'Income:Unknown'
-                  end
+        if options[:fail_on_unknown_account] && possible_answers.empty?
+          raise %(Couldn't find any matches for '#{row[:description]}'
+            Try adding an account token with --account-tokens)
+        end
+
+        default = options[:default_outof_account]
+        default = options[:default_into_account] if row[:pretty_money][0] == '-'
         return possible_answers[0] || default
       end
 
@@ -252,7 +255,7 @@ module Reckon
     end
 
     def ledger_format(row, line1, line2)
-      out = "#{row[:pretty_date]}\t#{row[:description]}\t; #{row[:note]}\n"
+      out = "#{row[:pretty_date]}\t#{row[:description]}#{row[:note] ? "\t; " + row[:note]: ""}\n"
       out += "\t#{line1.first}\t\t\t#{line1.last}\n"
       out += "\t#{line2.first}\t\t\t#{line2.last}\n\n"
       out
@@ -277,138 +280,12 @@ module Reckon
       exit
     end
 
-    def output_table
+    def output_table(fh = $stdout)
       rows = []
       each_row_backwards do |row|
         rows << row
       end
-      print_transaction(rows)
-    end
-
-    def self.parse_opts(args = ARGV)
-      options = { :output_file => STDOUT }
-      parser = OptionParser.new do |opts|
-        opts.banner = "Usage: Reckon.rb [options]"
-        opts.separator ""
-
-        opts.on("-f", "--file FILE", "The CSV file to parse") do |file|
-          options[:file] = file
-        end
-
-        opts.on("-a", "--account NAME", "The Ledger Account this file is for") do |a|
-          options[:bank_account] = a
-        end
-
-        opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
-          options[:verbose] = v
-        end
-
-        opts.on("-i", "--inverse", "Use the negative of each amount") do |v|
-          options[:inverse] = v
-        end
-
-        opts.on("-p", "--print-table", "Print out the parsed CSV in table form") do |p|
-          options[:print_table] = p
-        end
-
-        opts.on("-o", "--output-file FILE", "The ledger file to append to") do |o|
-          options[:output_file] = File.open(o, 'a')
-        end
-
-        opts.on("-l", "--learn-from FILE", "An existing ledger file to learn accounts from") do |l|
-          options[:existing_ledger_file] = l
-        end
-
-        opts.on("", "--ignore-columns 1,2,5", "Columns to ignore in the CSV file - the first column is column 1") do |ignore|
-          options[:ignore_columns] = ignore.split(",").map { |i| i.to_i }
-        end
-
-        opts.on("", "--money-column 2", Integer, "Specify the money column instead of letting Reckon guess - the first column is column 1") do |column_number|
-          options[:money_column] = column_number
-        end
-
-        opts.on("", "--date-column 3", Integer, "Specify the date column instead of letting Reckon guess - the first column is column 1") do |column_number|
-          options[:date_column] = column_number
-        end
-
-        opts.on("", "--contains-header [N]", "The first row of the CSV is a header and should be skipped. Optionally add the number of rows to skip.") do |contains_header|
-          options[:contains_header] = 1
-          options[:contains_header] = contains_header.to_i if contains_header
-        end
-
-        opts.on("", "--csv-separator ','", "Separator for parsing the CSV - default is comma.") do |csv_separator|
-          options[:csv_separator] = csv_separator
-        end
-
-        opts.on("", "--comma-separates-cents", "Use comma instead of period to deliminate dollars from cents when parsing ($100,50 instead of $100.50)") do |c|
-          options[:comma_separates_cents] = c
-        end
-
-        opts.on("", "--encoding 'UTF-8'", "Specify an encoding for the CSV file; not usually needed") do |e|
-          options[:encoding] = e
-        end
-
-        opts.on("-c", "--currency '$'", "Currency symbol to use, defaults to $ (£, EUR)") do |e|
-          options[:currency] = e
-        end
-
-        opts.on("", "--date-format '%d/%m/%Y'", "Force the date format (see Ruby DateTime strftime)") do |d|
-          options[:date_format] = d
-        end
-
-        opts.on("-u", "--unattended", "Don't ask questions and guess all the accounts automatically. Used with --learn-from or --account-tokens options.") do |n|
-          options[:unattended] = n
-        end
-
-        opts.on("-t", "--account-tokens FILE", "YAML file with manually-assigned tokens for each account (see README)") do |a|
-          options[:account_tokens_file] = a
-        end
-
-        opts.on("", "--default-into-account NAME", "Default into account") do |a|
-          options[:default_into_account] = a
-        end
-
-        opts.on("", "--default-outof-account NAME", "Default 'out of' account") do |a|
-          options[:default_outof_account] = a
-        end
-
-        opts.on("", "--suffixed", "If --currency should be used as a suffix. Defaults to false.") do |e|
-          options[:suffixed] = e
-        end
-
-        opts.on_tail("-h", "--help", "Show this message") do
-          puts opts
-          exit
-        end
-
-        opts.on_tail("--version", "Show version") do
-          puts VERSION
-          exit
-        end
-
-        opts.parse!(args)
-      end
-
-      unless options[:file]
-        options[:file] = @@cli.ask("What CSV file should I parse? ")
-        unless options[:file].length > 0
-          puts "\nYou must provide a CSV file to parse.\n"
-          puts parser
-          exit
-        end
-      end
-
-      unless options[:bank_account]
-        fail "Please specify --account for the unattended mode" if options[:unattended]
-
-        options[:bank_account] = @@cli.ask("What is the account name of this bank account in Ledger? ") do |q|
-          q.readline = true
-          q.validate = /^.{2,}$/
-          q.default = "Assets:Bank:Checking"
-        end
-      end
-
-      options
+      print_transaction(rows, fh)
     end
   end
 end

data/lib/reckon/cosine_similarity.rb

@@ -1,47 +1,52 @@
 require 'matrix'
 require 'set'
 
-# Implementation of consine similarity using TF-IDF for vectorization.
-# Used to suggest which account a transaction should be assigned to
+# Implementation of cosine similarity using TF-IDF for vectorization.
+#
+# In information retrieval, tf–idf, short for term frequency–inverse document frequency,
+# is a numerical statistic that is intended to reflect how important a word is to a
+# document in a collection or corpus
+#
+# Cosine Similarity a measurement to determine how similar 2 documents are to each other.
+#
+# These weights and measures are used to suggest which account a transaction should be
+# assigned to.
 module Reckon
   class CosineSimilarity
+    DocumentInfo = Struct.new(:tokens, :accounts)
+
     def initialize(options)
+      @docs = DocumentInfo.new({}, {})
       @options = options
-      @tokens = {}
-      @accounts = Hash.new(0)
     end
 
     def add_document(account, doc)
-      tokenize(doc).each do |n|
+      tokens = tokenize(doc)
+      LOGGER.info "doc tokens: #{tokens}"
+      tokens.each do |n|
         (token, count) = n
 
-        @tokens[token] ||= {}
-        @tokens[token][account] ||= 0
-        @tokens[token][account] += count
-        @accounts[account] += count
+        @docs.tokens[token] ||= Hash.new(0)
+        @docs.tokens[token][account] += count
+        @docs.accounts[account] ||= Hash.new(0)
+        @docs.accounts[account][token] += count
       end
     end
 
     # find most similar documents to query
     def find_similar(query)
-      (query_scores, corpus_scores) = td_idf_scores_for(query)
+      LOGGER.info "find_similar #{query}"
 
-      query_vector = Vector.elements(query_scores, false)
+      accounts = docs_to_check(query).map do |a|
+        [a, tfidf(@docs.accounts[a])]
+      end
 
-      # For each doc, calculate the similarity to the query
-      suggestions = corpus_scores.map do |account, scores|
-        acct_vector = Vector.elements(scores, false)
+      q = tfidf(tokenize(query))
 
-        acct_query_dp = acct_vector.inner_product(query_vector)
-        # similarity is a float between 1 and -1, where 1 is exactly the same and -1 is
-        # exactly opposite
-        # see https://en.wikipedia.org/wiki/Cosine_similarity
-        # cos(theta) = (A . B) / (||A|| ||B||)
-        # where A . B is the "dot product" and ||A|| is the magnitude of A
-        # ruby has the 'matrix' library we can use to do these calculations.
+      suggestions = accounts.map do |a, d|
         {
-          similarity: acct_query_dp / (acct_vector.magnitude * query_vector.magnitude),
-          account: account,
+          similarity: calc_similarity(q, d),
+          account: a
         }
       end.select { |n| n[:similarity] > 0 }.sort_by { |n| -n[:similarity] }
 
@@ -52,50 +57,51 @@ module Reckon
 
     private
 
-    def td_idf_scores_for(query)
-      query_tokens = tokenize(query)
-      corpus = Set.new
-      corpus_scores = {}
-      query_scores = []
-      num_docs = @accounts.length
-
-      query_tokens.each do |n|
-        (token, _count) = n
-        next unless @tokens[token]
-        corpus = corpus.union(Set.new(@tokens[token].keys))
+    def docs_to_check(query)
+      return tokenize(query).reduce(Set.new) do |corpus, t|
+        corpus.union(Set.new(@docs.tokens[t[0]]&.keys))
       end
+    end
 
-      query_tokens.each do |n|
-        (token, count) = n
-
-        # if no other docs have token, ignore it
-        next unless @tokens[token]
+    def tfidf(tokens)
+      scores = {}
 
-        ## First, calculate scores for our query as we're building scores for the corpus
-        query_scores << calc_tf_idf(
-          count,
-          query_tokens.length,
-          @tokens[token].length,
-          num_docs
+      tokens.each do |t, n|
+        scores[t] = calc_tf_idf(
+          n,
+          tokens.length,
+          @docs.tokens[t]&.length&.to_f || 0,
+          @docs.accounts.length
         )
-
-        ## Next, calculate for the corpus, where our "account" is a document
-        corpus.each do |account|
-          corpus_scores[account] ||= []
-
-          corpus_scores[account] << calc_tf_idf(
-            (@tokens[token][account] || 0),
-            @accounts[account].to_f,
-            @tokens[token].length.to_f,
-            num_docs
-          )
-        end
       end
-      [query_scores, corpus_scores]
+
+      return scores
     end
 
-    def calc_tf_idf(token_count, num_words_in_doc, df, num_docs)
+    # Cosine similarity is used to compare how similar 2 documents are. Returns a float
+    # between 1 and -1, where 1 is exactly the same and -1 is exactly opposite.
+    #
+    # see https://en.wikipedia.org/wiki/Cosine_similarity
+    # cos(theta) = (A . B) / (||A|| ||B||)
+    # where A . B is the "dot product" and ||A|| is the magnitude of A
+    #
+    # The variables A and B are the set of unique terms in q and d.
+    #
+    # For example, when q = "big red balloon" and d ="small green balloon" then the
+    # variables are (big,red,balloon,small,green) and a = (1,1,1,0,0) and b =
+    # (0,0,1,1,1).
+    #
+    # query and doc are hashes of token => tf/idf score
+    def calc_similarity(query, doc)
+      tokens = Set.new(query.keys + doc.keys)
+
+      a = Vector.elements(tokens.map { |n| query[n] || 0 }, false)
+      b = Vector.elements(tokens.map { |n| doc[n] || 0 }, false)
+
+      return a.inner_product(b) / (a.magnitude * b.magnitude)
+    end
 
+    def calc_tf_idf(token_count, num_words_in_doc, df, num_docs)
       # tf(t,d) = count of t in d / number of words in d
       tf = token_count / num_words_in_doc.to_f
 
@@ -109,14 +115,13 @@ module Reckon
     end
 
     def tokenize(str)
-      mk_tokens(str).inject(Hash.new(0)) do |memo, n|
+      mk_tokens(str).each_with_object(Hash.new(0)) do |n, memo|
         memo[n] += 1
-        memo
       end.to_a
     end
 
     def mk_tokens(str)
-      str.downcase.tr(';', ' ').tr("'", '').split(/[^a-z0-9.]+/)
+      str.downcase.tr(';', ' ').tr("'", '').split(/[^a-z0-9.]+/).reject(&:empty?)
     end
   end
 end

data/lib/reckon/csv_parser.rb

@@ -89,12 +89,7 @@ module Reckon
           money_score += Money::likelihood( entry )
           possible_neg_money_count += 1 if entry =~ /^\$?[\-\(]\$?\d+/
           possible_pos_money_count += 1 if entry =~ /^\+?\$?\+?\d+/
-          date_score += 10 if entry =~ /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i
-          date_score += 5 if entry =~ /^[\-\/\.\d:\[\]]+$/
-          date_score += entry.gsub(/[^\-\/\.\d:\[\]]/, '').length if entry.gsub(/[^\-\/\.\d:\[\]]/, '').length > 3
-          date_score -= entry.gsub(/[\-\/\.\d:\[\]]/, '').length
-          date_score += 30 if entry =~ /^\d+[:\/\.-]\d+[:\/\.-]\d+([ :]\d+[:\/\.]\d+)?$/
-          date_score += 10 if entry =~ /^\d+\[\d+:GMT\]$/i
+          date_score += DateColumn.likelihood(entry)
 
           # Try to determine if this is a balance column
           entry_as_num = entry.gsub(/[^\-\d\.]/, '').to_f
@@ -168,7 +163,7 @@ module Reckon
       results = evaluate_columns(columns)
 
       if options[:money_column]
-        self.money_column_indices = [ options[:money_column] - 1 ]
+        self.money_column_indices = [options[:money_column] - 1]
       else
         self.money_column_indices = results.select { |n| n[:is_money_column] }.map { |n| n[:index] }
         if self.money_column_indices.length == 1

data/lib/reckon/date_column.rb

@@ -56,5 +56,15 @@ module Reckon
       date.iso8601
     end
 
+    def self.likelihood(entry)
+      date_score = 0
+      date_score += 10 if entry =~ /\b(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i
+      date_score += 5 if entry =~ /^[\-\/\.\d:\[\]]+$/
+      date_score += entry.gsub(/[^\-\/\.\d:\[\]]/, '').length if entry.gsub(/[^\-\/\.\d:\[\]]/, '').length > 3
+      date_score -= entry.gsub(/[\-\/\.\d:\[\]]/, '').length
+      date_score += 30 if entry =~ /^\d+[:\/\.-]\d+[:\/\.-]\d+([ :]\d+[:\/\.]\d+)?$/
+      date_score += 10 if entry =~ /^\d+\[\d+:GMT\]$/i
+      return date_score
+    end
   end
 end

data/lib/reckon/money.rb

@@ -5,12 +5,13 @@ module Reckon
   class Money
     include Comparable
     attr_accessor :amount, :currency, :suffixed
-    def initialize( amount, options = {} )
-      if options[:inverse]
-        @amount = -1*amount.to_f
-      else
-        @amount = amount.to_f
-      end
+    def initialize(amount, options = {})
+      @options = options
+      @amount_raw = amount
+      @raw = options[:raw]
+
+      @amount = parse(amount, options)
+      @amount = -@amount if options[:inverse]
       @currency = options[:currency] || "$"
       @suffixed = options[:suffixed]
     end
@@ -19,11 +20,19 @@ module Reckon
       return @amount
     end
 
+    def to_s
+      return @options[:raw] ? "#{@amount_raw} | #{@amount}" : @amount
+    end
+
+    # unary minus
+    # ex
+    # m = Money.new
+    # -m
     def -@
-      Money.new( -@amount, :currency => @currency, :suffixed => @suffixed )
+      Money.new(-@amount, :currency => @currency, :suffixed => @suffixed)
     end
 
-    def <=>( mon )
+    def <=>(mon)
       other_amount = mon.to_f
       if @amount < other_amount
         -1
@@ -34,42 +43,41 @@ module Reckon
       end
     end
 
-    def pretty( negate = false )
-      if @suffixed
-        (@amount >= 0 ? " " : "") + sprintf("%0.2f #{@currency}", @amount * (negate ? -1 : 1))
-      else
-        (@amount >= 0 ? " " : "") + sprintf("%0.2f", @amount * (negate ? -1 : 1)).gsub(/^((\-)|)(?=\d)/, "\\1#{@currency}")
+    def pretty(negate = false)
+      if @raw
+        return @amount_raw unless negate
+
+        return @amount_raw[0] == '-' ? @amount_raw[1..-1] : "-#{@amount_raw}"
       end
+
+      amt = pretty_amount(@amount * (negate ? -1 : 1))
+      amt = if @suffixed
+              "#{amt} #{@currency}"
+            else
+              amt.gsub(/^((-)|)(?=\d)/, "\\1#{@currency}")
+            end
+
+      return (@amount >= 0 ? " " : "") + amt
+    end
+
+    def pretty_amount(amount)
+      sprintf("%0.2f", amount).reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
     end
 
-    def Money::from_s( value, options = {} )
+    def parse(value, options = {})
+      value = value.to_s
       # Empty string is treated as money with value 0
-      return Money.new( 0.00, options ) if value.empty?
-
-      # Remove 1000 separaters and replace , with . if comma_separates_cents
-      # 1.000,00 -> 1000.00
-      value = value.gsub(/\./, '').gsub(/,/, '.') if options[:comma_separates_cents]
-      value = value.gsub(/,/, '')
-
-      money_format_regex = /^(.*?)(\d+\.\d\d)/ # Money has two decimal precision
-      any_number_regex = /^(.*?)([\d\.]+)/
-
-      # Prefer matching the money_format, match any number otherwise
-      m = value.match( money_format_regex ) ||
-        value.match( any_number_regex )
-      if m
-        amount = m[2].to_f
-        # Check whether the money had a - or (, which indicates negative amounts
-        if (m[1].match( /^[\(-]/ ) || m[1].match( /-$/  ))
-          amount *= -1
-        end
-        return Money.new( amount, options )
-      else
-        return nil
-      end
+      return value.to_f if value.to_s.empty?
+
+      invert = value.match(/^\(.*\)$/)
+      value = value.gsub(/[^0-9,.-]/, '')
+      value = value.tr('.', '').tr(',', '.') if options[:comma_separates_cents]
+      value = value.tr(',', '')
+      value = value.to_f
+      return invert ? -value : value
     end
 
-    def Money::likelihood( entry )
+    def Money::likelihood(entry)
       money_score = 0
       # digits separated by , or . with no more than 2 trailing digits
       money_score += 40 if entry.match(/\d+[,.]\d{2}[^\d]*$/)
@@ -83,31 +91,30 @@ module Reckon
   end
 
   class MoneyColumn < Array
-    def initialize( arr = [], options = {} )
-      arr.each { |str| self.push( Money.from_s( str, options ) ) }
+    def initialize(arr = [], options = {})
+      arr.each { |str| push(Money.new(str, options)) }
     end
 
     def positive?
-      self.each do |money|
-        return false if money < 0 if money
+      each do |money|
+        return false if money && money < 0
       end
       true
     end
 
-    def merge!( other_column )
+    def merge!(other_column)
       invert = false
-      invert = true if self.positive? && other_column.positive?
-      self.each_with_index do |mon, i|
+      invert = true if positive? && other_column.positive?
+      each_with_index do |mon, i|
         other = other_column[i]
-        return nil if (!mon || !other)
-        if mon != 0.00 && other == 0.0
-          if invert
-            self[i]= -mon
-          end
-        elsif mon == 0.00 && other != 0.00
+        return nil if !mon || !other
+
+        if mon != 0.0 && other == 0.0
+          self[i] = -mon if invert
+        elsif mon == 0.0 && other != 0.0
           self[i] = other
         else
-          return nil
+          self[i] = Money.new(0)
         end
       end
       self