reckon 0.9.6 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f1e02a1c5dde18325b138afa66518fc2516ea30e8c39eae5cdff2808f5f8bb4
-  data.tar.gz: 3d70362d3b8c8a3e6e147f4ca3466e35db7905e2bfe60571de9d219c330e23db
+  metadata.gz: 58ef387a88fd33e7aeddc98842e3c5cf147e74830b659393ec5f75d0179d6bba
+  data.tar.gz: 3983fafdcce5b9e016dfd09cfa7a905bc0bf829d3b15f383f162b5738bbf0f49
 SHA512:
-  metadata.gz: 707f23ee5da2df3837f0ae5c520e8b62d802d812dd9ab3acef72853c5e7d27d51b7aeaa269f89c66de1b73ab0e6891fb9ced9d3f59fb11c49d7679fa84c0e4f5
-  data.tar.gz: 73bbdf6dec9a09c23735695d8c192fde6b94df1d4931e7e79a256bed20a736e46d1503985dc29689f7f4fbaefd5c11413f4a41eb58516691b77daef99c2c8c75
+  metadata.gz: 44731aea841dc2c3a4830085eda5dccc3b8958502eb21b8aee7ea4d52f2429e7dc2125de4c15084c804970ebc1199a859ef87437518f9635d0f8c33ae7c4444c
+  data.tar.gz: c2f4ed1e9ccf0f169a285d5337b3f97b7867d018a8541f4c009fe4dc105f0141884066071860a68b9baf688452b44ccc34b109d6ad4ff576645213c0ece3019e

data/.rubocop.yml

@@ -18,3 +18,15 @@ Metrics/AbcSize:
 
 Style/NumericPredicate:
   Enabled: False
+
+Metrics/PerceivedComplexity:
+  Enabled: False
+
+Metrics/CyclomaticComplexity:
+  Enabled: False
+
+Style/FormatString:
+  Enabled: False
+
+Naming/MethodParameterName:
+  Enabled: False

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [v0.11.0](https://github.com/cantino/reckon/tree/v0.11.0) (2025-03-04)
+
+[Full Changelog](https://github.com/cantino/reckon/compare/v0.10.0...v0.11.0)
+
+**Closed issues:**
+
+- Feature request: confidence score [\#134](https://github.com/cantino/reckon/issues/134)
+- feature request [\#133](https://github.com/cantino/reckon/issues/133)
+
+## [v0.10.0](https://github.com/cantino/reckon/tree/v0.10.0) (2024-11-27)
+
+[Full Changelog](https://github.com/cantino/reckon/compare/v0.9.6...v0.10.0)
+
+**Closed issues:**
+
+- Is it possible to use reckon with only no info about incoming or going out? [\#131](https://github.com/cantino/reckon/issues/131)
+- Reckon fails immediately with error about uninitialized constant `Readline` [\#129](https://github.com/cantino/reckon/issues/129)
+
+## [v0.9.6](https://github.com/cantino/reckon/tree/v0.9.6) (2024-03-27)
+
+[Full Changelog](https://github.com/cantino/reckon/compare/v0.9.5...v0.9.6)
+
+**Closed issues:**
+
+- reckon can't learn from a file with the "wrong" date format [\#130](https://github.com/cantino/reckon/issues/130)
+
 ## [v0.9.5](https://github.com/cantino/reckon/tree/v0.9.5) (2024-01-08)
 
 [Full Changelog](https://github.com/cantino/reckon/compare/v0.9.4...v0.9.5)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    reckon (0.9.6)
+    reckon (0.11.0)
       chronic (>= 0.3.0)
       highline (~> 2.0)
       matrix (>= 0.4.2)
@@ -21,7 +21,7 @@ GEM
       method_source (~> 1.0)
     rake (13.0.6)
     rantly (1.2.0)
-    rchardet (1.8.0)
+    rchardet (1.9.0)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)

data/README.md

@@ -49,6 +49,8 @@ Learn more:
                                      Column number of the money columns, starts from 1 (1 or 2 columns)
         --raw-money
                                      Don't format money column (for stocks)
+        --sort DATE|DESC|AMT
+                                     Sort file by date, description, or amount
         --date-column 3
                                      Column number of the date column, starts from 1
         --contains-header [N]
@@ -106,6 +108,9 @@ Would tokenize to 'ING', 'Direct' and 'Deposit'.  The matcher would then suggest
 Here's an example of `tokens.yaml`:
 
 ```
+config:
+    similarity_threshold: 2  # range 0-10
+
 Income:
   Salary:
     - 'LÖN'
@@ -124,6 +129,8 @@ Expenses:
 
 Reckon will use `Income:Unknown` or `Expenses:Unknown` if it can't match a transaction to an account.
 
+The config key is a special key used to set configuration when running in unattended mode. The only config variable is similarity_threshold (currently).
+
 You can override these names with the `--default_outof_account` and `--default_into_account` options.
 
 ### Substring Match

data/lib/reckon/app.rb

@@ -3,6 +3,8 @@
 require 'yaml'
 require 'stringio'
 
+UnattendedConfig = Struct.new(:similarity_threshold)
+
 module Reckon
   # The main app
   class App
@@ -10,10 +12,11 @@ module Reckon
 
     def initialize(opts = {})
       self.options = opts
-      LOGGER.level = Logger::INFO if options[:verbose]
+      LOGGER.level = options[:verbose] || Logger::WARN
 
       self.regexps = {}
       self.seen = Set.new
+      options[:sort] ||= :date
       @cli = HighLine.new
       @csv_parser = CSVParser.new(options)
       @matcher = CosineSimilarity.new(options)
@@ -41,7 +44,11 @@ module Reckon
 
       raise "#{filename} doesn't exist!" unless File.exist?(filename)
 
-      extract_account_tokens(YAML.load_file(filename)).each do |account, tokens|
+      tokens = YAML.load_file(filename)
+      cfg = build_unattended_config(tokens.delete('config'))
+      @options[:similarity_threshold] = cfg.similarity_threshold if cfg
+
+      extract_account_tokens(tokens).each do |account, tokens|
         tokens.each do |t|
           if t.start_with?('/')
             add_regexp(account, t)
@@ -52,6 +59,13 @@ module Reckon
       end
     end
 
+    def build_unattended_config(cfg)
+      return unless cfg
+      invalid = cfg.keys - UnattendedConfig.members.map(&:to_s)
+      raise "Invalid keys in config: #{invalid}" if invalid.any?
+      return UnattendedConfig.new(*cfg.values_at(*UnattendedConfig.members.map(&:to_s)))
+    end
+
     def learn_from_ledger_file(ledger_file)
       return unless ledger_file
 
@@ -80,7 +94,7 @@ module Reckon
 
     # Add tokens from account_tokens_file to accounts
     def extract_account_tokens(subtree, account = nil)
-      if subtree.nil?
+      if subtree.nil? || !subtree
         puts "Warning: empty #{account} tree"
         {}
       elsif subtree.is_a?(Array)
@@ -141,6 +155,11 @@ module Reckon
           line2 = [options[:bank_account], row[:pretty_money]]
         end
 
+        if answer == '~~SKIP~~'
+          LOGGER.info "skipping transaction: #{row}"
+          next
+        end
+
         finish if %w[quit q].include?(answer)
         if %w[skip s].include?(answer)
           interactive_output "Skipping"
@@ -168,18 +187,29 @@ module Reckon
                   :money => @csv_parser.money_for(index),
                   :description => @csv_parser.description_for(index) }
       end
-      rows.sort_by { |n| [n[:date], -n[:money], n[:description]] }.each { |row| yield row }
+      rows.sort_by do |n|
+        [n[options[:sort]], -n[:money], n[:description]]
+      end.each do |row|
+        yield row
+      end
     end
 
     def print_transaction(rows, fh = $stdout)
       str = "\n"
-      header = %w[Date Amount Description Note]
+      header = %w[Date Amount Description]
+      header += ["Note"] if rows.map { |r| r[:note] }.any?
       maxes = header.map(&:length)
-
-      rows = rows.map { |r| [r[:pretty_date], r[:pretty_money], r[:description], r[:note]] }
+      rows = rows.map do |r|
+        [r[:pretty_date], r[:pretty_money], r[:description], r[:note]].compact
+      end
 
       rows.each do |r|
-        r.length.times { |i| l = r[i] ? r[i].length : 0; maxes[i] = l if maxes[i] < l }
+        r.length.times do |i|
+          l = 0
+          l = r[i].length if r[i]
+          maxes[i] ||= 0
+          maxes[i] = l if maxes[i] < l
+        end
       end
 
       header.each_with_index do |n, i|
@@ -199,6 +229,15 @@ module Reckon
     end
 
     def ask_account_question(msg, row)
+      # return account token if it matches
+      token_answer = most_specific_regexp_match(row)
+      if token_answer.any?
+        row[:note] = "Matched account token"
+        puts "NOTE: Matched account token"
+        puts token_answer[0]
+        return token_answer[0]
+      end
+
       possible_answers = suggest(row)
       LOGGER.info "possible_answers===> #{possible_answers.inspect}"
 
@@ -261,7 +300,10 @@ module Reckon
 
     def suggest(row)
       most_specific_regexp_match(row) +
-        @matcher.find_similar(row[:description]).map { |n| n[:account] }
+        @matcher.find_similar(row[:description]).filter do |n|
+          !@options[:similarity_threshold] ||
+          n.fetch(:simliarity, 0) * 10 >= @options[:similarity_threshold]
+        end.map { |n| n[:account] }
     end
 
     def output(ledger_line)

data/lib/reckon/cosine_similarity.rb

@@ -12,6 +12,7 @@ require 'set'
 # These weights and measures are used to suggest which account a transaction should be
 # assigned to.
 module Reckon
+  # Calculates cosine similarity for tf/idf
   class CosineSimilarity
     DocumentInfo = Struct.new(:tokens, :accounts)
 

data/lib/reckon/csv_parser.rb

@@ -64,13 +64,13 @@ module Reckon
     private
 
     def filter_csv
-      if options[:ignore_columns]
-        new_columns = []
-        columns.each_with_index do |column, index|
-          new_columns << column unless options[:ignore_columns].include?(index + 1)
-        end
-        @columns = new_columns
+      return unless options[:ignore_columns]
+
+      new_columns = []
+      columns.each_with_index do |column, index|
+        new_columns << (options[:ignore_columns].include?(index + 1) ? [''] * column.length : column)
       end
+      @columns = new_columns
     end
 
     def evaluate_columns(cols)
@@ -222,7 +222,8 @@ module Reckon
       # convert to a stringio object to handle multi-line fields
       parser_opts = {
         col_sep: separator,
-        skip_blanks: true
+        skip_blanks: true,
+        row_sep: :auto
       }
       begin
         rows = CSV.parse(StringIO.new(data), **parser_opts)
@@ -235,7 +236,7 @@ module Reckon
           index = data.index("\n", index) + 1 # skip over newline character
           count += 1
         end
-        rows = CSV.parse(StringIO.new(data[index..-1]), **parser_opts)
+        rows = CSV.parse(StringIO.new(data[index..]), **parser_opts)
         rows[0..-footer_lines_to_skip]
       end
     end

data/lib/reckon/ledger_parser.rb

@@ -175,7 +175,7 @@ module Reckon
     end
 
     def format_row(row, line1, line2)
-      note = row[:note] ? "\t; row[:note]" : ""
+      note = row[:note] ? "\t; #{row[:note]}" : ""
       out = "#{row[:pretty_date]}\t#{row[:description]}#{note}\n"
       out += "\t#{line1.first}\t\t\t#{line1.last}\n"
       out += "\t#{line2.first}\t\t\t#{line2.last}\n\n"

data/lib/reckon/logger.rb

@@ -1,4 +1,8 @@
 module Reckon
   LOGGER = Logger.new(STDERR)
   LOGGER.level = Logger::WARN
+
+  def log(tag, msg)
+    LOGGER.add(Logger::WARN, msg, tag)
+  end
 end

data/lib/reckon/options.rb

@@ -4,7 +4,6 @@ module Reckon
   # Singleton class for parsing command line flags
   class Options
     def self.parse_command_line_options(args = ARGV, stdin = $stdin)
-      cli = HighLine.new
       options = { output_file: $stdout }
       OptionParser.new do |opts|
         opts.banner = "Usage: Reckon.rb [options]"
@@ -18,8 +17,17 @@ module Reckon
           options[:bank_account] = a
         end
 
-        opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
-          options[:verbose] = v
+        options[:verbose] = Logger::WARN
+        opts.on("-v", "--v", "Run verbosely (show info log messages)") do
+          options[:verbose] = Logger::INFO
+        end
+
+        opts.on("", "--verbose", "Run verbosely (show info log messages)") do
+          options[:verbose] = Logger::INFO
+        end
+
+        opts.on("", "--vv", "Run very verbosely (show debug log messages)") do
+          options[:verbose] = Logger::DEBUG
         end
 
         opts.on("-i", "--inverse", "Use the negative of each amount") do |v|
@@ -58,6 +66,19 @@ module Reckon
           options[:raw] = n
         end
 
+        options[:sort] = :date
+        opts.on("", "--sort DATE|DESC|AMT", "Sort file by date, description, or amount") do |s|
+          if s == 'DESC'
+            options[:sort] = :description
+          elsif s == 'AMT'
+            options[:sort] = :money
+          elsif s == 'DATE'
+            options[:sort] = :date
+          else
+            raise "'#{s}' is not valid. valid sort options are DATE, DESC, AMT"
+          end
+        end
+
         opts.on("", "--date-column 3", Integer,
                 "Column number of the date column, starts from 1") do |col|
           options[:date_column] = col
@@ -161,26 +182,35 @@ module Reckon
         options[:string] = stdin.read
       end
 
+      validate_options(options)
+
+      return options
+    end
+
+    def self.validate_options(options)
+      cli = HighLine.new
       unless options[:file]
         options[:file] = cli.ask("What CSV file should I parse? ")
-        unless options[:file].empty?
-          puts "\nYou must provide a CSV file to parse.\n"
-          puts parser
+        if options[:file].empty?
+          puts "\nERROR: You must provide a CSV file to parse.\n"
           exit
         end
       end
 
       unless options[:bank_account]
-        raise "Must specify --account in unattended mode" if options[:unattended]
+        if options[:unattended]
+          puts "ERROR: Must specify --account in unattended mode"
+          exit
+        end
 
-        options[:bank_account] = cli.ask("What is this account named in Ledger?\n") do |q|
+        options[:bank_account] = cli.ask("What is the Ledger account name?\n") do |q|
           q.readline = true
           q.validate = /^.{2,}$/
           q.default = "Assets:Bank:Checking"
         end
       end
 
-      return options
+      return true
     end
   end
 end

data/lib/reckon/version.rb

@@ -1,3 +1,3 @@
 module Reckon
-  VERSION = "0.9.6"
+  VERSION = "0.11.0"
 end

data/spec/cosine_training_and_test.rb

@@ -8,7 +8,7 @@ ledger_file = ARGV[0]
 account = ARGV[1]
 seed = ARGV[2] ? ARGV[2].to_i : Random.new_seed
 
-ledger = Reckon::LedgerParser.new(File.new(ledger_file))
+ledger = Reckon::LedgerParser.new.parse(File.new(ledger_file))
 matcher = Reckon::CosineSimilarity.new({})
 
 train = []
@@ -50,3 +50,4 @@ end
 # pp result.compact
 puts "using #{seed} as random seed"
 puts "true: #{result.count(nil)} false: #{result.count { |v| !v.nil? }}"
+puts(result.filter { |v| !v.nil? })

data/spec/integration/ask_for_account/expected_output

@@ -1,11 +1,11 @@
 
-    Date    |   Amount   |                    Description                     | Note |
- 2003-12-24 |  $2,105.00 | CREDIT; Some Company vendorpymt PPD ID: 5KL3832735 |      |
- 2004-12-24 |   -$116.22 |         CREDIT; PAYPAL TRANSFER PPD ID: PAYPALSDSL |      |
- 2005-12-24 |     -$0.96 | DEBIT; WEBSITE-BALANCE-10DEC09 12 12/10WEBSITE-BAL |      |
- 2006-12-24 |      $0.23 | DEBIT; WEBSITE-BALANCE-17DEC09 12 12/17WEBSITE-BAL |      |
- 2007-12-24 |  $1,558.52 |     CREDIT; Blarg BLARG REVENUE PPD ID: 00jah78563 |      |
- 2008-12-24 |  $3,520.00 | CREDIT; Some Company vendorpymt PPD ID: 59728JSL20 |      |
- 2009-12-24 |     -$7.00 |       DEBIT; GITHUB 041287430274 CA 12/22GITHUB 04 |      |
- 2010-12-24 |    -$20.00 |                                  CHECK; CHECK 2656 |      |
- 2011-12-24 |    -$85.00 |         DEBIT; HOST 037196321563 MO 12/22SLICEHOST |      |
+    Date    |   Amount   |                    Description                     |
+ 2003-12-24 |  $2,105.00 | CREDIT; Some Company vendorpymt PPD ID: 5KL3832735 |
+ 2004-12-24 |   -$116.22 |         CREDIT; PAYPAL TRANSFER PPD ID: PAYPALSDSL |
+ 2005-12-24 |     -$0.96 | DEBIT; WEBSITE-BALANCE-10DEC09 12 12/10WEBSITE-BAL |
+ 2006-12-24 |      $0.23 | DEBIT; WEBSITE-BALANCE-17DEC09 12 12/17WEBSITE-BAL |
+ 2007-12-24 |  $1,558.52 |     CREDIT; Blarg BLARG REVENUE PPD ID: 00jah78563 |
+ 2008-12-24 |  $3,520.00 | CREDIT; Some Company vendorpymt PPD ID: 59728JSL20 |
+ 2009-12-24 |     -$7.00 |       DEBIT; GITHUB 041287430274 CA 12/22GITHUB 04 |
+ 2010-12-24 |    -$20.00 |                                  CHECK; CHECK 2656 |
+ 2011-12-24 |    -$85.00 |         DEBIT; HOST 037196321563 MO 12/22SLICEHOST |

data/spec/integration/unattended_config/input.csv

@@ -0,0 +1,9 @@
+DEBIT,20091224120000[0:GMT],"HOST 037196321563 MO        12/22SLICEHOST",-85.00
+CHECK,20091224120000[0:GMT],"Book Store",-20.00
+DEBIT,20091224120000[0:GMT],"GITHUB 041287430274 CA           12/22GITHUB 04",-7.00
+CREDIT,20091223120000[0:GMT],"Some Company vendorpymt                 PPD ID: 59728JSL20",3520.00
+CREDIT,20091223120000[0:GMT],"Blarg BLARG REVENUE                  PPD ID: 00jah78563",1558.52
+DEBIT,20091221120000[0:GMT],"WEBSITE-BALANCE-17DEC09 12        12/17WEBSITE-BAL",-12.23
+DEBIT,20091214120000[0:GMT],"WEBSITE-BALANCE-10DEC09 12        12/10WEBSITE-BAL",-20.96
+CREDIT,20091211120000[0:GMT],"PAYPAL           TRANSFER                   PPD ID: PAYPALSDSL",-116.22
+CREDIT,20091210120000[0:GMT],"Some Company vendorpymt                 PPD ID: 5KL3832735",2105.00

data/spec/integration/unattended_config/output.ledger

@@ -0,0 +1,36 @@
+2009-12-10	CREDIT; Some Company vendorpymt PPD ID: 5KL3832735
+	Assets:Bank:Checking			 $2,105.00
+	Income:Unknown			
+
+2009-12-11	CREDIT; PAYPAL TRANSFER PPD ID: PAYPALSDSL
+	Expenses:Unknown			
+	Assets:Bank:Checking			-$116.22
+
+2009-12-14	DEBIT; WEBSITE-BALANCE-10DEC09 12 12/10WEBSITE-BAL
+	Expenses:Websites			
+	Assets:Bank:Checking			-$20.96
+
+2009-12-21	DEBIT; WEBSITE-BALANCE-17DEC09 12 12/17WEBSITE-BAL
+	Expenses:Websites			
+	Assets:Bank:Checking			-$12.23
+
+2009-12-23	CREDIT; Some Company vendorpymt PPD ID: 59728JSL20
+	Assets:Bank:Checking			 $3,520.00
+	Income:Unknown			
+
+2009-12-23	CREDIT; Blarg BLARG REVENUE PPD ID: 00jah78563
+	Assets:Bank:Checking			 $1,558.52
+	Income:Unknown			
+
+2009-12-24	DEBIT; GITHUB 041287430274 CA 12/22GITHUB 04
+	Expenses:Unknown			
+	Assets:Bank:Checking			-$7.00
+
+2009-12-24	CHECK; Book Store
+	Expenses:Unknown			
+	Assets:Bank:Checking			-$20.00
+
+2009-12-24	DEBIT; HOST 037196321563 MO 12/22SLICEHOST
+	Expenses:Unknown			
+	Assets:Bank:Checking			-$85.00
+

data/spec/integration/unattended_config/test_args

@@ -0,0 +1,2 @@
+-f input.csv --unattended --account 'Assets:Bank:Checking' \
+   --account-tokens tokens.yml

data/spec/integration/unattended_config/tokens.yml

@@ -0,0 +1,19 @@
+config:
+  similarity_threshold: 6
+
+Income:
+  Salary:
+    - 'LÖN'
+    - 'Salary'
+Expenses:
+  Bank:
+    - 'Comission'
+    - /mastercard/i
+  Rent:
+    - '0011223344' # Landlord bank number
+  Websites:
+    - /web/i
+  Books:
+    - 'Book'
+'[Internal:Transfer]': # Virtual account
+  - '4433221100' # Your own account number

data/spec/reckon/ledger_parser_spec.rb

@@ -59,7 +59,7 @@ describe Reckon::LedgerParser do
           ledger += choose(*single_line_comments) + "\n"
           ledger
         end
-      end.check(1000) do |s|
+      end.check(100) do |s|
         filter_format = lambda { |n|
           [n['date'], n['desc'], n['name'],
            sprintf("%.02f", n['amount'])]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reckon
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.11.0
 platform: ruby
 authors:
 - Andrew Cantino
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-03-27 00:00:00.000000000 Z
+date: 2025-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -274,6 +274,10 @@ files:
 - spec/integration/two_money_columns_manual/input.csv
 - spec/integration/two_money_columns_manual/output.ledger
 - spec/integration/two_money_columns_manual/test_args
+- spec/integration/unattended_config/input.csv
+- spec/integration/unattended_config/output.ledger
+- spec/integration/unattended_config/test_args
+- spec/integration/unattended_config/tokens.yml
 - spec/integration/yyyymmdd_date_example/input.csv
 - spec/integration/yyyymmdd_date_example/output.ledger
 - spec/integration/yyyymmdd_date_example/test_args