appydave-tools 0.85.0 → 0.86.0

data/lib/appydave/tools/bank_reconciliation/_doc.md

@@ -0,0 +1,36 @@
+# Bank reconciliation
+
+[ChatGPT conversation](https://chatgpt.com/c/5d382562-95e5-4243-9b74-c3807d363486)
+
+
+## Code structure
+
+```bash
+├─ lib
+│ ├─ appydave
+│ │ └─ tools
+│ │   ├─ bank_reconciliation
+│ │   │ ├─ clean
+│ │   │ │ ├─ read_transactions.rb
+│ │   │ │ ├─ transaction_cleaner.rb
+│ │   │ ├─ models
+│ │   │ │ ├─ raw_transaction.rb
+│ │   │ │ └─ reconciled_transaction.rb
+│ │   └─ configuration
+│ │     └─ models
+│ │       └─ bank_reconciliation_config.rb
+└─ spec
+  ├─ appydave
+  │ ├─ tools
+  │ │ ├─ bank_reconciliation
+  │ │ │ ├─ clean
+  │ │ │ │ ├─ read_transactions_spec.rb
+  │ │ │ ├─ models
+  │ │ │ │ └─ raw_transaction_spec.rb
+  │ │ └─ configuration
+  │ │   └─ models
+  │ │     └─ bank_reconciliation_config_spec.rb
+  └─ fixtures
+    └─ bank-reconciliation
+      └─ bank-west.csv
+```

data/lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+require 'clipboard'
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Clean transactions
+        class CleanTransactions
+          include KLog::Logging
+          include Appydave::Tools::Configuration::Configurable
+          include Appydave::Tools::Debuggable
+
+          attr_reader :transaction_folder
+          attr_reader :output_folder
+          attr_reader :transactions
+
+          # (config_file)
+          def initialize(transaction_folder: nil, output_folder: nil, debug: false)
+            @debug = debug
+            # needs to use config.bank_reconciliation.transaction_folder
+            transaction_folder ||= '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions'
+            output_folder ||= File.join(transaction_folder, 'clean')
+
+            @transaction_folder = transaction_folder
+            @output_folder = output_folder
+          end
+
+          def clean_transactions(input_globs, output_file)
+            log_info("Starting transaction cleaning with input patterns: #{input_globs}")
+
+            raw_transactions = grab_raw_transactions(input_globs)
+            log_info("Total raw transactions collected: #{raw_transactions.size}")
+
+            transactions, duplicates_count = deduplicate_across_files(raw_transactions)
+            log_info("Duplicates found and removed: #{duplicates_count}")
+
+            transactions = Mapper.new.map(transactions)
+            log_info('Transactions mapped to chart of accounts and bank accounts')
+
+            log_kv 'Deduped consolidated transactions', duplicates_count if duplicates_count.positive?
+
+            save_to_csv(transactions, output_file)
+
+            csv_to_clipboard(output_file)
+
+            @transactions = transactions
+          end
+
+          private
+
+          def grab_raw_transactions(input_globs)
+            original_dir = Dir.pwd
+            transactions = []
+
+            begin
+              Dir.chdir(transaction_folder)
+
+              input_globs.each do |glob|
+                Dir.glob(glob).each do |file|
+                  log_kv 'Reading transactions from', file
+                  raw_transactions = ReadTransactions.new(file).read
+                  deduped_transactions, duplicates_count = deduplicate_within_file(raw_transactions)
+
+                  if duplicates_count.positive?
+                    log_kv 'Duplicates count within file', duplicates_count
+                    log_kv 'File', file
+                  end
+
+                  transactions += deduped_transactions
+                end
+              end
+            ensure
+              Dir.chdir(original_dir)
+            end
+
+            transactions
+          end
+
+          def deduplicate_within_file(transactions)
+            unique_transactions = transactions.uniq do |transaction|
+              [
+                transaction.bsb_number,
+                transaction.account_number,
+                transaction.transaction_date,
+                transaction.narration,
+                transaction.cheque_number,
+                transaction.debit,
+                transaction.credit,
+                transaction.balance,
+                transaction.transaction_type
+              ]
+            end
+
+            duplicates = transactions.size - unique_transactions.size
+
+            [unique_transactions, duplicates]
+          end
+
+          def deduplicate_across_files(transactions)
+            grouped_transactions = transactions.group_by do |transaction|
+              [
+                transaction.bsb_number,
+                transaction.account_number,
+                transaction.transaction_date,
+                transaction.narration,
+                transaction.cheque_number,
+                transaction.debit,
+                transaction.credit,
+                transaction.balance,
+                transaction.transaction_type
+              ]
+            end
+
+            unique_transactions = []
+            duplicates_count = 0
+
+            grouped_transactions.each_value do |dupes|
+              unique_transaction = dupes.first
+              unique_transaction.source_files = dupes.flat_map(&:source_files).uniq
+              duplicates_count += dupes.size - 1
+              unique_transactions << unique_transaction
+            end
+
+            [unique_transactions, duplicates_count]
+          end
+
+          def full_output_file(output_file)
+            File.join(output_folder, output_file)
+          end
+
+          def save_to_csv(transactions, output_file)
+            FileUtils.mkdir_p(output_folder)
+            output_file = full_output_file(output_file)
+            # puts "Output file: #{output_file}"
+
+            # lets sort the transactions by the by the transaction date which is formated as 2023-07-10
+            transactions.sort_by!(&:transaction_date)
+
+            CSV.open(output_file, 'w') do |csv|
+              csv << Appydave::Tools::BankReconciliation::Models::Transaction.csv_headers
+              transactions.each do |transaction|
+                csv << transaction.to_csv_row
+              end
+            end
+
+            log_kv('Transaction Output File', full_output_file(output_file))
+          end
+
+          def csv_to_clipboard(output_file)
+            Clipboard.copy(File.read(full_output_file(output_file)))
+            log_info('Transactions copied to clipboard')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/bank_reconciliation/clean/mapper.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Map transactions to chart of accounts and bank accounts
+        class Mapper
+          include Appydave::Tools::Configuration::Configurable
+
+          def map(transactions)
+            # tp transactions, :coa_match_type, :coa_code, :narration, :debit, :credit
+            transactions.map do |original_transaction|
+              transaction = original_transaction.dup
+
+              transaction = map_chart_of_account(transaction)
+              clean_amount(transaction)
+              map_bank_account(transaction)
+            end
+          end
+
+          private
+
+          def map_chart_of_account(transaction)
+            equality_match(transaction) ||
+              trigram_match(transaction, 0.9, '90%') ||
+              trigram_match(transaction, 0.8, '80%') ||
+              trigram_match(transaction, 0.7, '70%') ||
+              trigram_match(transaction, 0.6, '60%') ||
+              trigram_match(transaction, 0.5, '50%') ||
+              start_with_match(transaction) ||
+              includes(transaction)
+            transaction
+          end
+
+          def map_bank_account(transaction)
+            bank_account = config.bank_reconciliation.get_bank_account(transaction.account_number, transaction.bsb_number)
+
+            if bank_account
+              transaction.account_name = bank_account.name
+              transaction.platform = bank_account.platform
+            else
+              puts "Bank account not found for BSB#: #{transaction.bsb_number} | Account#: #{transaction.account_number}"
+            end
+
+            transaction
+          end
+
+          def equality_match(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              chart_of_account.narration.to_s.delete(' ').downcase == transaction.narration.delete(' ').downcase
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'equality'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def start_with_match(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              transaction.narration.to_s.delete(' ').downcase.start_with?(chart_of_account.narration.to_s.downcase)
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'starts_with'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def includes(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              transaction.narration.to_s.delete(' ').downcase.include?(chart_of_account.narration.delete(' ').to_s.downcase)
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'includes'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def trigram_match(transaction, score_threshold, match_type)
+            scored_transactions = config.bank_reconciliation.chart_of_accounts.map do |coa|
+              {
+                coa: coa,
+                score: compare(coa.narration, transaction.narration)
+              }
+            end
+
+            scored_transactions.sort_by! { |t| t[:score] }.reverse!
+
+            best = scored_transactions.first
+
+            return nil unless best
+            return nil if best[:score] < score_threshold
+
+            coa = best[:coa]
+
+            transaction.coa_match_type = match_type
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def compare(text1, text2)
+            text1_trigs = trigramify(text1)
+            text2_trigs = trigramify(text2)
+
+            all_cnt = (text1_trigs | text2_trigs).size
+            same_cnt = (text1_trigs & text2_trigs).size
+
+            same_cnt.to_f / all_cnt
+            # rescue StandardError => e
+            #   puts "Error comparing text: #{e.message}"
+          end
+
+          def trigramify(text)
+            trigs = []
+            text.chars.each_cons(3) { |v| trigs << v.join }
+            trigs
+          end
+
+          def clean_amount(transaction)
+            transaction.debit = transaction.clean_amount(transaction.debit)
+            transaction.credit = transaction.clean_amount(transaction.credit)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require 'bigdecimal'
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Read transactions from a CSV file
+        class ReadTransactions
+          WISE_HEADER_PREFIX = 'ID,Status,Direction,"Created on","Finished on","Source fee amount",' \
+                               '"Source fee currency","Target fee amount","Target fee currency","Source name",' \
+                               '"Source amount (after fees)","Source currency","Target name",' \
+                               '"Target amount (after fees)","Target currency","Exchange rate",Reference,Batch'
+          PAYPAL_HEADER_PREFIX = '"Date","Time","Time zone","Name","Type","Status","Currency",' \
+                                 '"Amount","Fees","Total","Exchange Rate","Receipt ID","Balance",' \
+                                 '"Transaction ID","Item Title"'
+          # Exact match — short header risks start_with? collision with future formats
+          COMMONWEALTH_SIMPLE_HEADER = 'Date,Amount,Description,Balance'
+          attr_reader :platform
+          attr_reader :transactions
+
+          def initialize(file)
+            @file = file
+          end
+
+          def read
+            csv_lines = File.read(@file).lines
+            csv_lines[0] = csv_lines[0].sub("\xEF\xBB\xBF", '') if csv_lines[0]&.start_with?("\xEF\xBB\xBF")
+
+            @platform = detect_platform(csv_lines)
+
+            case platform
+            when :bankwest
+              read_bankwest(csv_lines)
+            when :bankwest2
+              read_bankwest2(csv_lines)
+            when :commonwealth1
+              read_commonwealth1(csv_lines)
+            when :wise
+              read_wise(csv_lines)
+            when :paypal
+              read_paypal(csv_lines)
+            when :commonwealth_simple
+              read_commonwealth_simple(csv_lines)
+            else
+              raise Appydave::Tools::Error, 'Unknown platform X'
+            end
+          end
+
+          private
+
+          def read_bankwest(csv_lines)
+            @transactions = []
+
+            # Skip the header line and parse each subsequent line
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              transaction = Models::Transaction.new(
+                bsb_number: row['BSB Number'] || '',
+                account_number: row['Account Number'] || '',
+                transaction_date: row['Transaction Date'],
+                narration: row['Narration'],
+                cheque_number: row['Cheque Number'],
+                debit: row['Debit'],
+                credit: row['Credit'],
+                balance: row['Balance'],
+                transaction_type: row['Transaction Type']
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          def read_bankwest2(csv_lines)
+            @transactions = []
+
+            # Skip the header line and parse each subsequent line
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              values = row['BSB / Account Number'].split(' - ')
+
+              transaction = Models::Transaction.new(
+                bsb_number: values.length > 1 ? values.first : '',
+                account_number: values.length > 1 ? values.last : row['BSB / Account Number'],
+                transaction_date: row['Transaction Date'],
+                narration: row['Narration'],
+                cheque_number: row['Cheque Number'],
+                debit: row['Debit'],
+                credit: row['Credit'],
+                balance: row['Balance'],
+                transaction_type: row['Transaction Type']
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          def read_commonwealth1(csv_lines)
+            @transactions = []
+
+            # Skip the header line and parse each subsequent line
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              transaction = Models::Transaction.new(
+                bsb_number: row['bsb_number'],
+                account_number: row['account_number'],
+                transaction_date: row['transaction_date'],
+                narration: row['description'],
+                cheque_number: '',
+                debit: row['amount'].to_f.negative? ? row['amount'] : '',
+                credit: row['amount'].to_f.positive? ? row['amount'] : '',
+                balance: row['balance'],
+                transaction_type: ''
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          # Wise CSV header:
+          # ID,Status,Direction,"Created on","Finished on","Source fee amount","Source fee currency",
+          # "Target fee amount","Target fee currency","Source name","Source amount (after fees)",
+          # "Source currency","Target name","Target amount (after fees)","Target currency",
+          # "Exchange rate",Reference,Batch
+          def read_wise(csv_lines)
+            @transactions = []
+
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              amount   = row['Source amount (after fees)']
+              incoming = row['Direction'].to_s.strip.upcase == 'IN'
+
+              puts "Wise: unexpected Direction '#{row['Direction']}' on row #{row['ID']} — defaulting to debit" \
+                unless %w[IN OUT].include?(row['Direction'].to_s.strip.upcase)
+
+              transaction = Models::Transaction.new(
+                bsb_number: '',
+                account_number: 'WISE',
+                transaction_date: row['Created on'],
+                narration: row['Reference'] || '',
+                cheque_number: "#{row['Source currency']}|#{row['Target currency']}|#{row['Exchange rate']}|#{row['Target name']}",
+                debit: incoming ? '' : amount,
+                credit: incoming ? amount : '',
+                balance: '',
+                transaction_type: ''
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          # PayPal CSV header:
+          # "Date","Time","Time zone","Name","Type","Status","Currency","Amount","Fees","Total",
+          # "Exchange Rate","Receipt ID","Balance","Transaction ID","Item Title"
+          def read_paypal(csv_lines)
+            @transactions = []
+
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              next if row['Date'].to_s.strip.empty?
+
+              name      = row['Name'].to_s.strip
+              item      = row['Item Title'].to_s.strip
+              narration = name.empty? ? item : name
+              currency  = row['Currency'].to_s.strip
+              narration = "#{narration} (#{currency})" if !currency.empty? && currency != 'AUD'
+
+              amount = row['Amount'].to_s.strip
+              debit  = amount.to_f.negative? ? amount : ''
+              credit = amount.to_f.positive? ? amount : ''
+
+              transaction = Models::Transaction.new(
+                bsb_number: '',
+                account_number: 'PAYPAL',
+                transaction_date: row['Date'],
+                narration: narration,
+                cheque_number: '',
+                debit: debit,
+                credit: credit,
+                balance: row['Balance'],
+                transaction_type: row['Type'].to_s
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          # CBA 4-column export (Date,Amount,Description,Balance). The CSV carries
+          # no account identity, so we tag rows with the generic identifier 'CBA-SIMPLE'
+          # and let the downstream mapper resolve it via the local
+          # ~/.config/appydave/bank-reconciliation.json config (same pattern as WISE
+          # and PAYPAL). Real BSB / account number stay out of source.
+          def read_commonwealth_simple(csv_lines)
+            @transactions = []
+
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              next if row['Date'].to_s.strip.empty?
+
+              amount = parse_signed_amount(row['Amount'])
+              debit  = amount.negative? ? amount.to_s('F') : ''
+              credit = amount.positive? ? amount.to_s('F') : ''
+
+              transaction = Models::Transaction.new(
+                bsb_number: '',
+                account_number: 'CBA-SIMPLE',
+                transaction_date: row['Date'],
+                narration: row['Description'].to_s,
+                cheque_number: nil,
+                debit: debit,
+                credit: credit,
+                balance: parse_signed_amount(row['Balance']).to_s('F'),
+                transaction_type: nil
+              )
+              transaction.add_source_file(@file)
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          # For bankwest the first row is the CSV will look like:
+          # BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type
+          def detect_platform(csv_lines)
+            return :bankwest if csv_lines.first.start_with?('BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
+            return :bankwest2 if csv_lines.first.start_with?('Account Name,BSB / Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
+            return :commonwealth1 if csv_lines.first.start_with?('bsb_number,account_number,transaction_date,amount,description,balance') # Standard Account
+            return :commonwealth2 if csv_lines.first.start_with?('transaction_date,narration,debit_credit_amount,debit_credit_currency,balance_amount,balance_currency') # Travel Money
+            return :wise if csv_lines.first.start_with?(WISE_HEADER_PREFIX)
+            return :paypal if csv_lines.first.start_with?(PAYPAL_HEADER_PREFIX)
+            return :commonwealth_simple if csv_lines.first.strip == COMMONWEALTH_SIMPLE_HEADER
+
+            puts "Unknown platform detected. CSV columns are: #{csv_lines.first.strip}"
+            raise Appydave::Tools::Error, 'Unknown platform'
+          end
+
+          def parse_signed_amount(str)
+            return BigDecimal('0') if str.nil? || str.to_s.strip.empty?
+
+            clean = str.to_s.gsub(/[$,\s"]/, '')
+            return BigDecimal('0') if clean.empty?
+
+            case clean[0]
+            when '+' then BigDecimal(clean[1..])
+            when '-' then -BigDecimal(clean[1..])
+            else BigDecimal(clean)
+            end
+          end
+        end
+      end
+    end
+  end
+end