appydave-tools 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2822e09cd6989d2fc8576f8d6b842e6ad40429d9931ae6bbdb557d4bb3a2991c
-  data.tar.gz: 4a9bc9044bcde2bd179a66349b5544430fb9fc8cb1a7b7a656fc5ed3ba8bd405
+  metadata.gz: 14a6bd2963b58f85f247282a6cf8c537495b9a18300b04c3a3794ef5df4ecaa1
+  data.tar.gz: 5ec47924ab4268377a297e1f02f6fa6b4f62207179f21fe8ccdcd69d62cd4f8d
 SHA512:
-  metadata.gz: 4c619668264809fc3be5b5bab73f360dc0480b7dde5b9e232e336edd62a3f416110dbf59ba705174b6b5f8f158819b01817ccacbb123498ad2e17af1330bcaef
-  data.tar.gz: 4f4c8c4ffbce01ac24a7860939b13f403e9e729d75cb3d7bd68e17a9c37580f2605db3b9667ab4f43a5236ff5803639721ce2452580884c2fe23a3414f031fd9
+  metadata.gz: 253546f33096bad9f9daaa5776fa9d98142a4e7cb43607bf2040ab648d49f83712c8f491b8730de4bd602bcd4080a9360cb9bdd082a1b249df4210d66083ebf6
+  data.tar.gz: f1f638a4a299df238694c8b4e42c008e8663a08cb2b83931a953796ec6679fa83038bebe3f749b0351a7edd848220cb906c53ff076abbf953564ff0356347b29

data/.rubocop.yml

@@ -80,6 +80,7 @@ Style/EmptyMethod:
 Metrics/ParameterLists:
   Exclude:
     - "**/spec/**/*"
+    - "**/models/**/*"
 Layout/EmptyLineBetweenDefs:
   Exclude:
     - "**/spec/**/*"
@@ -118,3 +119,14 @@ RSpec/PendingWithoutReason:
 
 Metrics/AbcSize:
   Max: 25
+  Exclude:
+    - "bin/*"
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - "**/models/**/*"
+    - "lib/appydave/tools/bank_reconciliation/clean/mapper.rb"
+Metrics/PerceivedComplexity:
+  Exclude:
+    - "**/models/**/*"
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [0.6.1](https://github.com/klueless-io/appydave-tools/compare/v0.6.0...v0.6.1) (2024-05-26)
+
+
+### Bug Fixes
+
+* improved configuration printing ([89d769d](https://github.com/klueless-io/appydave-tools/commit/89d769d0741fc75b44db90931cf981feea83027f))
+
 # [0.6.0](https://github.com/klueless-io/appydave-tools/compare/v0.5.0...v0.6.0) (2024-05-26)
 
 

data/bin/bank_reconciliation.rb

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+
+require 'pry'
+require 'appydave/tools'
+
+# !/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Process command line arguments for any bank reconciliation operations
+class BankReconciliationCLI
+  def initialize
+    @commands = {
+      'clean' => method(:clean_transactions),
+      'process' => method(:process_transactions),
+      'filter' => method(:filter_transactions)
+    }
+  end
+
+  def run
+    command, *args = ARGV
+    if @commands.key?(command)
+      @commands[command].call(args)
+    else
+      puts "Unknown command: #{command}"
+      print_help
+    end
+  end
+
+  private
+
+  def clean_transactions(args)
+    options = {}
+    OptionParser.new do |opts|
+      opts.banner = 'Usage: bank_reconciliation.rb clean [options]'
+      opts.on('-i', '--include PATTERN', 'GLOB pattern for source transaction files') { |v| options[:include] = v }
+      opts.on('-f', '--transaction FOLDER', 'Transaction CSV folder where original banking CSV files are stored') { |v| options[:transaction_folder] = v }
+      opts.on('-o', '--output FILE', 'Output CSV file name') { |v| options[:output] = v }
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end.parse!(args)
+
+    # Implement cleaning and normalizing transactions
+    puts "Cleaning transactions with options: #{options}"
+  end
+
+  def process_transactions(args)
+    options = {}
+    OptionParser.new do |opts|
+      opts.banner = 'Usage: bank_reconciliation.rb process [options]'
+      opts.on('-i', '--input FILE', 'Input CSV file with transactions') { |v| options[:input] = v }
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end.parse!(args)
+
+    # Implement processing transactions with chart of accounts lookup
+    puts "Processing transactions with options: #{options}"
+  end
+
+  def filter_transactions(args)
+    options = {}
+    OptionParser.new do |opts|
+      opts.banner = 'Usage: bank_reconciliation.rb filter [options]'
+      opts.on('-i', '--input FILE', 'Input CSV file with processed transactions') { |v| options[:input] = v }
+      opts.on('-y', '--year YEAR', 'Filter by financial year') { |v| options[:year] = v }
+      opts.on('-b', '--begin DATE', 'Filter by dates greater than or eqaul to DDMMYY') { |v| options[:year] = v }
+      opts.on('-e', '--end DATE', 'Filter by dates less than or eqaul to DDMMYY') { |v| options[:year] = v }
+      opts.on('-c', '--codes CODES', 'Filter by chart of account codes (comma-separated)') { |v| options[:codes] = v }
+      opts.on('-w', '--wild TEXT', 'Wildcard text match') { |v| options[:text] = v }
+      opts.on('-d', '--display', 'Display filtered transactions in table format') { |v| options[:display] = v }
+      opts.on('-o', '--output FILE', 'Output CSV file name') { |v| options[:output] = v }
+      opts.on_tail('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end.parse!(args)
+
+    # Implement filtering of processed transactions
+    puts "Filtering transactions with options: #{options}"
+  end
+
+  def print_help
+    puts 'Usage: bank_reconciliation.rb [command] [options]'
+    puts 'Commands:'
+    puts '  clean    Clean and normalize transaction files'
+    puts '  process  Process transaction list via chart of accounts lookup'
+    puts '  filter   Filter processed transaction list'
+    puts "Run 'bank_reconciliation.rb [command] --help' for more information on a command."
+  end
+end
+
+BankReconciliationCLI.new.run
+# BankReconciliationCLI.new.run if __FILE__ == $PROGRAM_NAME

data/lib/appydave/tools/bank_reconciliation/_doc.md

@@ -0,0 +1,36 @@
+# Bank reconciliation
+
+[ChatGPT conversation](https://chatgpt.com/c/5d382562-95e5-4243-9b74-c3807d363486)
+
+
+## Code structure
+
+```bash
+├─ lib
+│ ├─ appydave
+│ │ └─ tools
+│ │   ├─ bank_reconciliation
+│ │   │ ├─ clean
+│ │   │ │ ├─ read_transactions.rb
+│ │   │ │ ├─ transaction_cleaner.rb
+│ │   │ ├─ models
+│ │   │ │ ├─ raw_transaction.rb
+│ │   │ │ └─ reconciled_transaction.rb
+│ │   └─ configuration
+│ │     └─ models
+│ │       └─ bank_reconciliation_config.rb
+└─ spec
+  ├─ appydave
+  │ ├─ tools
+  │ │ ├─ bank_reconciliation
+  │ │ │ ├─ clean
+  │ │ │ │ ├─ read_transactions_spec.rb
+  │ │ │ ├─ models
+  │ │ │ │ └─ raw_transaction_spec.rb
+  │ │ └─ configuration
+  │ │   └─ models
+  │ │     └─ bank_reconciliation_config_spec.rb
+  └─ fixtures
+    └─ bank-reconciliation
+      └─ bank-west.csv
+```

data/lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Clean transactions
+        class CleanTransactions
+          include Appydave::Tools::Configuration::Configurable
+          include KLog::Logging
+
+          attr_reader :transaction_folder
+          attr_reader :transactions
+
+          # (config_file)
+          def initialize(transaction_folder: '/Volumes/Expansion/Sync/bank-reconciliation/original-transactions')
+            @transaction_folder = transaction_folder
+          end
+
+          def clean_transactions(input_globs, _output_file)
+            raw_transactions = grab_raw_transactions(input_globs)
+            transactions, duplicates_count = deduplicate(raw_transactions)
+
+            transactions = Mapper.new.map(transactions)
+            # tp transactions
+
+            log.kv 'Deduped consolidated transactions', duplicates_count if duplicates_count.positive?
+
+            # transactions = normalize(transactions)
+            # save_to_csv(transactions, output_file)
+            @transactions = transactions
+          end
+
+          private
+
+          def grab_raw_transactions(input_globs)
+            original_dir = Dir.pwd
+            transactions = []
+
+            begin
+              Dir.chdir(transaction_folder)
+
+              input_globs.each do |glob|
+                Dir.glob(glob).each do |file|
+                  raw_transactions = ReadTransactions.new(file).read
+                  deduped_transactions, duplicates_count = deduplicate(raw_transactions)
+
+                  if duplicates_count.positive?
+                    log.kv 'Duplicates count', duplicates_count
+                    log.kv 'File', file
+                  end
+
+                  transactions += deduped_transactions
+                end
+              end
+            ensure
+              Dir.chdir(original_dir)
+            end
+
+            transactions
+          end
+
+          def deduplicate(transactions)
+            unique_transactions = transactions.uniq do |transaction|
+              [
+                transaction.bsb_number,
+                transaction.account_number,
+                transaction.transaction_date,
+                transaction.narration,
+                transaction.cheque_number,
+                transaction.debit,
+                transaction.credit,
+                transaction.balance,
+                transaction.transaction_type
+              ]
+            end
+
+            duplicates = transactions.size - unique_transactions.size
+
+            [unique_transactions, duplicates]
+          end
+
+          def save_to_csv(transactions, output_file)
+            CSV.open(output_file, 'w') do |csv|
+              csv << ReconciledTransaction.csv_headers
+              transactions.each do |transaction|
+                csv << transaction.to_csv_row
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/bank_reconciliation/clean/mapper.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Map transactions to chart of accounts and bank accounts
+        class Mapper
+          include Appydave::Tools::Configuration::Configurable
+
+          # "bank_accounts": [
+          #   {
+          #     "account_number": "5435 6859 0116 7736",
+          #     "bsb": "",
+          #     "name": "Mastercard",
+          #     "platform": "Bankwest"
+          #   },
+          #   {
+          #     "account_number": "303-092",
+          #     "bsb": "1361644",
+          #     "name": "atcall",
+          #     "platform": "Bankwest"
+          #   },
+
+          def map(transactions)
+            transactions.map do |original_transaction|
+              transaction = original_transaction.dup
+
+              transaction = map_chart_of_account(transaction)
+              map_bank_account(transaction)
+            end
+          end
+
+          private
+
+          def map_chart_of_account(transaction)
+            equality_match(transaction) ||
+              trigram_match(transaction, 0.9, '90%') ||
+              trigram_match(transaction, 0.8, '80%') ||
+              trigram_match(transaction, 0.7, '70%') ||
+              trigram_match(transaction, 0.6, '60%') ||
+              trigram_match(transaction, 0.5, '50%') ||
+              transaction
+          end
+
+          def map_bank_account(transaction)
+            bank_account = config.bank_reconciliation.get_bank_account(transaction.account_number, transaction.bsb_number)
+
+            if bank_account
+              transaction.account_name = bank_account.name
+              transaction.platform = bank_account.platform
+            end
+
+            transaction
+          end
+
+          def equality_match(transaction)
+            coa = config.bank_reconciliation.chart_of_accounts.find do |chart_of_account|
+              chart_of_account.narration.to_s.delete(' ') == transaction.narration.delete(' ')
+            end
+
+            return nil unless coa
+
+            transaction.coa_match_type = 'equality'
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def trigram_match(transaction, score_threshold, match_type)
+            scored_transactions = config.bank_reconciliation.chart_of_accounts.map do |coa|
+              {
+                coa: coa,
+                score: compare(coa.narration, transaction.narration)
+              }
+            end
+
+            scored_transactions.sort_by! { |t| t[:score] }.reverse!
+
+            best = scored_transactions.first
+
+            return nil unless best
+            return nil if best[:score] < score_threshold
+
+            coa = best[:coa]
+
+            transaction.coa_match_type = match_type
+            transaction.coa_code = coa.code
+            transaction
+          end
+
+          def compare(text1, text2)
+            text1_trigs = trigramify(text1)
+            text2_trigs = trigramify(text2)
+
+            all_cnt = (text1_trigs | text2_trigs).size
+            same_cnt = (text1_trigs & text2_trigs).size
+
+            same_cnt.to_f / all_cnt
+          end
+
+          def trigramify(text)
+            trigs = []
+            text.chars.each_cons(3) { |v| trigs << v.join }
+            trigs
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Clean
+        # Read transactions from a CSV file
+        class ReadTransactions
+          attr_reader :platform
+          attr_reader :transactions
+
+          def initialize(file)
+            @file = file
+          end
+
+          def read
+            csv_lines = File.read(@file).lines
+
+            @platform = detect_platform(csv_lines)
+
+            case platform
+            when :bankwest
+              read_bankwest(csv_lines)
+            end
+          end
+
+          private
+
+          def read_bankwest(csv_lines)
+            @transactions = []
+
+            # Skip the header line and parse each subsequent line
+            CSV.parse(csv_lines.join, headers: true).each do |row|
+              transaction = Models::Transaction.new(
+                bsb_number: row['BSB Number'],
+                account_number: row['Account Number'],
+                transaction_date: row['Transaction Date'],
+                narration: row['Narration'],
+                cheque_number: row['Cheque Number'],
+                debit: row['Debit'],
+                credit: row['Credit'],
+                balance: row['Balance'],
+                transaction_type: row['Transaction Type']
+              )
+              @transactions << transaction
+            end
+
+            @transactions
+          end
+
+          # For bankwest the first row is the CSV will look like:
+          # BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type
+          def detect_platform(csv_lines)
+            return :bankwest if csv_lines.first.start_with?('BSB Number,Account Number,Transaction Date,Narration,Cheque Number,Debit,Credit,Balance,Transaction Type')
+
+            raise Appydave::Tools::Error, 'Unknown platform'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/bank_reconciliation/models/transaction.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Appydave
+  module Tools
+    module BankReconciliation
+      module Models
+        # Unified transaction model for raw and reconciled data
+        class Transaction
+          attr_accessor :bsb_number,
+                        :account_number,
+                        :transaction_date,
+                        :narration,
+                        :cheque_number,
+                        :debit,
+                        :credit,
+                        :balance,
+                        :transaction_type,
+                        :platform,
+                        :coa_code,
+                        :coa_match_type,
+                        :account_name
+
+          def initialize(bsb_number: nil,
+                         account_number: nil,
+                         transaction_date: nil,
+                         narration: nil,
+                         cheque_number: nil,
+                         debit: nil,
+                         credit: nil,
+                         balance: nil,
+                         transaction_type: nil,
+                         platform: nil,
+                         coa_code: nil,
+                         coa_match_type: nil,
+                         account_name: nil)
+            @bsb_number = bsb_number&.strip
+            @account_number = account_number&.strip
+            @transaction_date = transaction_date&.strip
+            @transaction_date = Date.strptime(@transaction_date, '%d/%m/%Y')
+            @narration = narration&.gsub(/\s{2,}/, ' ')&.strip
+            @cheque_number = cheque_number&.strip
+            @debit = debit&.strip
+            @credit = credit&.strip
+            @balance = balance&.strip
+            @transaction_type = transaction_type&.strip
+            @platform = platform
+            @coa_code = coa_code
+            @coa_match_type = coa_match_type
+            @account_name = account_name
+          end
+        end
+      end
+    end
+  end
+end

data/lib/appydave/tools/configuration/models/bank_reconciliation_config.rb

@@ -6,6 +6,7 @@ module Appydave
       module Models
         # Bank reconciliation configuration
         class BankReconciliationConfig < ConfigBase
+          # def
           # Retrieve all bank accounts
           def bank_accounts
             data['bank_accounts'].map do |account|
@@ -54,13 +55,13 @@ module Appydave
 
           # Inner class to represent a bank account
           class BankAccount
-            attr_accessor :account_number, :bsb, :name, :bank
+            attr_accessor :account_number, :bsb, :name, :platform
 
             def initialize(data)
               @account_number = data['account_number']
               @bsb = data['bsb']
               @name = data['name']
-              @bank = data['bank']
+              @platform = data['platform']
             end
 
             def to_h
@@ -68,7 +69,7 @@ module Appydave
                 'account_number' => @account_number,
                 'bsb' => @bsb,
                 'name' => @name,
-                'bank' => @bank
+                'platform' => @platform
               }
             end
           end

data/lib/appydave/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Appydave
   module Tools
-    VERSION = '0.6.1'
+    VERSION = '0.7.0'
   end
 end

data/lib/appydave/tools.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'clipboard'
+require 'csv'
 require 'fileutils'
 require 'json'
 require 'open3'
@@ -19,6 +20,10 @@ require 'appydave/tools/configuration/models/settings_config'
 require 'appydave/tools/configuration/models/bank_reconciliation_config'
 require 'appydave/tools/configuration/models/channels_config'
 require 'appydave/tools/name_manager/project_name'
+require 'appydave/tools/bank_reconciliation/clean/clean_transactions'
+require 'appydave/tools/bank_reconciliation/clean/read_transactions'
+require 'appydave/tools/bank_reconciliation/clean/mapper'
+require 'appydave/tools/bank_reconciliation/models/transaction'
 
 Appydave::Tools::Configuration::Config.set_default do |config|
   config.config_path = File.expand_path('~/.config/appydave')

data/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "appydave-tools",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "appydave-tools",
-      "version": "0.6.1",
+      "version": "0.7.0",
       "devDependencies": {
         "@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
         "@semantic-release/changelog": "^6.0.3",

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "appydave-tools",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "AppyDave YouTube Automation Tools",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: appydave-tools
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - David Cruwys
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-05-26 00:00:00.000000000 Z
+date: 2024-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clipboard
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: dotenv
   requirement: !ruby/object:Gem::Requirement
@@ -86,12 +100,18 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/bank_reconciliation.rb
 - bin/configuration.rb
 - bin/console
 - bin/gpt_context.rb
 - bin/setup
 - images.log
 - lib/appydave/tools.rb
+- lib/appydave/tools/bank_reconciliation/_doc.md
+- lib/appydave/tools/bank_reconciliation/clean/clean_transactions.rb
+- lib/appydave/tools/bank_reconciliation/clean/mapper.rb
+- lib/appydave/tools/bank_reconciliation/clean/read_transactions.rb
+- lib/appydave/tools/bank_reconciliation/models/transaction.rb
 - lib/appydave/tools/configuration/_doc.md
 - lib/appydave/tools/configuration/config.rb
 - lib/appydave/tools/configuration/configurable.rb