ynab_convert 1.0.8 → 2.0.3

data/lib/ynab_convert/documents/ynab4_files/ynab4_file.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Documents
+  module YNAB4Files
+    # Represents the YNAB4 formatted CSV data for importing into YNAB4
+    class YNAB4File
+      attr_reader :csv_export_options
+
+      def initialize(institution_name:, format: :flows)
+        @format = format
+        @institution_name = institution_name
+        @csv_export_options = {
+          converters: %i[numeric date],
+          force_quotes: true,
+          write_headers: true,
+          headers: headers
+        }
+      end
+
+      def update_dates(row)
+        date_index = 0
+        transaction_date = row[date_index]
+        unless transaction_date.is_a?(Date)
+          transaction_date = Date.parse(transaction_date)
+        end
+
+        update_start_date(transaction_date)
+        update_end_date(transaction_date)
+      end
+
+      def filename
+        from_date = @start_date.strftime('%Y%m%d')
+        to_date = @end_date.strftime('%Y%m%d')
+
+        "#{@institution_name.snake_case}_#{from_date}-#{to_date}_ynab4.csv"
+      end
+
+      private
+
+      def update_start_date(date)
+        @start_date = date if @start_date.nil? || date < @start_date
+      end
+
+      def update_end_date(date)
+        @end_date = date if @end_date.nil? || date > @end_date
+      end
+
+      def headers
+        base_headers = %w[Date Payee Memo]
+        extra_headers = %w[Outflow Inflow]
+
+        extra_headers = %w[Amount] if @format == :amounts
+
+        base_headers.concat(extra_headers)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Groups Statements and YNAB4File
+module Documents
+  documents = %w[statement ynab4_file]
+
+  # Load all known Documents
+  documents.each do |d|
+    # Require the base classes first so that its children can find the parent
+    # class since files are otherwise loaded in alphabetical order
+    require File.join(__dir__, 'documents', "#{d}s", "#{d}.rb")
+
+    Dir[File.join(__dir__, 'documents', "#{d}s", '*.rb')].sort.each do |file|
+      require file
+    end
+  end
+end

data/lib/ynab_convert/logger.rb

@@ -6,7 +6,7 @@ require 'logger'
 module YnabLogger
   def logger
     @logger unless @logger.nil?
-    @logger ||= Logger.new(STDERR)
+    @logger ||= Logger.new($stderr)
     @logger.level = Logger::FATAL
     @logger.level = Logger::DEBUG if ENV['YNAB_CONVERT_DEBUG'] == 'true'
     @logger

data/lib/ynab_convert/processors/example_processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/transformers'
+require 'ynab_convert/processors/processor'
+
+module Processors
+  # Example Processor
+  class Example < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Formatters::Example.new
+      ]
+      statement = Documents::Statements::Example.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(
+        format: :flows, institution_name: statement.institution_name
+      )
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/n26_processor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/transformers'
+require 'ynab_convert/processors/processor'
+
+module Processors
+  # Processor for N26 statements
+  class N26 < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Cleaners::N26.new,
+        Transformers::Formatters::N26.new,
+        Transformers::Enhancers::N26.new
+      ]
+      statement = Documents::Statements::N26.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(format: :amounts,
+                                                        institution_name:
+                                             statement.institution_name)
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/processor.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/validators'
+require 'csv'
+require 'fileutils'
+
+module Processors
+  # A processor instantiates the Documents and Transformers required to turn a
+  # Statement into a YNAB4File
+  class Processor
+    # @param statement [Documents::Statement] The CSV statement to process
+    # @param ynab4_file [Documents::YNAB4Files::YNAB4File] An instance of
+    #   YNAB4File
+    # @param converters [Hash<Symbol, Proc>] A hash of converters to process
+    #   each Statement row. The key is the name of the custom converter.
+    #   The Proc receives the cell's content as a string and returns the
+    #   converted value. See CSV::Converters.
+    # @param transformers [Array<Transformers::Transformer>] The Transformers
+    #   to run in sequense
+    def initialize(statement:, ynab4_file:, transformers:, converters: {})
+      @statement = statement
+      @transformers = transformers
+      @validators = [::Validators::YNAB4Row]
+      @uid = rand(36**8).to_s(36)
+      @ynab4_file = ynab4_file
+      register_converters(converters)
+    end
+
+    def to_ynab!
+      convert
+      rename_temp_file
+    end
+
+    private
+
+    def convert
+      CSV.open(temp_filepath, 'wb',
+               **@ynab4_file.csv_export_options) do |ynab4_csv|
+        CSV.foreach(@statement.filepath, 'rb',
+                    **@statement.csv_import_options) do |statement_row|
+          transformed_row = @transformers.reduce(statement_row) do |row, t|
+            t.run(row)
+          end
+
+          row_valid = @validators.reduce(true) do |is_valid, v|
+            is_valid && v.valid?(transformed_row)
+          end
+
+          if row_valid
+            @ynab4_file.update_dates(transformed_row)
+            ynab4_csv << transformed_row
+          end
+        end
+      end
+    end
+
+    def register_converters(converters)
+      converters.each do |name, block|
+        CSV::Converters[name] = block
+      end
+    end
+
+    def temp_filepath
+      basename = File.basename(@statement.filepath, '.csv')
+      financial_institution = @statement.institution_name
+
+      "#{basename}_#{financial_institution.snake_case}_#{@uid}_ynab4.csv"
+    end
+
+    def rename_temp_file
+      FileUtils.mv(temp_filepath, @ynab4_file.filename)
+    end
+  end
+end

data/lib/ynab_convert/processors/ubs_chequing_processor.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Processors
+  # UBS Switzerland Chequing accounts processor
+  class UBSChequing < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Cleaners::UBSChequing.new,
+        Transformers::Formatters::UBSChequing.new
+      ]
+      statement = Documents::Statements::UBSChequing.new(filepath: filepath)
+      ynab4_file_options = { format: :flows,
+                             institution_name: statement.institution_name }
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(ynab4_file_options)
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/ubs_credit_processor.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Processors
+  # UBS Switzerland Credit Card accounts processor
+  class UBSCredit < Processor
+    def initialize(filepath:)
+      statement = Documents::Statements::UBSCredit.new(filepath: filepath)
+      ynab4_file_options = { institution_name: statement.institution_name }
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(ynab4_file_options)
+      transformers = [Transformers::Cleaners::UBSCredit.new,
+                      Transformers::Formatters::UBSCredit.new]
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/wise_processor.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Processors
+  # Wise card accounts processor
+  class Wise < Processor
+    def initialize(filepath:)
+      statement = Documents::Statements::Wise.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(
+        institution_name: statement.institution_name, format: :amounts
+      )
+      transformers = [Transformers::Cleaners::Wise.new,
+                      Transformers::Formatters::Wise.new,
+                      Transformers::Enhancers::Wise.new]
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Base processor must be loaded first as all others inherit from it
-require 'ynab_convert/processor/base'
+require 'ynab_convert/processors/processor'
 
 # Load all known processors
-Dir[File.join(__dir__, 'processor', '*.rb')].each { |file| require file }
+Dir[File.join(__dir__, 'processors', '*.rb')].sort.each { |file| require file }

data/lib/ynab_convert/transformers/cleaners/cleaner.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # A Cleaner tidies up the Statement fields. For instance, it removes junk
+    # from the transaction descriptions/payee name, combines several
+    # columns into one to build the payee name, etc.
+    class Cleaner
+      # Cleans up a row
+      # @param row [CSV::Row] The row to parse
+      # @return [CSV::Row] The cleaned up row
+      def run(_row)
+        raise NotImplementedError, :run
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/n26_cleaner.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # Cleans N26 Statements
+    class N26 < Cleaner
+      def run(row)
+        # No cleaning required
+        row
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/ubs_chequing_cleaner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # UBS Switzerland Chequing accounts cleaner
+    class UBSChequing < Cleaner
+      HEADERS = {
+        date: 9,
+        payee_line1: 12,
+        payee_line2: 13,
+        payee_line3: 14,
+        outflow: 18,
+        inflow: 19
+      }.freeze
+
+      def run(row)
+        date = parse_transaction_date(row[HEADERS[:date]])
+        payee = clean_payee_lines(row)
+        outflow = parse_amount(row[HEADERS[:outflow]])
+        inflow = parse_amount(row[HEADERS[:inflow]])
+
+        cleaned_row = row.dup
+        cleaned_row[HEADERS[:date]] = date
+        # Put all the relevant payee data in the first line and empty the other
+        # two lines
+        cleaned_row[HEADERS[:payee_line1]] = payee
+        cleaned_row[HEADERS[:payee_line2]] = ''
+        cleaned_row[HEADERS[:payee_line3]] = ''
+        cleaned_row[HEADERS[:outflow]] = outflow
+        cleaned_row[HEADERS[:inflow]] = inflow
+
+        cleaned_row
+      end
+
+      private
+
+      def parse_transaction_date(date)
+        # Transaction dates are dd.mm.YYYY which Date#parse understands, but
+        # the CSV::Converter[:date] doesn't recognize it as it's not looking
+        # for dot separators.
+        return Date.parse(date) unless date.is_a?(Date) || date.nil?
+
+        date
+      end
+
+      def parse_amount(amount)
+        unless amount.nil? || amount.is_a?(Numeric)
+          return amount.delete("'").to_f
+        end
+
+        amount
+      end
+
+      def clean_payee_lines(row)
+        # Some lines are just bogus without any payee info. These will get
+        # weeded out down the line by the validators.
+        return row if row[HEADERS[:payee_line1]].nil? ||
+                      row[HEADERS[:payee_line1]].empty?
+
+        # Transaction description is spread over 3 columns.
+        # There are two types of entries:
+        # 1. only the first column contains data
+        # 2. all three columns contain data, most of it junk, with only cols 2
+        #   and 3 having meaningful data
+        # Cleaning them up means dropping the first column if there is anything
+        # in the other columns;
+        # Then removing the rest of the junk appended after the worthwhile data;
+        # Finally removing the CARD 00000000-0 0000 at the beginning of debit
+        # card payment entries
+        raw_payee_line = [row[HEADERS[:payee_line2]],
+                          row[HEADERS[:payee_line3]]].join(' ')
+        if row[HEADERS[:payee_line2]].nil?
+          raw_payee_line = row[HEADERS[:payee_line1]]
+        end
+
+        # UBS thought wise to append a bunch of junk information after the
+        # transaction details within the third description field. *Most* of
+        # this junk starts after the meaningful data and starts with ", OF", ",
+        # ON", ", ESR", ", QRR", two digits then five groups of five digits
+        # then ", TN" so we discard it; YNAB4 being unable to automatically
+        # categorize new transactions at the same store/payee if the payee
+        # always looks different (thanks to the variable nature of the appended
+        # junk).
+
+        # rubocop:disable Layout/LineLength
+        junk_desc_regex = /,? (O[FN]|ESR|QRR|\d{2} \d{5} \d{5} \d{5} \d{5} \d{5}, TN).*/
+        # rubocop:enable Layout/LineLength
+
+        # Of course, it wouldn't be complete with more junk information at the
+        # beginning of *some* lines (debit card payments) in the following
+        # form: "CARD 00000000-0 0000"
+        debit_card_junk_regex = /CARD \d{8}-\d \d{4} /
+
+        raw_payee_line.sub(junk_desc_regex, '').sub(debit_card_junk_regex, '')
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/ubs_credit_cleaner.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # UBS Switzerland Credit Card accounts cleaner
+    class UBSCredit < Cleaner
+      HEADERS = {
+        date: 3,
+        payee: 4,
+        outflow: 10,
+        inflow: 11
+      }.freeze
+
+      def run(row)
+        date = parse_transaction_date(row[HEADERS[:date]])
+        outflow = parse_amount(row[HEADERS[:outflow]])
+        inflow = parse_amount(row[HEADERS[:inflow]])
+
+        cleaned_row = row.dup
+        cleaned_row[HEADERS[:date]] = date
+        cleaned_row[HEADERS[:outflow]] = outflow
+        cleaned_row[HEADERS[:inflow]] = inflow
+
+        cleaned_row
+      end
+
+      def parse_transaction_date(date)
+        # Transaction dates are dd.mm.YYYY which Date#parse understands, but
+        # the CSV::Converter[:date] doesn't recognize it as it's not looking
+        # for dot separators.
+        return Date.parse(date) unless date.is_a?(Date) || date.nil?
+
+        date
+      end
+
+      def parse_amount(amount)
+        unless amount.nil? || amount.is_a?(Numeric)
+          return amount.delete("'").to_f
+        end
+
+        amount
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/wise_cleaner.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # Wise card accounts cleaner
+    class Wise < Cleaner
+      def run(row)
+        date_index = 1
+        payee_index = 13
+        cleaned_row = row.dup
+        date = parse_date(cleaned_row[date_index])
+        payee = clean_payee(cleaned_row[payee_index])
+
+        cleaned_row[date_index] = date
+        cleaned_row[payee_index] = payee
+
+        cleaned_row
+      end
+
+      # turn String "dd-mm-YYYY" into a Date since the
+      # CSV::Transformers[:date] doesn't recognize the dd-mm-YYYY format
+      # @param date [String] the date string to parse (format "dd-mm-YYYY")
+      # @return Date the parsed date string
+      def parse_date(date)
+        Date.parse(date)
+      end
+
+      # The payee data can include some junk (mostly for PayPal/Ebay
+      # transactions)
+      # @param payee [String] the payee string to clean
+      # @return String the payee string without the junk
+      def clean_payee(payee)
+        return payee if payee.nil?
+
+        payee.gsub(/O\*\d{2}-\d{5}-\d{5}\s/, '')
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/enhancer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Enhancers
+    # An Enhancer parses YNAB4 rows (Date, Payee, Memo, Amount or Inflow and
+    # Outflow) and augments the data therein.
+    # A typical case would be converting from one currency to the user's YNAB
+    # base currency when the Statement is in a different currency (see
+    # n26_enhancer.rb for an example)
+    class Enhancer
+      def initialize; end
+
+      # @param _row [CSV::Row] The row to enhance
+      # @return [CSV::Row] The enhanced row
+      def run(_row)
+        raise NotImplementedError, :run
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/n26_enhancer.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/transformers/enhancers/enhancer'
+require 'ynab_convert/api_clients/currency_api'
+
+module Transformers
+  module Enhancers
+    # Converts amounts from EUR to CHF
+    class N26 < Enhancer
+      def initialize
+        @api_client = APIClients::CurrencyAPI.new
+        super()
+      end
+
+      # @param row [CSV::Row] The YNAB4 formatted row to enhance
+      # @return [CSV::Row] A YNAB4 formatted row with amounts converted
+      # @note The currency is hardcoded to CHF for now
+      # @note Takes a YNAB4 formatted CSV::Row, with the transaction's currency
+      # in the Memo field
+      def run(row)
+        amount_index = 3
+        amount = row[amount_index]
+
+        # Transaction currency should be in the memo field
+        base_currency = row[2].to_sym
+
+        date = row[0]
+
+        # TODO: Implement a config file for currencies
+        converted_amount = convert_amount(amount: amount,
+                                          base_currency: base_currency,
+                                          target_currency: :chf,
+                                          date: date)
+
+        enhanced_row = row.dup
+        enhanced_row[amount_index] = converted_amount
+        # Put original amount and currency in Memo
+        memo_line = 'Original amount: '\
+          "#{format('%<amount>.2f', amount: amount)} #{base_currency}"
+        enhanced_row[2] = memo_line
+        enhanced_row
+      end
+
+      private
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency (base currency)
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Float] The conversion rate for amount_currency into CHF
+      def get_rate_for_date(base_currency:, target_currency:, date:)
+        rates = @api_client.historical(base_currency: base_currency, date: date)
+        rates[target_currency]
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param amount [Numeric] The amount in amount_currency to convert
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Numeric] The converted amount
+      def convert_amount(amount:, base_currency:, target_currency:, date:)
+        rate = get_rate_for_date(base_currency: base_currency,
+                                 target_currency: target_currency,
+                                 date: date)
+
+        # format('%<converted>.2f', converted: amount * rate)
+        (amount * rate).round(2)
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/wise_enhancer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Enhancers
+    # Wise card accounts enhancer
+    class Wise < Enhancer
+      def initialize
+        @api_client = APIClients::CurrencyAPI.new
+        @indices = { date: 0, amount: 3, memo: 2 }
+
+        super()
+      end
+
+      def run(ynab_row)
+        amount = ynab_row[@indices[:amount]]
+        date = ynab_row[@indices[:date]]
+        metadata = deserialize_metadata(ynab_row[@indices[:memo]])
+        enhanced_row = ynab_row.dup
+
+        # Some transactions, like topups, don't have require conversion
+        unless metadata[:original_amount].nil?
+          converted_amount = convert_amount(
+            amount: amount,
+            base_currency: metadata[:amount_currency],
+            target_currency: :chf,
+            date: date
+          )
+        end
+
+        # Inlude the original transaction amount in the original currency in
+        # the Memo field if conversion was performed
+        unless converted_amount.nil?
+          enhanced_row[@indices[:amount]] = converted_amount
+          # Put original amount and currency in Memo
+          enhanced_row[@indices[:memo]] = 'Original amount: '\
+            "#{metadata[:original_amount]}"
+        end
+
+        # If not, clear the metadata from the Memo field
+        enhanced_row[@indices[:memo]] = '' if converted_amount.nil?
+
+        enhanced_row
+      end
+
+      private
+
+      # @param memo [String] string to deserialize (format is
+      #   `<amount_currency>,<original_amount>`, as formatted by the
+      #   wise_formatter.rb)
+      def deserialize_metadata(memo)
+        # metadata is `<amount_currency>,<original_amount>`
+        split = memo.split(',')
+
+        # amount_currency is a String but the CurrencyAPI needs it to be a
+        # Symbol
+        { amount_currency: split[0].to_sym, original_amount: split[1] }
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency (base currency)
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Float] The conversion rate for amount_currency into CHF
+      def get_rate_for_date(base_currency:, target_currency:, date:)
+        rates = @api_client.historical(base_currency: base_currency, date: date)
+        rates[target_currency]
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param amount [Numeric] The amount in amount_currency to convert
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Numeric] The converted amount
+      def convert_amount(amount:, base_currency:, target_currency:, date:)
+        rate = get_rate_for_date(base_currency: base_currency,
+                                 target_currency: target_currency,
+                                 date: date)
+
+        # format('%<converted>.2f', converted: amount * rate)
+        (amount * rate).round(2)
+      end
+    end
+  end
+end