ynab_convert 1.0.6 → 2.0.0

data/lib/ynab_convert/processors/n26_processor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/transformers'
+require 'ynab_convert/processors/processor'
+
+module Processors
+  # Processor for N26 statements
+  class N26 < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Cleaners::N26.new,
+        Transformers::Formatters::N26.new,
+        Transformers::Enhancers::N26.new
+      ]
+      statement = Documents::Statements::N26.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(format: :amounts,
+                                                        institution_name:
+                                             statement.institution_name)
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/processor.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/validators'
+require 'csv'
+require 'fileutils'
+
+module Processors
+  # A processor instantiates the Documents and Transformers required to turn a
+  # Statement into a YNAB4File
+  class Processor
+    # @param statement [Documents::Statement] The CSV statement to process
+    # @param ynab4_file [Documents::YNAB4Files::YNAB4File] An instance of
+    #   YNAB4File
+    # @param converters [Hash<Symbol, Proc>] A hash of converters to process
+    #   each Statement row. The key is the name of the custom converter.
+    #   The Proc receives the cell's content as a string and returns the
+    #   converted value. See CSV::Converters.
+    # @param transformers [Array<Transformers::Transformer>] The Transformers
+    #   to run in sequense
+    def initialize(statement:, ynab4_file:, transformers:, converters: {})
+      @statement = statement
+      @transformers = transformers
+      @validators = [::Validators::YNAB4Row]
+      @uid = rand(36**8).to_s(36)
+      @ynab4_file = ynab4_file
+      register_converters(converters)
+    end
+
+    def to_ynab!
+      convert
+      rename_temp_file
+    end
+
+    private
+
+    def convert
+      CSV.open(temp_filepath, 'wb',
+               **@ynab4_file.csv_export_options) do |ynab4_csv|
+        CSV.foreach(@statement.filepath, 'rb',
+                    **@statement.csv_import_options) do |statement_row|
+          transformed_row = @transformers.reduce(statement_row) do |row, t|
+            t.run(row)
+          end
+
+          row_valid = @validators.reduce(true) do |is_valid, v|
+            is_valid && v.valid?(transformed_row)
+          end
+
+          if row_valid
+            @ynab4_file.update_dates(transformed_row)
+            ynab4_csv << transformed_row
+          end
+        end
+      end
+    end
+
+    def register_converters(converters)
+      converters.each do |name, block|
+        CSV::Converters[name] = block
+      end
+    end
+
+    def temp_filepath
+      basename = File.basename(@statement.filepath, '.csv')
+      financial_institution = @statement.institution_name
+
+      "#{basename}_#{financial_institution.snake_case}_#{@uid}_ynab4.csv"
+    end
+
+    def rename_temp_file
+      FileUtils.mv(temp_filepath, @ynab4_file.filename)
+    end
+  end
+end

data/lib/ynab_convert/processors/ubs_chequing_processor.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Processors
+  # UBS Switzerland Chequing accounts processor
+  class UBSChequing < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Cleaners::UBSChequing.new,
+        Transformers::Formatters::UBSChequing.new
+      ]
+      statement = Documents::Statements::UBSChequing.new(filepath: filepath)
+      ynab4_file_options = { format: :flows,
+                             institution_name: statement.institution_name }
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(ynab4_file_options)
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/ubs_credit_processor.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Processors
+  # UBS Switzerland Credit Card accounts processor
+  class UBSCredit < Processor
+    def initialize(filepath:)
+      statement = Documents::Statements::UBSCredit.new(filepath: filepath)
+      ynab4_file_options = { institution_name: statement.institution_name }
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(ynab4_file_options)
+      transformers = [Transformers::Cleaners::UBSCredit.new,
+                      Transformers::Formatters::UBSCredit.new]
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors/wise_processor.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Processors
+  # Wise card accounts processor
+  class Wise < Processor
+    def initialize(filepath:)
+      statement = Documents::Statements::Wise.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(
+        institution_name: statement.institution_name, format: :amounts
+      )
+      transformers = [Transformers::Cleaners::Wise.new,
+                      Transformers::Formatters::Wise.new,
+                      Transformers::Enhancers::Wise.new]
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+transformers)
+    end
+  end
+end

data/lib/ynab_convert/processors.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 # Base processor must be loaded first as all others inherit from it
-require 'ynab_convert/processor/base'
+require 'ynab_convert/processors/processor'
 
 # Load all known processors
-Dir[File.join(__dir__, 'processor', '*.rb')].each { |file| require file }
+Dir[File.join(__dir__, 'processors', '*.rb')].sort.each { |file| require file }

data/lib/ynab_convert/transformers/cleaners/cleaner.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # A Cleaner tidies up the Statement fields. For instance, it removes junk
+    # from the transaction descriptions/payee name, combines several
+    # columns into one to build the payee name, etc.
+    class Cleaner
+      # Cleans up a row
+      # @param row [CSV::Row] The row to parse
+      # @return [CSV::Row] The cleaned up row
+      def run(_row)
+        raise NotImplementedError, :run
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/n26_cleaner.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # Cleans N26 Statements
+    class N26 < Cleaner
+      def run(row)
+        # No cleaning required
+        row
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/ubs_chequing_cleaner.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # UBS Switzerland Chequing accounts cleaner
+    class UBSChequing < Cleaner
+      HEADERS = {
+        date: 9,
+        payee_line1: 12,
+        payee_line2: 13,
+        payee_line3: 14,
+        outflow: 18,
+        inflow: 19
+      }.freeze
+
+      def run(row)
+        date = parse_transaction_date(row[HEADERS[:date]])
+        payee = clean_payee_lines(row)
+        outflow = parse_amount(row[HEADERS[:outflow]])
+        inflow = parse_amount(row[HEADERS[:inflow]])
+
+        cleaned_row = row.dup
+        cleaned_row[HEADERS[:date]] = date
+        # Put all the relevant payee data in the first line and empty the other
+        # two lines
+        cleaned_row[HEADERS[:payee_line1]] = payee
+        cleaned_row[HEADERS[:payee_line2]] = ''
+        cleaned_row[HEADERS[:payee_line3]] = ''
+        cleaned_row[HEADERS[:outflow]] = outflow
+        cleaned_row[HEADERS[:inflow]] = inflow
+
+        cleaned_row
+      end
+
+      private
+
+      def parse_transaction_date(date)
+        # Transaction dates are dd.mm.YYYY which Date#parse understands, but
+        # the CSV::Converter[:date] doesn't recognize it as it's not looking
+        # for dot separators.
+        return Date.parse(date) unless date.is_a?(Date) || date.nil?
+
+        date
+      end
+
+      def parse_amount(amount)
+        unless amount.nil? || amount.is_a?(Numeric)
+          return amount.delete("'").to_f
+        end
+
+        amount
+      end
+
+      def clean_payee_lines(row)
+        # Some lines are just bogus without any payee info. These will get
+        # weeded out down the line by the validators.
+        return row if row[HEADERS[:payee_line1]].nil? ||
+                      row[HEADERS[:payee_line1]].empty?
+
+        # Transaction description is spread over 3 columns.
+        # There are two types of entries:
+        # 1. only the first column contains data
+        # 2. all three columns contain data, most of it junk, with only cols 2
+        #   and 3 having meaningful data
+        # Cleaning them up means dropping the first column if there is anything
+        # in the other columns;
+        # Then removing the rest of the junk appended after the worthwhile data;
+        # Finally removing the CARD 00000000-0 0000 at the beginning of debit
+        # card payment entries
+        raw_payee_line = [row[HEADERS[:payee_line2]],
+                          row[HEADERS[:payee_line3]]].join(' ')
+        if row[HEADERS[:payee_line2]].nil?
+          raw_payee_line = row[HEADERS[:payee_line1]]
+        end
+
+        # UBS thought wise to append a bunch of junk information after the
+        # transaction details within the third description field. *Most* of
+        # this junk starts after the meaningful data and starts with ", OF", ",
+        # ON", ", ESR", ", QRR", two digits then five groups of five digits
+        # then ", TN" so we discard it; YNAB4 being unable to automatically
+        # categorize new transactions at the same store/payee if the payee
+        # always looks different (thanks to the variable nature of the appended
+        # junk).
+
+        # rubocop:disable Layout/LineLength
+        junk_desc_regex = /,? (O[FN]|ESR|QRR|\d{2} \d{5} \d{5} \d{5} \d{5} \d{5}, TN).*/
+        # rubocop:enable Layout/LineLength
+
+        # Of course, it wouldn't be complete with more junk information at the
+        # beginning of *some* lines (debit card payments) in the following
+        # form: "CARD 00000000-0 0000"
+        debit_card_junk_regex = /CARD \d{8}-\d \d{4} /
+
+        raw_payee_line.sub(junk_desc_regex, '').sub(debit_card_junk_regex, '')
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/ubs_credit_cleaner.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # UBS Switzerland Credit Card accounts cleaner
+    class UBSCredit < Cleaner
+      HEADERS = {
+        date: 3,
+        payee: 4,
+        outflow: 10,
+        inflow: 11
+      }.freeze
+
+      def run(row)
+        date = parse_transaction_date(row[HEADERS[:date]])
+        outflow = parse_amount(row[HEADERS[:outflow]])
+        inflow = parse_amount(row[HEADERS[:inflow]])
+
+        cleaned_row = row.dup
+        cleaned_row[HEADERS[:date]] = date
+        cleaned_row[HEADERS[:outflow]] = outflow
+        cleaned_row[HEADERS[:inflow]] = inflow
+
+        cleaned_row
+      end
+
+      def parse_transaction_date(date)
+        # Transaction dates are dd.mm.YYYY which Date#parse understands, but
+        # the CSV::Converter[:date] doesn't recognize it as it's not looking
+        # for dot separators.
+        return Date.parse(date) unless date.is_a?(Date) || date.nil?
+
+        date
+      end
+
+      def parse_amount(amount)
+        unless amount.nil? || amount.is_a?(Numeric)
+          return amount.delete("'").to_f
+        end
+
+        amount
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/cleaners/wise_cleaner.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Cleaners
+    # Wise card accounts cleaner
+    class Wise < Cleaner
+      def run(row)
+        date_index = 1
+        payee_index = 13
+        cleaned_row = row.dup
+        date = parse_date(cleaned_row[date_index])
+        payee = clean_payee(cleaned_row[payee_index])
+
+        cleaned_row[date_index] = date
+        cleaned_row[payee_index] = payee
+
+        cleaned_row
+      end
+
+      # turn String "dd-mm-YYYY" into a Date since the
+      # CSV::Transformers[:date] doesn't recognize the dd-mm-YYYY format
+      # @param date [String] the date string to parse (format "dd-mm-YYYY")
+      # @return Date the parsed date string
+      def parse_date(date)
+        Date.parse(date)
+      end
+
+      # The payee data can include some junk (mostly for PayPal/Ebay
+      # transactions)
+      # @param payee [String] the payee string to clean
+      # @return String the payee string without the junk
+      def clean_payee(payee)
+        return payee if payee.nil?
+
+        payee.gsub(/O\*\d{2}-\d{5}-\d{5}\s/, '')
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/enhancer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Enhancers
+    # An Enhancer parses YNAB4 rows (Date, Payee, Memo, Amount or Inflow and
+    # Outflow) and augments the data therein.
+    # A typical case would be converting from one currency to the user's YNAB
+    # base currency when the Statement is in a different currency (see
+    # n26_enhancer.rb for an example)
+    class Enhancer
+      def initialize; end
+
+      # @param _row [CSV::Row] The row to enhance
+      # @return [CSV::Row] The enhanced row
+      def run(_row)
+        raise NotImplementedError, :run
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/n26_enhancer.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/transformers/enhancers/enhancer'
+require 'ynab_convert/api_clients/currency_api'
+
+module Transformers
+  module Enhancers
+    # Converts amounts from EUR to CHF
+    class N26 < Enhancer
+      def initialize
+        @api_client = APIClients::CurrencyAPI.new
+        super()
+      end
+
+      # @param row [CSV::Row] The YNAB4 formatted row to enhance
+      # @return [CSV::Row] A YNAB4 formatted row with amounts converted
+      # @note The currency is hardcoded to CHF for now
+      # @note Takes a YNAB4 formatted CSV::Row, with the transaction's currency
+      # in the Memo field
+      def run(row)
+        amount_index = 3
+        amount = row[amount_index]
+
+        # Transaction currency should be in the memo field
+        base_currency = row[2].to_sym
+
+        date = row[0]
+
+        # TODO: Implement a config file for currencies
+        converted_amount = convert_amount(amount: amount,
+                                          base_currency: base_currency,
+                                          target_currency: :chf,
+                                          date: date)
+
+        enhanced_row = row.dup
+        enhanced_row[amount_index] = converted_amount
+        # Put original amount and currency in Memo
+        memo_line = 'Original amount: '\
+          "#{format('%<amount>.2f', amount: amount)} #{base_currency}"
+        enhanced_row[2] = memo_line
+        enhanced_row
+      end
+
+      private
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency (base currency)
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Float] The conversion rate for amount_currency into CHF
+      def get_rate_for_date(base_currency:, target_currency:, date:)
+        rates = @api_client.historical(base_currency: base_currency, date: date)
+        rates[target_currency]
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param amount [Numeric] The amount in amount_currency to convert
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Numeric] The converted amount
+      def convert_amount(amount:, base_currency:, target_currency:, date:)
+        rate = get_rate_for_date(base_currency: base_currency,
+                                 target_currency: target_currency,
+                                 date: date)
+
+        # format('%<converted>.2f', converted: amount * rate)
+        (amount * rate).round(2)
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/enhancers/wise_enhancer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Enhancers
+    # Wise card accounts enhancer
+    class Wise < Enhancer
+      def initialize
+        @api_client = APIClients::CurrencyAPI.new
+        @indices = { date: 0, amount: 3, memo: 2 }
+
+        super()
+      end
+
+      def run(ynab_row)
+        amount = ynab_row[@indices[:amount]]
+        date = ynab_row[@indices[:date]]
+        metadata = deserialize_metadata(ynab_row[@indices[:memo]])
+        enhanced_row = ynab_row.dup
+
+        # Some transactions, like topups, don't have require conversion
+        unless metadata[:original_amount].nil?
+          converted_amount = convert_amount(
+            amount: amount,
+            base_currency: metadata[:amount_currency],
+            target_currency: :chf,
+            date: date
+          )
+        end
+
+        # Inlude the original transaction amount in the original currency in
+        # the Memo field if conversion was performed
+        unless converted_amount.nil?
+          enhanced_row[@indices[:amount]] = converted_amount
+          # Put original amount and currency in Memo
+          enhanced_row[@indices[:memo]] = 'Original amount: '\
+            "#{metadata[:original_amount]}"
+        end
+
+        # If not, clear the metadata from the Memo field
+        enhanced_row[@indices[:memo]] = '' if converted_amount.nil?
+
+        enhanced_row
+      end
+
+      private
+
+      # @param memo [String] string to deserialize (format is
+      #   `<amount_currency>,<original_amount>`, as formatted by the
+      #   wise_formatter.rb)
+      def deserialize_metadata(memo)
+        # metadata is `<amount_currency>,<original_amount>`
+        split = memo.split(',')
+
+        # amount_currency is a String but the CurrencyAPI needs it to be a
+        # Symbol
+        { amount_currency: split[0].to_sym, original_amount: split[1] }
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency (base currency)
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Float] The conversion rate for amount_currency into CHF
+      def get_rate_for_date(base_currency:, target_currency:, date:)
+        rates = @api_client.historical(base_currency: base_currency, date: date)
+        rates[target_currency]
+      end
+
+      # @param base_currency [Symbol] The ISO symbol of the amount's
+      #   currency
+      # @param target_currency [Symbol] The ISO symbol of the currency to
+      #   convert the amount to (target currency)
+      # @param amount [Numeric] The amount in amount_currency to convert
+      # @param date [Date] The date on which to fetch the rate for conversion
+      # @return [Numeric] The converted amount
+      def convert_amount(amount:, base_currency:, target_currency:, date:)
+        rate = get_rate_for_date(base_currency: base_currency,
+                                 target_currency: target_currency,
+                                 date: date)
+
+        # format('%<converted>.2f', converted: amount * rate)
+        (amount * rate).round(2)
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/formatters/example_formatter.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Formatters
+    # Example Formatter
+    class Example < Formatter
+      def initialize
+        super({ date: [0], payee: [2], outflow: [3], inflow: [4] })
+      end
+    end
+  end
+end

data/lib/ynab_convert/transformers/formatters/formatter.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Transformers
+  module Formatters
+    # Formats Statements rows into YNAB4 rows (Date, Payee, Memo, Amount or
+    # Outflow and Inflow.)
+    class Formatter
+      # @param [Hash] headers_indices the indices at which to find each
+      #   header's name
+      # @option  headers_indices [Array<Numeric>] :date transaction date
+      # @option  headers_indices [Array<Numeric>] :payee transaction
+      #   payee/description
+      # @option  headers_indices [Array<Numeric>] :memo transaction memo or
+      #   currency if currency conversion will be performed
+      # @option  headers_indices [Array<Numeric>] :amount transaction amount (if
+      #   Statement is using the :amounts format)
+      # @option  headers_indices [Array<Numeric>] :outflow transaction outflow
+      #   (if using the :flows format)
+      # @option  headers_indices [Array<Numeric>] :inflow transaction inflow (if
+      #   using the :flows format)
+      def initialize(**headers_indices)
+        default_values = {
+          memo: [] # The Memo field tends to be empty for most institutions
+        }
+
+        @format = :flows
+        unless headers_indices[:amount].nil? || headers_indices[:amount].empty?
+          @format = :amounts
+        end
+        @headers_indices = default_values.merge(headers_indices)
+      end
+
+      # Turns CSV rows into YNAB4 rows (Date, Payee, Memo, Amount or Outflow and
+      # Inflow)
+      # @param row [CSV::Row] The CSV row to parse
+      # @return [Array<String>] The YNAB4 formatted row
+      def run(row)
+        ynab_row = [date(row), payee(row), memo(row)]
+
+        if @format == :amounts
+          ynab_row << amount(row)
+        else
+          ynab_row << outflow(row)
+          ynab_row << inflow(row)
+        end
+
+        ynab_row
+      end
+
+      # Processes columns for each row. Based on the method name that is called,
+      # it will extract the corresponding column (field).
+      # @note In more complex cases, some heuristics are required to format some
+      #   of the columns. In that case, any of the aliased #field methods
+      #   (#date, #payee, #memo, #amount, #outflow, #inflow) can be
+      #   overridden in the child.
+      # @param row [CSV::Row] The row to process
+      # @return [String] The corresponding field(s)
+      def field(row)
+        # Figure out the aliased name the method was called with, to derive
+        # which field to return from the row.
+        requested_field = __callee__.to_sym
+        assembled_field = @headers_indices[requested_field].reduce([]) do
+          |fields_data, i|
+          fields_data << row[i]
+        end
+
+        # Avoid turning Dates and Numerics back to strings
+        # If the assembled_field isn't a composite from several Statement
+        # fields, there is no need to join(' ') and turn it into a String
+        formatted_field = assembled_field[0]
+        if assembled_field.length > 1
+          formatted_field = assembled_field.join(' ')
+        end
+
+        # Avoid "nil" values in the output
+        return '' if formatted_field.nil?
+
+        formatted_field
+      end
+
+      # Create alias names for the field method. This allows the function to
+      # figure out which field to extract from its method name.
+      alias date field
+      alias payee field
+      alias memo field
+      alias amount field
+      alias outflow field
+      alias inflow field
+    end
+  end
+end