ynab_convert 0.1.0.pre

data/lib/ynab_convert/processor/base.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'core_extensions/string'
+require 'csv'
+require 'ynab_convert/logger'
+
+module Processor
+  # Base class for a Processor, all processors must inherit from it
+  # rubocop:disable Metrics/ClassLength
+  class Base
+    include YnabLogger
+    include CoreExtensions::String::Inflections
+
+    attr_reader :loader_options
+
+    # @option opts [String] :file Path to the CSV file to process
+    def initialize(opts)
+      logger.debug "Initializing processor with options: `#{opts.to_h}'"
+      raise ::Errno::ENOENT unless File.exist? opts[:file]
+
+      @file = opts[:file]
+      @headers = { transaction_date: nil, payee: nil, debit: nil, credit: nil }
+    end
+
+    def to_ynab!
+      begin
+        convert!
+        rename_file
+      rescue YnabConvert::Error
+        invalid_csv_file
+      end
+    ensure
+      logger.debug "Deleting temp file `#{temp_filename}'"
+      delete_temp_csv
+    end
+
+    protected
+
+    attr_accessor :statement_from, :statement_to, :headers
+
+    def inflow_or_outflow_missing?(row)
+      inflow_index = 3
+      outflow_index = 4
+      # If there is neither inflow and outflow values, or their value is 0,
+      # then the row is not valid to YNAB4
+      (row[inflow_index].nil? || row[inflow_index].empty? ||
+       row[inflow_index] == '0.00') &&
+        (row[outflow_index].nil? || row[outflow_index].empty? ||
+         row[outflow_index] == '0.00')
+    end
+
+    def skip_row(row)
+      logger.debug "Found empty row, skipping it: #{row.to_h}"
+      throw :skip_row
+    end
+
+    def delete_temp_csv
+      FileUtils.remove_file temp_filename, force: true
+    end
+
+    def transaction_date_missing?(ynab_row)
+      ynab_row[0].nil? || [0].empty?
+    end
+
+    def extract_transaction_date(ynab_row)
+      transaction_date_index = 0
+      ynab_row[transaction_date_index]
+    end
+
+    def record_statement_interval_dates(ynab_row)
+      transaction_date_index = 0
+      date = Date.parse(ynab_row[transaction_date_index])
+
+      if date_is_further_away?(date)
+        logger.debug "Replacing statement_from `#{statement_from.inspect}' "\
+          "with `#{date}'"
+        self.statement_from = date
+      end
+      # rubocop:disable Style/GuardClause
+      if date_is_more_recent?(date)
+        logger.debug "Replacing statement_to `#{statement_to.inspect}' with "\
+          "`#{date}'"
+        self.statement_to = date
+      end
+      # rubocop:enable Style/GuardClause
+    end
+
+    def date_is_more_recent?(date)
+      statement_to.nil? || statement_to < date
+    end
+
+    def date_is_further_away?(date)
+      statement_from.nil? || statement_from > date
+    end
+
+    def convert!
+      logger.debug "Will write to `#{temp_filename}'"
+
+      CSV.open(temp_filename, 'wb', output_options) do |converted|
+        CSV.foreach(@file, 'rb', loader_options) do |row|
+          logger.debug "Parsing row: `#{row.to_h}'"
+          # Some rows don't contain valid or useful data
+          catch :skip_row do
+            extract_header_names(row)
+            ynab_row = transformers(row)
+            if inflow_or_outflow_missing?(ynab_row) ||
+               transaction_date_missing?(ynab_row)
+              logger.debug 'Empty row, skipping it'
+              skip_row(row)
+            end
+            converted << ynab_row
+            record_statement_interval_dates(ynab_row)
+          end
+
+          logger.debug 'Done converting'
+        end
+      end
+    end
+
+    def rename_file
+      File.rename(temp_filename, output_filename)
+      logger.debug "Renamed temp file `#{temp_filename}' to "\
+        "`#{output_filename}'"
+    end
+
+    def invalid_csv_file
+      raise YnabConvert::Error, "Unable to parse file `#{@file}'. Is it a "\
+        "valid CSV file from #{@institution_name}?"
+    end
+
+    def file_uid
+      @file_uid ||= rand(36**8).to_s(36)
+    end
+
+    def temp_filename
+      "#{File.basename(@file, '.csv')}_#{@institution_name.snake_case}_"\
+        "#{file_uid}_ynab4.csv"
+    end
+
+    def output_filename
+      # If the file contained no parsable CSV data, from and to dates will be
+      # nil.
+      # This is to avoid a NoMethodError on NilClass.
+      raise YnabConvert::Error if statement_from.nil? || statement_to.nil?
+
+      from = statement_from.strftime('%Y%m%d')
+      to = statement_to.strftime('%Y%m%d')
+
+      "#{File.basename(@file, '.csv')}_#{@institution_name.snake_case}_"\
+        "#{from}-#{to}_ynab4.csv"
+    end
+
+    def ynab_headers
+      %w[Date Payee Memo Outflow Inflow]
+    end
+
+    def output_options
+      {
+        converters: %i[numeric date],
+        force_quotes: true,
+        write_headers: true,
+        headers: ynab_headers
+      }
+    end
+
+    def transformers
+      raise NotImplementedError, :transformers
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/ynab_convert/processor/example.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Processor
+  # An example of how to implement a custom processor
+  # Processes CSV files with this format:
+  # <<~ROWS
+  # "Date","Payee","Memo","Outflow","Inflow"
+  # "23/12/2019","coaxial","","1000000.00",""
+  # "30/12/2019","Santa","","50000.00",""
+  # "02/02/2020","Someone Else","","45.00",""
+  # ROWS
+  # The file name for the processor should be the institution name in
+  # camel case. It's ok to skip "Bank" or "Credit Union" when naming the file
+  # if it's redundant. For instance, this parser is for "Example Bank" but it's
+  # named "example.rb", its corresponding spec is
+  # "spec/example_processor_spec.rb" and its fixture would be
+  # "spec/fixtures/example.csv"
+  class Example < Processor::Base
+    # @option options [String] :file Path to the CSV file to process
+    def initialize(options)
+      # Custom converters can be added so that the CSV data is parsed when
+      # loading the original file
+      register_custom_converters
+
+      # These are the options for the CSV module (see
+      # https://ruby-doc.org/stdlib-2.6/libdoc/csv/rdoc/CSV.html#method-c-new)
+      # They should match the format for the CSV file that the financial
+      # institution generates.
+      @loader_options = {
+        col_sep: ';',
+        # Use your converters, if any
+        converters: %i[transaction_date my_converter],
+        headers: true
+      }
+
+      # This is the financial institution's full name as it calls itself. This
+      # usually matches the institution's letterhead and/or commercial name.
+      # It can happen that the same institution needs different parsers because
+      # its credit card CSV files are in one format, and its chequing accounts
+      # in another. In that case, more details can be added in parens.
+      # For instance:
+      # 'Example Bank (credit cards)' and 'Example Bank (chequing)'
+      @institution_name = 'Example Bank'
+
+      # This is mandatory.
+      super(options)
+    end
+
+    private
+
+    def register_custom_converters
+      CSV::Converters[:transaction_date] = lambda { |s|
+        # Only match strings that have two digits, a dot, two digits, a dot,
+        # two digits, i.e. the dates in this institution's CSV files.
+        date_regex = /^\d{2}\.\d{2}\.\d{2}$/
+
+        if !s.nil? && s.match(date_regex)
+          parsed_date = Date.strptime(s, '%d.%m.%y')
+          logger.debug "Converted `#{s.inspect}' into date "\
+            "`#{parsed_date}'"
+          return parsed_date
+        end
+
+        s
+      }
+
+      CSV::Converters[:my_converter] = lambda { |s|
+        # A contrived example, just to illustrate multiple converters
+        if s.respond_to?(:downcase)
+          converted_s = s.downcase
+          logger.debug "Converted `#{s.inspect}' into downcased string "\
+            "`#{converted_s}'"
+          return converted_s
+        end
+
+        s
+      }
+    end
+
+    protected
+
+    # Converts the institution's CSV rows into YNAB4 rows.
+    # The YNAB4 columns are:
+    # "Date', "Payee", "Memo", "Outflow", "Inflow"
+    # which match Example Bank's "transaction_date" (after parsing),
+    # "beneficiary", nothing, "debit", and "credit" respectively.
+    # Note that Example Bank doesn't include any relevant column for YNAB4's
+    # "Memo" column so it's skipped and gets '' as its value.
+    def transformers(row)
+      # Convert the original transaction_date to DD/MM/YYYY as YNAB4 expects
+      # it.
+      unless row[headers[:transaction_date]].nil?
+        transaction_date = row[headers[:transaction_date]].strftime('%d/%m/%Y')
+      end
+      payee = row[headers[:payee]]
+      debit = row[headers[:debit]]
+      credit = row[headers[:credit]]
+
+      # CSV files can have funny data in them, including invalid or empty rows.
+      # These rows can be skipped from the converted YNAB4 file by calling
+      # skip_row when detected. In this particular case, if there is no
+      # transaction date, it means the row is empty or invalid and we discard
+      # it.
+      skip_row(row) if transaction_date.nil?
+      converted_row = [transaction_date, payee, nil, debit, credit]
+      logger.debug "Converted row: #{converted_row}"
+      converted_row
+    end
+
+    private
+
+    # Institutions love translating the column names, apparently. Rather than
+    # hardcoding the column name as a string, use the headers array at the
+    # right index.
+    # These lookups aren't particularly expensive but they're done on each row
+    # so why not memoize them with ||=
+    def extract_header_names(row)
+      headers[:transaction_date] ||= row.headers[0]
+      headers[:payee] ||= row.headers[2]
+      headers[:debit] ||= row.headers[3]
+      headers[:credit] ||= row.headers[4]
+    end
+  end
+end

data/lib/ynab_convert/processor/revolut.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'i18n'
+
+module Processor
+  # Processes CSV files from Revolut
+  class Revolut < Processor::Base
+    # @option options [String] :file Path to the CSV file to process
+    def initialize(options)
+      register_custom_converters
+      @loader_options = {
+        col_sep: ';',
+        converters: %i[amounts transaction_dates],
+        quote_char: nil,
+        encoding: Encoding::UTF_8,
+        headers: true
+      }
+      @institution_name = 'Revolut'
+
+      super(options)
+    end
+
+    protected
+
+    def transformers(row)
+      date = extract_transaction_date(row).strftime('%d/%m/%Y')
+      payee = row[headers[:payee]]
+      unless row[headers[:debit]].nil?
+        debit = format('%<amount>.2f', amount: row[headers[:debit]])
+      end
+      unless row[headers[:credit]].nil?
+        credit = format('%<amount>.2f', amount: row[headers[:credit]])
+      end
+
+      ynab_row = [
+        date,
+        payee,
+        nil,
+        debit,
+        credit
+      ]
+
+      logger.debug "Converted row: #{ynab_row}"
+      ynab_row
+    end
+
+    private
+
+    def extract_header_names(row)
+      @headers[:transaction_date] ||= row.headers[0]
+      @headers[:payee] ||= row.headers[1]
+      @headers[:debit] ||= row.headers[2]
+      @headers[:credit] ||= row.headers[3]
+    end
+
+    def register_custom_converters
+      CSV::Converters[:amounts] = lambda { |s|
+        # Yes, amount come with a non breaking trailing space... Which is
+        # matched with \p{Zs} (c.f.
+        # https://ruby-doc.org/core-2.6/Regexp.html#class-Regexp-label-Character+Properties)
+        # Also, thousands separators can be non breaking spaces.
+        amount_regex = /^[\d'\.,\p{Zs}]+[\.,]\d{2}\p{Zs}$/
+        # narrow_nbsp = "\0xE2\0x80\0xAF"
+        narrow_nbsp = "\u{202F}"
+        readability_separators = "',. #{narrow_nbsp}"
+
+        if !s.nil? && s.match(amount_regex)
+          # This is a bit hacky because we don't have the luxury of Rails' i18n
+          # helpers. If we have an amount, strip all the separators in it, turn
+          # it to a float, and divide by 100 to get the right amount back
+          amount = s.delete(readability_separators).to_f / 100
+          logger.debug "Converted `#{s}' into amount `#{amount}'"
+          return amount
+        end
+
+        logger.debug "Not an amount, not parsing `#{s.inspect}'"
+        s
+      }
+
+      # rubocop:disable Style/AsciiComments
+      CSV::Converters[:transaction_dates] = lambda { |s|
+        begin
+          # Date.parse('6 decembre') is fine, but Date.parse('6 décembre') is
+          # an invalid date so we must remove diacritics before trying to parse
+          I18n.available_locales = [:en]
+          transliterated_s = I18n.transliterate s
+          logger.debug "Converted `#{s.inspect}' into date "\
+            "`#{Date.parse(transliterated_s)}'"
+          Date.parse(transliterated_s)
+        rescue StandardError
+          logger.debug "Not a date, not parsing #{s.inspect}"
+          s
+        end
+      }
+      # rubocop:enable Style/AsciiComments
+    end
+
+    def missing_transaction_date?(row)
+      # If It's missing a transaction date, it's most likely invalid
+      row[headers[:transaction_date]].nil?
+    end
+  end
+end

data/lib/ynab_convert/processor/ubs_chequing.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Processor
+  # Processes CSV files from UBS Personal Banking Switzerland (French)
+  class UbsChequing < Processor::Base
+    # @option options [String] :file Path to the CSV file to process
+    def initialize(options)
+      register_custom_converters
+      @loader_options = {
+        col_sep: ';',
+        converters: %i[amounts transaction_dates],
+        quote_char: nil,
+        encoding: Encoding::UTF_8,
+        headers: true
+      }
+      @institution_name = 'UBS (Chequing)'
+
+      super(options)
+    end
+
+    protected
+
+    def transformers(row)
+      date = extract_transaction_date(row).strftime('%d/%m/%Y')
+      payee = transaction_payee(row)
+      unless row[headers[:debit]].nil?
+        debit = format('%<amount>.2f', amount: row[headers[:debit]])
+      end
+      unless row[headers[:credit]].nil?
+        credit = format('%<amount>.2f', amount: row[headers[:credit]])
+      end
+
+      converted_row = [
+        date,
+        payee,
+        nil,
+        debit,
+        credit
+      ]
+
+      logger.debug "Converted row: #{converted_row}"
+      converted_row
+    end
+
+    def extract_transaction_date(row)
+      skip_row(row) if row[headers[:transaction_date]].nil?
+      row[headers[:transaction_date]]
+    end
+
+    private
+
+    def extract_header_names(row)
+      headers[:transaction_date] ||= row.headers[9]
+      headers[:payee_line_1] ||= row.headers[12]
+      headers[:payee_line_2] ||= row.headers[13]
+      headers[:payee_line_3] ||= row.headers[14]
+      headers[:debit] ||= row.headers[18]
+      headers[:credit] ||= row.headers[19]
+    end
+
+    def transaction_payee(row)
+      # Transaction description is spread over 3 columns
+      [
+        row[headers[:payee_line_1]],
+        row[headers[:payee_line_2]],
+        row[headers[:payee_line_3]]
+      ].join(' ')
+    end
+
+    def register_custom_converters
+      CSV::Converters[:amounts] = lambda { |s|
+        # Regex checks if string has only digits, apostrophes, and ends with a
+        # dot and two digits
+        amount_regex = /^[\d'?]+\.\d{2}$/
+
+        if !s.nil? && s.match(amount_regex)
+          amount = s.delete("'") .to_f
+          logger.debug "Converted `#{s}' into amount `#{amount}'"
+          return amount
+        end
+
+        logger.debug "Not an amount, not parsing `#{s.inspect}'"
+        s
+      }
+
+      CSV::Converters[:transaction_dates] = lambda { |s|
+        date_regex = /^\d{2}\.\d{2}\.\d{4}$/
+
+        if !s.nil? && s.match(date_regex)
+          parsed_date = Date.parse(s)
+          logger.debug "Converted `#{s.inspect}' into date "\
+            "`#{parsed_date}'"
+          parsed_date
+        else
+          logger.debug "Not a date, not parsing #{s.inspect}"
+          s
+        end
+      }
+    end
+  end
+end

data/lib/ynab_convert/processor/ubs_credit.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Processor
+  # Processes CSV files from UBS Credit Cards Switzerland (French)
+  class UbsCredit < Processor::Base
+    # @option options [String] :file Path to the CSV file to process
+    def initialize(options)
+      register_custom_converters
+      @loader_options = {
+        col_sep: ';',
+        converters: %i[amounts transaction_dates],
+        quote_char: nil,
+        encoding: "#{Encoding::ISO_8859_1}:#{Encoding::UTF_8}",
+        headers: true,
+        # CSV FTW, the first line in these files is not the headers but the
+        # separator specification
+        skip_lines: 'sep=;'
+      }
+      @institution_name = 'UBS (Credit cards)'
+
+      super(options)
+    end
+
+    protected
+
+    def transformers(row)
+      unless row[headers[:transaction_date]].nil?
+        date = row[headers[:transaction_date]].strftime('%d/%m/%Y')
+      end
+      payee = row[headers[:payee]]
+      unless row[headers[:debit]].nil?
+        debit = format('%<amount>.2f', amount: row[headers[:debit]])
+      end
+      unless row[headers[:credit]].nil?
+        credit = format('%<amount>.2f', amount: row[headers[:credit]])
+      end
+
+      converted_row = [date, payee, nil, debit, credit]
+      logger.debug "Converted row: #{converted_row}"
+      converted_row
+    end
+
+    private
+
+    def extract_header_names(row)
+      headers[:transaction_date] ||= row.headers[3]
+      headers[:payee] ||= row.headers[4]
+      headers[:debit] ||= row.headers[10]
+      headers[:credit] ||= row.headers[11]
+    end
+
+    def register_custom_converters
+      CSV::Converters[:amounts] = lambda { |s|
+        # Regex checks if string has only digits, apostrophes, and ends with a
+        # dot and two digits
+        amount_regex = /^[\d'?]+(\.\d{2})$/
+
+        if !s.nil? && s.match(amount_regex)
+          amount = s.delete("'") .to_f
+          logger.debug "Converted `#{s}' into amount `#{amount}'"
+          return amount
+        end
+
+        logger.debug "Not an amount, not parsing `#{s.inspect}'"
+        s
+      }
+
+      CSV::Converters[:transaction_dates] = lambda { |s|
+        date_regex = /^\d{2}\.\d{2}\.\d{4}$/
+
+        if !s.nil? && s.match(date_regex)
+          parsed_date = Date.parse(s)
+          logger.debug "Converted `#{s.inspect}' into date "\
+            "`#{parsed_date}'"
+          parsed_date
+        else
+          logger.debug "Not a date, not parsing #{s.inspect}"
+          s
+        end
+      }
+    end
+  end
+end

data/lib/ynab_convert/processors.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Base processor must be loaded first as all others inherit from it
+require 'ynab_convert/processor/base'
+
+# Load all known processors
+Dir[File.join(__dir__, 'processor', '*.rb')].each { |file| require file }

data/lib/ynab_convert/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module YnabConvert
+  VERSION = '0.1.0.pre'
+end