ynab_convert 1.0.8 → 2.0.3

data/lib/ynab_convert/processor/base.rb

@@ -1,226 +0,0 @@
-# frozen_string_literal: true
-
-require 'core_extensions/string'
-require 'csv'
-require 'ynab_convert/logger'
-
-module Processor
-  # Base class for a Processor, all processors must inherit from it.
-
-  # rubocop:disable Metrics/ClassLength
-  class Base
-    include YnabLogger
-    include CoreExtensions::String::Inflections
-
-    attr_reader :loader_options
-
-    # @option options [String] :file Path to the CSV file to process
-    # @option options [Symbol] :format YNAB4 format to use, one of :flows or
-    #   :amounts. :flows is useful for CSVs with separate debit and credit
-    #   columns, :amounts is for CSVs with only one amount columns and +/-
-    #   numbers. See
-    #   https://docs.youneedabudget.com/article/921-formatting-csv-file
-    def initialize(options)
-      default_options = { file: '', format: :flows }
-      opts = default_options.merge(options)
-
-      logger.debug "Initializing processor with options: `#{opts.to_h}'"
-      raise ::Errno::ENOENT unless File.exist? opts[:file]
-
-      @file = opts[:file]
-      @headers = { transaction_date: nil, payee: nil }
-      @format = opts[:format]
-
-      if @format == :amounts
-        amounts_columns = { amount: nil }
-        @headers.merge!(amounts_columns)
-      else
-        flows_columns = { inflow: nil, outflow: nil }
-        @headers.merge!(flows_columns)
-      end
-    end
-
-    def to_ynab!
-      begin
-        convert!
-        rename_file
-      rescue YnabConvert::Error
-        invalid_csv_file
-      end
-    ensure
-      logger.debug "Deleting temp file `#{temp_filename}'"
-      delete_temp_csv
-    end
-
-    protected
-
-    attr_accessor :statement_from, :statement_to, :headers
-
-    def amount_invalid?(row)
-      amount_index = 3
-
-      # If there is no amount,
-      # then the row is invalid.
-      row[amount_index].nil? || row[amount_index].empty?
-    end
-
-    def inflow_outflow_invalid?(row)
-      inflow_index = 3
-      outflow_index = 4
-
-      # If there is neither inflow and outflow values,
-      # or both the inflow and outflow amounts are 0,
-      # then the row is invalid.
-      (
-        row[inflow_index].nil? ||
-        row[inflow_index].empty? ||
-        row[inflow_index] == '0.00'
-      ) && (
-        row[outflow_index].nil? ||
-        row[outflow_index].empty? ||
-        row[outflow_index] == '0.00'
-      )
-    end
-
-    def amounts_missing?(row)
-      logger.debug "Checking for missing amount in `#{row}`"
-      if @format == :amounts
-        logger.debug 'Using `:amounts`'
-        amount_invalid?(row)
-      else
-        logger.debug 'Using `:flows`'
-        inflow_outflow_invalid?(row)
-      end
-    end
-
-    def skip_row(row)
-      logger.debug "Found empty row, skipping it: #{row.to_h}"
-      throw :skip_row
-    end
-
-    def delete_temp_csv
-      FileUtils.remove_file temp_filename, force: true
-    end
-
-    def transaction_date_missing?(ynab_row)
-      ynab_row[0].nil? || [0].empty?
-    end
-
-    def extract_transaction_date(ynab_row)
-      transaction_date_index = 0
-      ynab_row[transaction_date_index]
-    end
-
-    def record_statement_interval_dates(ynab_row)
-      transaction_date_index = 0
-      date = Date.parse(ynab_row[transaction_date_index])
-
-      if date_is_further_away?(date)
-        logger.debug "Replacing statement_from `#{statement_from.inspect}' "\
-          "with `#{date}'"
-        self.statement_from = date
-      end
-      # rubocop:disable Style/GuardClause
-      if date_is_more_recent?(date)
-        logger.debug "Replacing statement_to `#{statement_to.inspect}' with "\
-          "`#{date}'"
-        self.statement_to = date
-      end
-      # rubocop:enable Style/GuardClause
-    end
-
-    def date_is_more_recent?(date)
-      statement_to.nil? || statement_to < date
-    end
-
-    def date_is_further_away?(date)
-      statement_from.nil? || statement_from > date
-    end
-
-    def convert!
-      logger.debug "Will write to `#{temp_filename}'"
-
-      logger.debug(loader_options)
-      CSV.open(temp_filename, 'wb', **output_options) do |converted|
-        CSV.foreach(@file, 'rb', **loader_options) do |row|
-          logger.debug "Parsing row: `#{row.to_h}'"
-          # Some rows don't contain valid or useful data
-          catch :skip_row do
-            extract_header_names(row)
-            ynab_row = transformers(row)
-            if amounts_missing?(ynab_row) ||
-               transaction_date_missing?(ynab_row)
-              logger.debug 'Empty row, skipping it'
-              skip_row(row)
-            end
-            converted << ynab_row
-            record_statement_interval_dates(ynab_row)
-          end
-
-          logger.debug 'Done converting'
-        end
-      end
-    end
-
-    def rename_file
-      File.rename(temp_filename, output_filename)
-      logger.debug "Renamed temp file `#{temp_filename}' to "\
-        "`#{output_filename}'"
-    end
-
-    def invalid_csv_file
-      raise YnabConvert::Error, "Unable to parse file `#{@file}'. Is it a "\
-        "valid CSV file from #{@institution_name}?"
-    end
-
-    def file_uid
-      @file_uid ||= rand(36**8).to_s(36)
-    end
-
-    def temp_filename
-      "#{File.basename(@file, '.csv')}_#{@institution_name.snake_case}_"\
-        "#{file_uid}_ynab4.csv"
-    end
-
-    def output_filename
-      # If the file contained no parsable CSV data, from and to dates will be
-      # nil.
-      # This is to avoid a NoMethodError on NilClass.
-      raise YnabConvert::Error if statement_from.nil? || statement_to.nil?
-
-      from = statement_from.strftime('%Y%m%d')
-      to = statement_to.strftime('%Y%m%d')
-
-      "#{File.basename(@file, '.csv')}_#{@institution_name.snake_case}_"\
-        "#{from}-#{to}_ynab4.csv"
-    end
-
-    def ynab_headers
-      common_headers = %w[Date Payee Memo]
-
-      if @format == :amounts
-        amounts_headers = %w[Amount]
-        common_headers.concat(amounts_headers)
-      else
-        flows_headers = %w[Outflow Inflow]
-        common_headers.concat(flows_headers)
-      end
-
-      common_headers
-    end
-
-    def output_options
-      {
-        converters: %i[numeric date],
-        force_quotes: true,
-        write_headers: true,
-        headers: ynab_headers
-      }
-    end
-
-    def transformers
-      raise NotImplementedError, :transformers
-    end
-  end
-  # rubocop:enable Metrics/ClassLength
-end

data/lib/ynab_convert/processor/example.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-module Processor
-  # An example of how to implement a custom processor
-  # Processes CSV files with this format:
-  # <<~ROWS
-  # "Date","Payee","Memo","Outflow","Inflow"
-  # "23/12/2019","coaxial","","1000000.00",""
-  # "30/12/2019","Santa","","50000.00",""
-  # "02/02/2020","Someone Else","","45.00",""
-  # ROWS
-  # The file name for the processor should be the institution name in
-  # camel case. It's ok to skip "Bank" or "Credit Union" when naming the file
-  # if it's redundant. For instance, this parser is for "Example Bank" but it's
-  # named "example.rb", its corresponding spec is
-  # "spec/example_processor_spec.rb" and its fixture would be
-  # "spec/fixtures/example/statement.csv"
-  class Example < Processor::Base
-    # @option options [String] :file Path to the CSV file to process
-    def initialize(options)
-      # Custom converters can be added so that the CSV data is parsed when
-      # loading the original file
-      register_custom_converters
-
-      # These are the options for the CSV module (see
-      # https://ruby-doc.org/stdlib-2.6/libdoc/csv/rdoc/CSV.html#method-c-new)
-      # They should match the format for the CSV file that the financial
-      # institution generates.
-      @loader_options = {
-        col_sep: ';',
-        # Use your converters, if any
-        converters: %i[transaction_date my_converter],
-        headers: true
-      }
-
-      # This is the financial institution's full name as it calls itself. This
-      # usually matches the institution's letterhead and/or commercial name.
-      # It can happen that the same institution needs different parsers because
-      # its credit card CSV files are in one format, and its chequing accounts
-      # in another. In that case, more details can be added in parens.
-      # For instance:
-      # 'Example Bank (credit cards)' and 'Example Bank (chequing)'
-      @institution_name = 'Example Bank'
-
-      # This is mandatory.
-      super(options)
-    end
-
-    private
-
-    def register_custom_converters
-      CSV::Converters[:transaction_date] = lambda { |s|
-        # Only match strings that have two digits, a dot, two digits, a dot,
-        # two digits, i.e. the dates in this institution's CSV files.
-        date_regex = /^\d{2}\.\d{2}\.\d{2}$/
-
-        if !s.nil? && s.match(date_regex)
-          parsed_date = Date.strptime(s, '%d.%m.%y')
-          logger.debug "Converted `#{s.inspect}' into date "\
-            "`#{parsed_date}'"
-          return parsed_date
-        end
-
-        s
-      }
-
-      CSV::Converters[:my_converter] = lambda { |s|
-        # A contrived example, just to illustrate multiple converters
-        if s.respond_to?(:downcase)
-          converted_s = s.downcase
-          logger.debug "Converted `#{s.inspect}' into downcased string "\
-            "`#{converted_s}'"
-          return converted_s
-        end
-
-        s
-      }
-    end
-
-    protected
-
-    # Converts the institution's CSV rows into YNAB4 rows.
-    # The YNAB4 columns are:
-    # "Date', "Payee", "Memo", "Outflow", "Inflow"
-    # which match Example Bank's "transaction_date" (after parsing),
-    # "beneficiary", nothing, "debit", and "credit" respectively.
-    # Note that Example Bank doesn't include any relevant column for YNAB4's
-    # "Memo" column so it's skipped and gets '' as its value.
-    def transformers(row)
-      # Convert the original transaction_date to DD/MM/YYYY as YNAB4 expects
-      # it.
-      unless row[headers[:transaction_date]].nil?
-        transaction_date = row[headers[:transaction_date]].strftime('%d/%m/%Y')
-      end
-      payee = row[headers[:payee]]
-      debit = row[headers[:debit]]
-      credit = row[headers[:credit]]
-
-      # CSV files can have funny data in them, including invalid or empty rows.
-      # These rows can be skipped from the converted YNAB4 file by calling
-      # skip_row when detected. In this particular case, if there is no
-      # transaction date, it means the row is empty or invalid and we discard
-      # it.
-      skip_row(row) if transaction_date.nil?
-      converted_row = [transaction_date, payee, nil, debit, credit]
-      logger.debug "Converted row: #{converted_row}"
-      converted_row
-    end
-
-    private
-
-    # Institutions love translating the column names, apparently. Rather than
-    # hardcoding the column name as a string, use the headers array at the
-    # right index.
-    # These lookups aren't particularly expensive but they're done on each row
-    # so why not memoize them with ||=
-    def extract_header_names(row)
-      headers[:transaction_date] ||= row.headers[0]
-      headers[:payee] ||= row.headers[2]
-      headers[:debit] ||= row.headers[3]
-      headers[:credit] ||= row.headers[4]
-    end
-  end
-end

data/lib/ynab_convert/processor/n26.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-module Processor
-  # Processes CSV files from N26
-  class N26 < Processor::Base
-    # @option options [String] :file Path to the CSV file to process
-    def initialize(options)
-      # Custom converters can be added so that the CSV data is parsed when
-      # loading the original file
-      register_custom_converters
-
-      # These are the options for the CSV module (see
-      # https://ruby-doc.org/stdlib-2.6/libdoc/csv/rdoc/CSV.html#method-c-new)
-      # They should match the format for the CSV file that the financial
-      # institution generates.
-      @loader_options = {
-        col_sep: ',',
-        quote_char: '"',
-        # Use your converters, if any
-        # converters: %i[],
-        headers: true,
-        encoding: 'bom|utf-8'
-      }
-
-      # This is the financial institution's full name as it calls itself. This
-      # usually matches the institution's letterhead and/or commercial name.
-      # It can happen that the same institution needs different parsers because
-      # its credit card CSV files are in one format, and its chequing accounts
-      # in another. In that case, more details can be added in parens.
-      # For instance:
-      # 'Example Bank (credit cards)' and 'Example Bank (chequing)'
-      @institution_name = 'N26 Bank'
-      # N26's CSV only has one columns for all transactions instead of separate
-      # debit and credit columns
-      additional_processor_options = { format: :amounts }
-
-      # This is mandatory.
-      super(options.merge(additional_processor_options))
-    end
-
-    private
-
-    def register_custom_converters; end
-
-    protected
-
-    def transformers(row)
-      transaction_date = row[headers[:transaction_date]]
-      payee = row[headers[:payee]]
-      amount = row[headers[:amount]]
-
-      converted_row = [transaction_date, payee, nil, amount]
-      logger.debug "Converted row: #{converted_row}"
-      converted_row
-    end
-
-    private
-
-    # Institutions love translating the column names, apparently. Rather than
-    # hardcoding the column name as a string, use the headers array at the
-    # right index.
-    # These lookups aren't particularly expensive but they're done on each row
-    # so why not memoize them with ||=
-    def extract_header_names(row)
-      headers[:transaction_date] ||= row.headers[0]
-      headers[:payee] ||= row.headers[1]
-      headers[:amount] ||= row.headers[5]
-    end
-  end
-end

data/lib/ynab_convert/processor/revolut.rb

@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-require 'i18n'
-
-module Processor
-  # Processes CSV files from Revolut
-  class Revolut < Processor::Base
-    # @option options [String] :file Path to the CSV file to process
-    def initialize(options)
-      register_custom_converters
-      @loader_options = {
-        col_sep: ';',
-        converters: %i[amounts transaction_dates],
-        quote_char: nil,
-        encoding: Encoding::UTF_8,
-        headers: true
-      }
-      @institution_name = 'Revolut'
-
-      super(options)
-    end
-
-    protected
-
-    def transformers(row)
-      date = extract_transaction_date(row).strftime('%d/%m/%Y')
-      payee = row[headers[:payee]]
-      unless row[headers[:debit]].nil?
-        debit = format('%<amount>.2f', amount: row[headers[:debit]])
-      end
-      unless row[headers[:credit]].nil?
-        credit = format('%<amount>.2f', amount: row[headers[:credit]])
-      end
-
-      ynab_row = [
-        date,
-        payee,
-        nil,
-        debit,
-        credit
-      ]
-
-      logger.debug "Converted row: #{ynab_row}"
-      ynab_row
-    end
-
-    private
-
-    def extract_header_names(row)
-      @headers[:transaction_date] ||= row.headers[0]
-      @headers[:payee] ||= row.headers[1]
-      @headers[:debit] ||= row.headers[2]
-      @headers[:credit] ||= row.headers[3]
-    end
-
-    def register_custom_converters
-      CSV::Converters[:amounts] = lambda { |s|
-        # Yes, amount come with a non breaking trailing space... Which is
-        # matched with \p{Zs} (c.f.
-        # https://ruby-doc.org/core-2.6/Regexp.html#class-Regexp-label-Character+Properties)
-        # Also, thousands separators can be non breaking spaces.
-        amount_regex = /^[\d'\.,\p{Zs}]+[\.,]\d{2}\p{Zs}$/
-        # narrow_nbsp = "\0xE2\0x80\0xAF"
-        narrow_nbsp = "\u{202F}"
-        readability_separators = "',. #{narrow_nbsp}"
-
-        if !s.nil? && s.match(amount_regex)
-          # This is a bit hacky because we don't have the luxury of Rails' i18n
-          # helpers. If we have an amount, strip all the separators in it, turn
-          # it to a float, and divide by 100 to get the right amount back
-          amount = s.delete(readability_separators).to_f / 100
-          logger.debug "Converted `#{s}' into amount `#{amount}'"
-          return amount
-        end
-
-        logger.debug "Not an amount, not parsing `#{s.inspect}'"
-        s
-      }
-
-      # rubocop:disable Style/AsciiComments
-      CSV::Converters[:transaction_dates] = lambda { |s|
-        begin
-          # Date.parse('6 decembre') is fine, but Date.parse('6 décembre') is
-          # an invalid date so we must remove diacritics before trying to parse
-          I18n.available_locales = [:en]
-          transliterated_s = I18n.transliterate s
-          logger.debug "Converted `#{s.inspect}' into date "\
-            "`#{Date.parse(transliterated_s)}'"
-          Date.parse(transliterated_s)
-        rescue StandardError
-          logger.debug "Not a date, not parsing #{s.inspect}"
-          s
-        end
-      }
-      # rubocop:enable Style/AsciiComments
-    end
-
-    def missing_transaction_date?(row)
-      # If It's missing a transaction date, it's most likely invalid
-      row[headers[:transaction_date]].nil?
-    end
-  end
-end

data/lib/ynab_convert/processor/ubs_chequing.rb

@@ -1,137 +0,0 @@
-# frozen_string_literal: true
-
-module Processor
-  # Processes CSV files from UBS Personal Banking Switzerland
-  class UbsChequing < Processor::Base
-    # @option options [String] :file Path to the CSV file to process
-    def initialize(options)
-      register_custom_converters
-      @loader_options = {
-        col_sep: ';',
-        converters: %i[amounts transaction_dates],
-        quote_char: nil,
-        encoding: Encoding::UTF_8,
-        headers: true
-      }
-      @institution_name = 'UBS (Chequing)'
-
-      super(options)
-    end
-
-    protected
-
-    def transformers(row)
-      date = extract_transaction_date(row).strftime('%d/%m/%Y')
-      payee = transaction_payee(row)
-      unless row[headers[:debit]].nil?
-        debit = format('%<amount>.2f', amount: row[headers[:debit]])
-      end
-      unless row[headers[:credit]].nil?
-        credit = format('%<amount>.2f', amount: row[headers[:credit]])
-      end
-
-      converted_row = [
-        date,
-        payee,
-        nil,
-        debit,
-        credit
-      ]
-
-      logger.debug "Converted row: #{converted_row}"
-      converted_row
-    end
-
-    def extract_transaction_date(row)
-      skip_row(row) if row[headers[:transaction_date]].nil?
-      row[headers[:transaction_date]]
-    end
-
-    private
-
-    def extract_header_names(row)
-      headers[:transaction_date] ||= row.headers[9]
-      headers[:payee_line_1] ||= row.headers[12]
-      headers[:payee_line_2] ||= row.headers[13]
-      headers[:payee_line_3] ||= row.headers[14]
-      headers[:debit] ||= row.headers[18]
-      headers[:credit] ||= row.headers[19]
-    end
-
-    def transaction_payee(row)
-      raw_payee_line = [
-        row[headers[:payee_line_2]],
-        row[headers[:payee_line_3]]
-      ]
-
-      # Transaction description is spread over 3 columns.
-      # There are two types of entries:
-      # 1. only the first column contains data
-      # 2. all three columns contain data, most of it junk
-      #
-      # Cleaning them up means dropping the first column if there is anything
-      # in the other columns;
-      # removing the CARD 00000000-0 0000 at the beginning of debit card
-      # payment entries;
-      # removing the rest of the junk appended after the worthwhile data (see
-      # below for details on that)
-      if row[headers[:payee_line_2]].nil?
-        # Make it an Array, for consistency
-        raw_payee_line = [row[headers[:payee_line_1]]]
-      end
-
-      concat_payee_line = raw_payee_line.join(' ')
-
-      # Moreover, UBS thought wise to append a bunch of junk information after
-      # the transaction details within the third description field. *Most* of
-      # this junk starts after the meaningful data and starts with ", OF",
-      # ", ON", ", ESR", ", QRR", two digits then five groups of five digits
-      # then ", TN" so we discard it; YNAB4 being unable to automatically
-      # categorize new transactions at the same store/payee because the payee
-      # always looks different (thanks to the variable nature of the appended
-      # junk).
-      # See `spec/fixtures/ubs_chequing/statement.csv` L2 and L18--22
-
-      # rubocop:disable Metrics/LineLength
-      junk_desc_regex = /,? (O[FN]|ESR|QRR|\d{2} \d{5} \d{5} \d{5} \d{5} \d{5}, TN).*/
-      # rubocop:enable Metrics/LineLength
-
-      # Of course, it wouldn't be complete with more junk information at the
-      # beginning of *some* lines (debit card payments)
-      debit_card_junk_regex = /CARD \d{8}\-\d \d{4} /
-
-      concat_payee_line.sub(junk_desc_regex, '').sub(debit_card_junk_regex, '')
-    end
-
-    def register_custom_converters
-      CSV::Converters[:amounts] = lambda { |s|
-        # Regex checks if string has only digits, apostrophes, and ends with a
-        # dot and two digits
-        amount_regex = /^[\d'?]+\.\d{2}$/
-
-        if !s.nil? && s.match(amount_regex)
-          amount = s.delete("'") .to_f
-          logger.debug "Converted `#{s}' into amount `#{amount}'"
-          return amount
-        end
-
-        logger.debug "Not an amount, not parsing `#{s.inspect}'"
-        s
-      }
-
-      CSV::Converters[:transaction_dates] = lambda { |s|
-        date_regex = /^\d{2}\.\d{2}\.\d{4}$/
-
-        if !s.nil? && s.match(date_regex)
-          parsed_date = Date.parse(s)
-          logger.debug "Converted `#{s.inspect}' into date "\
-            "`#{parsed_date}'"
-          parsed_date
-        else
-          logger.debug "Not a date, not parsing #{s.inspect}"
-          s
-        end
-      }
-    end
-  end
-end