ynab_convert 1.0.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 697f0d80eaec169e83bea48407e9d00c1b000a362aaa6147160be65bafef72a7
-  data.tar.gz: 8e45ffa5ec9094c54057ebba3f241c00764fc0e256ec19bdfa0d9395f590a156
+  metadata.gz: 142532dddc82b58043e383e388306801471835c48004fd55b7804709a545999c
+  data.tar.gz: e7de2e56779760e27e41a3cf3bcbaf265bd66377a80525ad00f7133b57c1b05a
 SHA512:
-  metadata.gz: 8150a18508c46b6267b38ccf56ff6b542e82df49100c4fff164959a24445a36e8271cfca6bd07abbc583f76d372bfc77f9ab830a88dd9e37eb5d95af15ab6ef5
-  data.tar.gz: 7490a9741e4bf27fd21e526fe4aff6c777cca8882f64a9558ceb698f16b7c080f92ae24ebf136a21c3768f5e76077c4ccbe6536a81cdc0cd0dec1453e279a40d
+  metadata.gz: a4e37509faa4be4db330876e0c6af4bb2deb0927b4bb4aa8feb9b1943a69690d8765087cfe72ebeb49161b708999253e7443f5b3275f5cfc42bc3f23e93232cf
+  data.tar.gz: 4a0eb0cee6a35847ba39d747f6343c39c5cb0750ca8347f9af0ad70fb3d8382ac5e098bbb6b47c8d931b1d095d4829442fd0315b697d6005003ccc8aa7685638

data/.gitignore

@@ -9,3 +9,8 @@
 
 # rspec failure tracking
 .rspec_status
+
+# test artifacts
+*_ynab4.csv
+.byebug_history
+ynab_convert.yml

data/.rubocop.yml

@@ -1,14 +1,22 @@
 ---
 # See https://github.com/rubocop/rubocop/blob/master/config/default.yml for all
 # options
-require: rubocop-rake
+require:
+  - rubocop-rake
+  - rubocop-rspec
 
 AllCops:
   DisplayCopNames: true
+  NewCops: enable
+  TargetRubyVersion: 2.6
 
-Metrics/LineLength:
+Layout/LineLength:
+  AllowHeredoc: true
+  AllowURI: true
+  AutoCorrect: true
   Exclude:
     - ynab_convert.gemspec
+  Max: 80
 
 Metrics/BlockLength:
   Exclude:

data/Gemfile.lock

@@ -1,18 +1,23 @@
 PATH
   remote: .
   specs:
-    ynab_convert (1.0.6)
+    ynab_convert (2.0.0)
       i18n
       slop
+      timecop
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
     backport (1.1.2)
     byebug (11.0.1)
     coderay (1.1.2)
     concurrent-ruby (1.1.9)
+    crack (0.4.5)
+      rexml
     diff-lcs (1.3)
     docile (1.3.2)
     ffi (1.11.2)
@@ -34,6 +39,7 @@ GEM
     guard-rubocop (1.3.0)
       guard (~> 2.0)
       rubocop (~> 0.20)
+    hashdiff (1.0.1)
     htmlentities (4.3.4)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
@@ -52,23 +58,26 @@ GEM
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.19.0)
-    parser (2.6.5.0)
-      ast (~> 2.4.0)
+    parallel (1.21.0)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
     pry-byebug (3.7.0)
       byebug (~> 11.0)
       pry (~> 0.10)
+    public_suffix (4.0.6)
     racc (1.6.0)
-    rainbow (3.0.0)
+    rainbow (3.1.1)
     rake (13.0.1)
     rb-fsevent (0.10.3)
     rb-inotify (0.10.0)
       ffi (~> 1.0)
+    regexp_parser (2.2.1)
     reverse_markdown (1.3.0)
       nokogiri
+    rexml (3.2.5)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -82,16 +91,23 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.0)
-    rubocop (0.76.0)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.93.1)
       parallel (~> 1.10)
-      parser (>= 2.6)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
     rubocop-rake (0.5.0)
       rubocop
-    ruby-progressbar (1.10.1)
+    rubocop-rspec (1.44.1)
+      rubocop (~> 0.87)
+      rubocop-ast (>= 0.7.1)
+    ruby-progressbar (1.11.0)
     shellany (0.0.1)
     simplecov (0.17.1)
       docile (~> 1.1)
@@ -113,7 +129,13 @@ GEM
       yard (~> 0.9)
     thor (0.20.3)
     tilt (2.0.10)
-    unicode-display_width (1.6.0)
+    timecop (0.9.5)
+    unicode-display_width (1.8.0)
+    vcr (6.1.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     yard (0.9.20)
 
 PLATFORMS
@@ -129,8 +151,11 @@ DEPENDENCIES
   rspec-core
   rubocop
   rubocop-rake
+  rubocop-rspec
   simplecov
   solargraph
+  vcr
+  webmock
   ynab_convert!
 
 BUNDLED WITH

data/Guardfile

@@ -44,37 +44,9 @@ group :red_green_refactor, halt_on_fail: true do
     # Ruby files
     ruby = dsl.ruby
     dsl.watch_spec_files_for(ruby.lib_files)
-
-    # Rails files
-    rails = dsl.rails(view_extensions: %w[erb haml slim])
-    dsl.watch_spec_files_for(rails.app_files)
-    dsl.watch_spec_files_for(rails.views)
-
-    watch(rails.controllers) do |m|
-      [
-        rspec.spec.call("routing/#{m[1]}_routing"),
-        rspec.spec.call("controllers/#{m[1]}_controller"),
-        rspec.spec.call("acceptance/#{m[1]}")
-      ]
-    end
-
-    # Rails config changes
-    watch(rails.spec_helper)     { rspec.spec_dir }
-    watch(rails.routes)          { "#{rspec.spec_dir}/routing" }
-    watch(rails.app_controller)  { "#{rspec.spec_dir}/controllers" }
-
-    # Capybara features specs
-    watch(rails.view_dirs)     { |m| rspec.spec.call("features/#{m[1]}") }
-    watch(rails.layouts)       { |m| rspec.spec.call("features/#{m[1]}") }
-
-    # Turnip features and steps
-    watch(%r{^spec/acceptance/(.+)\.feature$})
-    watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$}) do |m|
-      Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance'
-    end
   end
 
-  guard :rubocop, cli: ['--auto-correct', '--display-cop-names'] do
+  guard :rubocop, cli: ['--auto-correct-all', '--display-cop-names'] do
     watch('Gemfile')
     watch('Rakefile')
     watch('bin/convert')

data/README.md

@@ -27,35 +27,106 @@ latest one on 2019-12-01.
 `-i` argument | Institution's full name | Institution's website | Remarks
 ---|---|---|---
 `example` | Example Bank | N/A | Reference processor implementation, not a real institution
-`n26` | N26 | [n26.com](n26.com) | N26 CSV statements
+`n26` | N26 | [n26.com](n26.com) | N26 CSV statements, will convert EUR amounts to CHF (hardcoded for now)
 `revolut` | Revolut Ltd | [revolut.com](https://www.revolut.com/) | The processor isn't aware of currencies. Make sure the statements processed with `revolut` are in the same currency that your YNAB is in
 `ubs_chequing` | UBS Switzerland (private banking) | [ubs.ch](https://ubs.ch) | Private chequing and joint accounts
 `ubs_credit` | UBS Switzerland (credit cards) | [ubs.ch](https://ubs.ch) | Both MasterCard and Visa
+`wise` | Wise (Transferwise) cards | [wise.com](https://wise.com) | Performs currency conversion (hardcoded to CHF for now)
 
 ## Contributing
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run
 `rake spec` to run the tests. You can also run `bin/console` for an interactive
-prompt that will allow you to experiment.
+prompt that will allow you to experiment. To run Rubocop and RSpec in watch
+mode, use `bundle exec guard`.
 
 To install this gem onto your local machine, run `bundle exec rake install`.
+Alternatively, the gem can also be run from `bin/ynab_convert`.
 
 Bug reports and pull requests are welcome on GitHub at
 https://github.com/coaxial/ynab_convert.
 
-### Enable debug output
+### Architecture
+
+Here is the class diagram:
+```mermaid
+classDiagram
+    Documents <|-- Statement
+    Documents <|-- YNAB4File
+    Transformers <|-- Cleaner
+    Transformers <|-- Formatter
+    Transformers <|-- Enhancer
+    Validators <|-- YNAB4Row
+    Processors <|-- Processor
+
+    class Statement{
+        #Hash csv_import_options
+        #String filepath
+        #String institution_name
+    }
+
+    class YNAB4File{
+        #Hash csv_export_options
+        #String filename
+        #update_dates(row)
+    }
+
+    class Processor{
+        #to_ynab!()
+    }
+
+    class Cleaner{
+        #run(row)
+    }
+
+    class Enhancer{
+        #run(row)
+    }
+
+    class Formatter{
+        #run(row)
+    }
+
+    class YNAB4Row{
+        +valid?(row)
+    }
+```
+
+Each financial institution gets its own class for most of these base classes.
+
+For instance, adding "Some Bank" would require creating the following new
+classes:
+
+- `class Processors::SomeBank < Processor`
+- `class Transformers::Cleaners::SomeBank < Cleaner`
+- `class Transformers::Formatter::SomeBank < Formatter`
+- `class Transformers::Enhancer::SomeBank < Enhancer`
+- `class Documents::Statements::SomeBank < Statement`
+
+Each of these classes would implement the expected interface for its type, and
+the `Processor::SomeBank` would instantiate them all. `Validators` and `YNAB4File`
+aren't related to a particular institution, there is no need to derive a child
+class for each bank.
+
+Note that any of the `Transformers::` classes are optional, and it is possible
+that some institution only requires a `Cleaner` but no `Formatter` or
+`Enhancer` (for example).
+
+### Debugging
 
 Run `ynab_convert` with `YNAB_CONVERT_DEBUG=true`, or use the rake task
 `spec:debug`. Debug logging goes to STDERR.
 
+Or add `byebug` or `pry` statements in the code (works with guard and with rspec).
+
 ### Adding a new financial institution
 
 If there is no processor for your financial institution, you can contribute one
 to the project.
 
 There is a commented example processor located at
-`lib/ynab_convert/processor/example.rb`. Looking at the other, real-world
-processors in that directory can also help.
+`lib/ynab_convert/processors/example_processor.rb`. Looking at the other,
+real-world processors in that directory can also help.
 
 Be sure to add tests to your processor as well before you make a PR.
 

data/lib/ynab_convert/api_clients/api_client.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'json'
+
+module APIClients
+  # Base APIClient
+  class APIClient
+    # @param api_base_path [String] Base path to the API
+    def initialize(api_base_path:)
+      @api_base_path = api_base_path
+    end
+
+    private
+
+    def make_request(endpoint:)
+      uri = URI(URI.join(@api_base_path, endpoint))
+
+      response = Net::HTTP.get_response(uri)
+
+      JSON.parse(response.body, symbolize_names: true)
+    end
+  end
+end

data/lib/ynab_convert/api_clients/currency_api.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/api_clients/api_client'
+
+module APIClients
+  # Client for currency-api
+  # (https://github.com/fawazahmed0/currency-api#readme)
+  class CurrencyAPI < APIClient
+    # The days that are missing from the API's otherwise normally available
+    # range
+    MISSING_DAYS = { '2021-09-14' => true }.freeze
+
+    def initialize
+      api_base_path = 'https://cdn.jsdelivr.net/gh/fawazahmed0/currency-api@1/'
+      @available_date_range = {
+        min: Date.parse('2020-11-22'),
+        max: Date.today - 1 # yesterday
+      }
+
+      super(api_base_path: api_base_path)
+    end
+
+    # @param base_currency [Symbol] ISO symbol for base currency
+    # @param date [Date, String] The date on which to get the rates for
+    # @return [Hash<Symbol, Numeric>] The rates for that day in base_currency
+    def historical(base_currency:, date:)
+      parsed_date = date.is_a?(Date) ? date : Date.parse(date)
+      handle_date_out_of_bounds(parsed_date) if out_of_bounds?(parsed_date)
+      # Some days are missing from the API, use the previous day's rate if
+      # a missing day is requested
+      parsed_date -= 1 if missing_day?(date)
+      currency = base_currency.downcase
+      endpoint = "#{parsed_date}/currencies/#{currency}.min.json"
+      rates = make_request(endpoint: endpoint)
+
+      rates[currency]
+    end
+
+    private
+
+    # The currency-api only has rates since 2020-11-22 and until yesterday
+    # (the current day's rate are updated at 23:59 on that day). This method
+    # ensures the requested date falls within the available range.
+    # @param date [Date] The date to check
+    # @return [Boolean] Whether the date is out of bounds for this API
+    def out_of_bounds?(date)
+      date < @available_date_range[:min] || date > @available_date_range[:max]
+    end
+
+    # @param date [Date] The date to show in the error message
+    def handle_date_out_of_bounds(date)
+      error_message = "#{date} is out of the currency-api available date "\
+      "range (#{@available_date_range[:min]}–#{@available_date_range[:max]})"
+
+      raise Errno::EDOM, error_message
+    end
+
+    # Indicates whether a date is missing from the API's normally available
+    # date range
+    # @param date [Date] the date to check
+    # @return [Boolean] whether the date is unavailable in the API
+    def missing_day?(date)
+      MISSING_DAYS.key?(date.to_s)
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/example_statement.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents/statements/statement'
+
+module Documents
+  module Statements
+    # Example of a Statement
+    class Example < Statement
+      def initialize(filepath:)
+        csv_import_options = { col_sep: ';', quote_char: nil, headers: true }
+
+        super(filepath: filepath, csv_import_options: csv_import_options)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/n26_statement.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents/statements/statement'
+
+module Documents
+  module Statements
+    # Represents a statement from N26 Bank
+    class N26 < Statement
+      # @param filepath [String] Path to CSV statement
+      # @return [void]
+      def initialize(filepath:)
+        csv_import_options = {
+          col_sep: ',',
+          quote_char: '"',
+          headers: true,
+          encoding: 'bom|utf-8'
+        }
+
+        super(filepath: filepath,
+              csv_import_options: csv_import_options,)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/statement.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Documents
+  module Statements
+    # The base Statement class from which other Statements inherit.
+    # Represents a CSV statement from a financial institution, typically from
+    # its online banking portal.
+    class Statement
+      attr_reader :csv_import_options, :filepath
+
+      # @param filepath [String] path to the CSV file
+      # @param csv_import_options [CSV::DEFAULT_OPTIONS] options describing
+      #   the particular CSV flavour (column separator, etc). Any
+      #   CSV::DEFAULT_OPTIONS is valid.
+      def initialize(filepath:, csv_import_options: CSV::DEFAULT_OPTIONS)
+        validate(filepath)
+
+        default_options = CSV::DEFAULT_OPTIONS.merge(converters: %i[numeric
+                                                                    date])
+        @filepath = filepath
+        @csv_import_options = default_options.merge(csv_import_options)
+      end
+
+      def institution_name
+        self.class.name.split('::').last
+      end
+
+      private
+
+      # Verifies that the file exists at path, raises an error if not.
+      # @param path [String] path to the file
+      def validate(path)
+        return if ::File.exist?(path)
+
+        raise Errno::ENOENT, "file not found #{path}"
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/ubs_chequing_statement.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Documents
+  module Statements
+    # UBS Switzerland Chequing accounts statement
+    class UBSChequing < Statement
+      # @param filepath [String] path to CSV statement
+      def initialize(filepath:)
+        csv_import_options = {
+          col_sep: ';',
+          quote_char: nil,
+          encoding: Encoding::UTF_8,
+          headers: true
+        }
+
+        super(filepath: filepath, csv_import_options: csv_import_options)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/ubs_credit_statement.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Documents
+  module Statements
+    # UBS Switzerland Credit Card accounts statement
+    class UBSCredit < Statement
+      def initialize(filepath:)
+        csv_import_options = {
+          col_sep: ';',
+          quote_char: nil,
+          headers: true,
+          encoding: "#{Encoding::ISO_8859_1}:#{Encoding::UTF_8}",
+          skip_lines: 'sep=;'
+        }
+        super(filepath: filepath, csv_import_options: csv_import_options)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/statements/wise_statement.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Documents
+  module Statements
+    # Wise card accounts statement
+    class Wise < Statement
+      def initialize(filepath:)
+        csv_import_options = {
+          col_sep: ',',
+          quote_char: '"',
+          headers: true
+        }
+        super(filepath: filepath, csv_import_options: csv_import_options)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents/ynab4_files/ynab4_file.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Documents
+  module YNAB4Files
+    # Represents the YNAB4 formatted CSV data for importing into YNAB4
+    class YNAB4File
+      attr_reader :csv_export_options
+
+      def initialize(institution_name:, format: :flows)
+        @format = format
+        @institution_name = institution_name
+        @csv_export_options = {
+          converters: %i[numeric date],
+          force_quotes: true,
+          write_headers: true,
+          headers: headers
+        }
+      end
+
+      def update_dates(row)
+        date_index = 0
+        transaction_date = row[date_index]
+        unless transaction_date.is_a?(Date)
+          transaction_date = Date.parse(transaction_date)
+        end
+
+        update_start_date(transaction_date)
+        update_end_date(transaction_date)
+      end
+
+      def filename
+        from_date = @start_date.strftime('%Y%m%d')
+        to_date = @end_date.strftime('%Y%m%d')
+
+        "#{@institution_name.snake_case}_#{from_date}-#{to_date}_ynab4.csv"
+      end
+
+      private
+
+      def update_start_date(date)
+        @start_date = date if @start_date.nil? || date < @start_date
+      end
+
+      def update_end_date(date)
+        @end_date = date if @end_date.nil? || date > @end_date
+      end
+
+      def headers
+        base_headers = %w[Date Payee Memo]
+        extra_headers = %w[Outflow Inflow]
+
+        extra_headers = %w[Amount] if @format == :amounts
+
+        base_headers.concat(extra_headers)
+      end
+    end
+  end
+end

data/lib/ynab_convert/documents.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Groups Statements and YNAB4File
+module Documents
+  documents = %w[statement ynab4_file]
+
+  # Load all known Documents
+  documents.each do |d|
+    # Require the base classes first so that its children can find the parent
+    # class since files are otherwise loaded in alphabetical order
+    require File.join(__dir__, 'documents', "#{d}s", "#{d}.rb")
+
+    Dir[File.join(__dir__, 'documents', "#{d}s", '*.rb')].sort.each do |file|
+      require file
+    end
+  end
+end

data/lib/ynab_convert/logger.rb

@@ -6,7 +6,7 @@ require 'logger'
 module YnabLogger
   def logger
     @logger unless @logger.nil?
-    @logger ||= Logger.new(STDERR)
+    @logger ||= Logger.new($stderr)
     @logger.level = Logger::FATAL
     @logger.level = Logger::DEBUG if ENV['YNAB_CONVERT_DEBUG'] == 'true'
     @logger

data/lib/ynab_convert/processors/example_processor.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'ynab_convert/documents'
+require 'ynab_convert/transformers'
+require 'ynab_convert/processors/processor'
+
+module Processors
+  # Example Processor
+  class Example < Processor
+    # @param filepath [String] path to the CSV file
+    def initialize(filepath:)
+      transformers = [
+        Transformers::Formatters::Example.new
+      ]
+      statement = Documents::Statements::Example.new(filepath: filepath)
+      ynab4_file = Documents::YNAB4Files::YNAB4File.new(
+        format: :flows, institution_name: statement.institution_name
+      )
+
+      super(statement: statement, ynab4_file: ynab4_file, transformers:
+            transformers)
+    end
+  end
+end