zilverline-mt940 1.0 → 2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    M2VjYjdlOGZlYzJkMDZmOWIzZTkzODkwNjc0YThmYjg0NTg0MDdmOA==
+    MTJlMDQ3YTFiYWIxM2NkODRkMzY0Y2I3MzRmZTRiNDA0YzlhZThiNA==
   data.tar.gz: !binary |-
-    Mjg3Mzg5YmYzMTE4NDkzYWIwYTBhMWVkMDlhM2U1M2UyYjg5ZTY1ZQ==
+    MjU0N2ZlY2NkYjU2YTFjMGVkZDU4MjhhMzkyYWM4ZGY4YjczNDRiNA==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YmRlMTM0ZTI5MzdmYmE3NGIwZTBmNGQwOTI1YmU1NzJlY2I1ZDNkNTk0YWY2
-    NWZhNzYzYWU1ZTFmNDJmNTE0NzRhZTgzN2M1NGIxNTNiN2Q1MjkzMzM5MzVj
-    ZjRjMDBlYjRiM2Y4M2IzY2Y5M2Q1OWVmYTFmMGQzNTMyMjI3NTM=
+    ZWVlNmQ0YmIyYmU0ZDc4M2E0ZWFlMWQzNzY1OTQ1YmI1NjVhNDFkNzJlMDA3
+    Y2NkMjAyZDkyNDdjYzM1N2ZjNDY5ZWJlZWU2MmQ1MmQzMjQ4Mjk3ODMwNDIw
+    YTQ0ZGNmMGM2YWY2YTVmY2I3MjY5NmVhYzY1MDcxYzRhMDEzOGY=
   data.tar.gz: !binary |-
-    NWU5Njk2YWU4N2E1ZTQ0YjUzZDVhZjkzY2ViNTEyOTQ0ZTE5MGQ0MjBkOWNi
-    NzRmOTNmNGZiODQ0MjZiMDE4ZDE4YzQ3N2M2MDYyYjc2OWY5YjAzNGJlMzUy
-    YzdmMGFmYzg2NjRiOTAwNDFkODkxODU4NTU2ODJmYjBhNTQzM2M=
+    Y2RjMzExNTk2MDA1OTg4MWM0MDIyOTFkZDExNjg5ZmVlNjllZDdmNzQ5Mzc2
+    NTA4NTI4ZDExMTljZDI5MTVjYjNiNzgzMjRlMDZmOTJjYzMwMjRkYzI5YjY1
+    NTYwZGNkZGY3NDM4OTE0Y2IzZTRjMDdiNDU1NGMxNTNhZTMyYzk=

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    zilverline-mt940 (1.0)
+    zilverline-mt940 (2.0)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -1,8 +1,9 @@
 MT940
 ======
 
-Full parser for MT940 files, see [MT940](http://nl.wikipedia.org/wiki/MT940). This is based on
-the original gem of [Frank Oxener - Agile Dovadi BV](http://github.com/dovadi/mt940) but completely redesigned and extended.
+Full parser for MT940 files, see [MT940](http://nl.wikipedia.org/wiki/MT940).
+Initially this is based on the original gem of [Frank Oxener - Agile Dovadi BV](http://github.com/dovadi/mt940)
+but as of version 2.0 completely rewritten in order to support MT940-structured format introduced by SEPA.
 
 The following Dutch banks are implemented:
 
@@ -18,7 +19,7 @@ With the file name as argument:
 
     file_name = '~/Downloads/ing.940'
 
-    @parse_result = MT940::Base.parse_mt940(file_name)
+    @parse_result = MT940Structured::Parser.parse_mt940(file_name)
 
 or with the file itself:
 
@@ -26,7 +27,7 @@ or with the file itself:
 
     file = File.open(file_name)
 
-    @parse_result = MT940::Base.parse_mt940(file)
+    @parse_result = MT940Structured::Parser.parse_mt940(file)
 
 after parsing:
 

data/lib/mt940.rb

@@ -2,9 +2,4 @@ require 'tempfile'
 require 'date'
 require_relative 'mt940/transaction'
 require_relative 'mt940/bank_statement'
-require_relative 'mt940/base'
-require_relative 'mt940/structured_format'
-require_relative 'mt940/banks/ing'
-require_relative 'mt940/banks/rabobank'
-require_relative 'mt940/banks/abnamro'
-require_relative 'mt940/banks/triodos'
+require_relative 'mt940_structured/mt940_structured'

data/lib/mt940/version.rb

@@ -1,3 +1,3 @@
 module MT940
-  VERSION = '1.0'
-end
+  VERSION = '2.0'
+end

data/lib/mt940_structured/file_content.rb

@@ -0,0 +1,46 @@
+class MT940Structured::FileContent
+  R_EOF_ING = /^-XXX$/
+  R_EOF_ABN_AMRO = /^-$/
+  R_EOF_TRIODOS = /^-$/
+
+  def initialize(raw_lines, join_lines_by = ' ')
+    @raw_lines = raw_lines.map{|line|line.strip}
+    @join_lines_by = join_lines_by
+  end
+
+  def get_header
+    MT940Structured::Header.new(@raw_lines)
+  end
+
+  def group_lines
+    body_lines = @raw_lines[start_index..(end_index-1)]
+    grouped_lines = []
+    previous_tag = nil
+    body_lines.each do |line|
+      mt940_line = line.match /^(:\d{2}[D|C|F|M]?:)/
+      if mt940_line && previous_tag != $1
+        previous_tag = $1
+        grouped_lines << line
+      else
+        next_line = if line.match /^(:\d{2}[D|C|F|M]?:)(.*)/
+                      $2
+                    else
+                      line
+                    end
+        grouped_lines[-1] = [grouped_lines.last, @join_lines_by, next_line].join
+      end
+    end
+    grouped_lines
+  end
+
+  private
+
+  def start_index
+    @raw_lines.index { |line| line.match /^:20:/ }
+  end
+
+  def end_index
+    @raw_lines.rindex { |line| line.match(R_EOF_ING) || line.match(R_EOF_ABN_AMRO) ||line.match(R_EOF_TRIODOS) } || 0
+  end
+
+end

data/lib/mt940_structured/header.rb

@@ -0,0 +1,30 @@
+module MT940Structured
+  class Header
+    R_RABOBANK = /^:940:/
+    R_ABN_AMRO = /ABNANL/
+    R_TRIODOS = /^:25:TRIODOSBANK/
+    R_ING = /INGBNL/
+
+    def initialize(raw_lines)
+      @raw_lines = raw_lines
+    end
+
+    def parser
+      if @raw_lines[0].match(R_RABOBANK)
+        MT940Structured::Parsers::Rabobank::Parser.new
+      elsif @raw_lines[0].match(R_ABN_AMRO)
+        MT940Structured::Parsers::Abnamro::Parser.new
+      elsif @raw_lines[1] && @raw_lines[1].match(R_TRIODOS)
+        MT940Structured::Parsers::Triodos::Parser.new
+      elsif @raw_lines[0].match(R_ING)
+        MT940Structured::Parsers::Ing::Parser.new
+      else
+        raise UnsupportedBankError.new
+      end
+    end
+
+  end
+  class UnsupportedBankError < StandardError
+
+  end
+end

data/lib/mt940_structured/mt940_structured.rb

@@ -0,0 +1,8 @@
+module MT940Structured
+
+end
+
+require_relative 'parsers/parsers'
+require_relative 'header'
+require_relative 'file_content'
+require_relative 'parser'

data/lib/mt940_structured/parser.rb

@@ -0,0 +1,18 @@
+module MT940Structured
+  class Parser
+    def self.parse_mt940(path, join_lines_by = ' ')
+      file_content = FileContent.new(readfile(path), join_lines_by)
+      grouped_lines = file_content.group_lines
+      file_content.get_header.parser.transform(grouped_lines)
+    end
+
+    private
+    def self.readfile(path)
+      File.open(path).readlines.map do |line|
+        line
+          .encode('UTF-8', 'binary', :invalid => :replace, :undef => :replace) # remove other obscure chars. god knows what people upload.
+          .gsub(/\u001A/, '') # remove eof chars in the middle of the string... yes it happens :-(
+      end
+    end
+  end
+end

data/lib/mt940_structured/parsers/abnamro/abnamro.rb

@@ -0,0 +1,5 @@
+module MT940Structured::Parsers::Abnamro
+end
+
+require_relative 'transaction_parser'
+require_relative 'parser'

data/lib/mt940_structured/parsers/abnamro/parser.rb

@@ -0,0 +1,15 @@
+module MT940Structured::Parsers::Abnamro
+  class Parser < MT940Structured::Parsers::Base
+    def initialize
+      super "Abnamro", TransactionParsers.new
+    end
+  end
+
+  class TransactionParsers
+
+    def for_format(_)
+      TransactionParser.new
+    end
+  end
+
+end

data/lib/mt940_structured/parsers/abnamro/transaction_parser.rb

@@ -0,0 +1,55 @@
+module MT940Structured::Parsers::Abnamro
+  class TransactionParser
+    include MT940Structured::Parsers::DateParser
+    include MT940Structured::Parsers::IbanSupport
+    include MT940Structured::Parsers::StructuredDescriptionParser
+    include MT940Structured::Parsers::DefaultLine61Parser
+
+    def get_regex_for_line_61
+      /^:61:(\d{6})\d{4}(C|D)(\d+),(\d{0,2})/
+    end
+
+    def enrich_transaction(transaction, line_86)
+      transaction.contra_account = "NONREF" #default
+      line_86 = line_86.gsub(/:86:/, '')
+      case line_86
+        when /^(GIRO)\s+(\d+)(.+)/
+          transaction.contra_account = $2.rjust(9, '000000000')
+          transaction.description = $3.strip
+        when /^(\d{2}.\d{2}.\d{2}.\d{3})(.+)/
+          transaction.description = $2.strip
+          transaction.contra_account = $1.gsub('.', '')
+        when /\/TRTP\/SEPA OVERBOEKING/
+          description_parts = line_86[4..-1].split('/')
+          transaction.contra_account_iban = parse_description_after_tag description_parts, "IBAN"
+          transaction.contra_account = iban_to_account transaction.contra_account_iban
+          transaction.contra_account_owner = parse_description_after_tag description_parts, "NAME"
+          transaction.description = parse_description_after_tag description_parts, "REMI"
+        when /SEPA IDEAL/
+          if line_86.match /OMSCHRIJVING\:(.+)?/
+            transaction.description = $1.strip
+          end
+          if line_86.match /IBAN\:(.+)?BIC\:/
+            transaction.contra_account_iban = $1.strip
+            transaction.contra_account = iban_to_account transaction.contra_account_iban
+          end
+          if line_86.match /NAAM\:(.+)?OMSCHRIJVING\:/
+            transaction.contra_account_owner = $1.strip
+          end
+        when /SEPA ACCEPTGIROBETALING/
+          if line_86.match /(BETALINGSKENM\.\:.+)/
+            transaction.description = $1.strip
+          end
+          if line_86.match /IBAN\:(.+)?BIC\:/
+            transaction.contra_account_iban = $1.strip
+            transaction.contra_account = iban_to_account transaction.contra_account_iban
+          end
+          if line_86.match /NAAM\:(.+)?BETALINGSKENM\.\:/
+            transaction.contra_account_owner = $1.strip
+          end
+        else
+          transaction.description = line_86
+      end
+    end
+  end
+end

data/lib/mt940_structured/parsers/balance_parser.rb

@@ -0,0 +1,12 @@
+module MT940Structured::Parsers
+  module BalanceParser
+    def parse_balance(line)
+      currency = line[12..14]
+      balance_date = parse_date(line[6..11])
+      type = line[5] == 'D' ? -1 : 1
+      amount = line[15..-1].gsub(",", ".").to_f * type
+      MT940::Balance.new(amount, balance_date, currency)
+    end
+
+  end
+end

data/lib/mt940_structured/parsers/bank_statement_parser.rb

@@ -0,0 +1,59 @@
+module MT940Structured::Parsers
+  class BankStatementParser
+    include DateParser, BalanceParser, IbanSupport
+    attr_reader :bank_statement
+
+    def initialize(bank, transaction_parsers, lines)
+      @bank = bank
+      @transaction_parsers = transaction_parsers
+      @bank_statement = MT940::BankStatement.new([])
+      lines.each do |line|
+        if line.match /^:(\d{2})(F|C|M)?:/
+          parse_method = "parse_line_#{$1}".to_sym
+          send(parse_method, line) if respond_to? parse_method
+        else
+          raise "nyi '#{$1}' - line #{line}"
+        end
+      end
+    end
+
+    def parse_line_25(line)
+      line.gsub!('.', '')
+      case line
+        when /^:\d{2}:NL/
+          @bank_statement.bank_account_iban = line[4, 18]
+          @bank_statement.bank_account = iban_to_account(@bank_statement.bank_account_iban)
+          @is_structured_format = true
+        when /^:\d{2}:\D*(\d*)/
+          @bank_statement.bank_account = $1.gsub(/\D/, '').gsub(/^0+/, '')
+          @is_structured_format = false
+        else
+          raise "Unknown format for tag 25: #{line}"
+      end
+    end
+
+    def parse_line_60(line)
+      @bank_statement.previous_balance = parse_balance(line)
+    end
+
+    def parse_line_61(line_61)
+      @is_structured_format = @transaction_parsers.structured?(line_61) if @transaction_parsers.respond_to?(:structured?)
+      @transaction_parser = @transaction_parsers.for_format @is_structured_format
+      transaction = @transaction_parser.parse_transaction(line_61)
+      transaction.bank_account = @bank_statement.bank_account
+      transaction.bank_account_iban = @bank_statement.bank_account_iban
+      transaction.currency = @bank_statement.previous_balance.currency
+      transaction.bank = @bank
+      @bank_statement.transactions << transaction
+    end
+
+    def parse_line_86(line)
+      @transaction_parser.enrich_transaction(@bank_statement.transactions.last, line)
+    end
+
+    def parse_line_62(line)
+      @bank_statement.new_balance = parse_balance(line)
+    end
+  end
+
+end

data/lib/mt940_structured/parsers/base.rb

@@ -0,0 +1,34 @@
+module MT940Structured::Parsers
+  class Base
+    def initialize(bank, transaction_parsers)
+      @bank = bank
+      @transaction_parsers = transaction_parsers
+    end
+
+    def transform(lines)
+      bank_statements = Hash.new { |h, k| h[k] = [] }
+      result = group_lines_by_tag(lines)
+      result.each do |bank_statement_lines|
+        bank_statement = BankStatementParser.new(@bank, @transaction_parsers, bank_statement_lines).bank_statement
+        bank_statements[bank_statement.bank_account] << bank_statement
+      end
+      bank_statements
+    end
+
+    private
+    def group_lines_by_tag(lines)
+      result = []
+      while !lines.empty? do
+        start_index = lines.index { |line| line.match(/^:20:/)}
+        end_index = lines.index { |line| line.match(/^:62F:/)}
+        if start_index && end_index > start_index
+          result << lines[start_index..end_index]
+          lines = lines.drop(end_index + 1)
+        else
+          lines = []
+        end
+      end
+      result
+    end
+  end
+end

data/lib/mt940_structured/parsers/date_parser.rb

@@ -0,0 +1,7 @@
+module MT940Structured::Parsers
+  module DateParser
+    def parse_date(string)
+      Date.new(2000 + string[0..1].to_i, string[2..3].to_i, string[4..5].to_i) if string
+    end
+  end
+end

data/lib/mt940_structured/parsers/default_line61_parser.rb

@@ -0,0 +1,25 @@
+module MT940Structured::Parsers
+  ##
+  # Basic line 61 parser. Retrieves the date and amount from the line :61:.
+  # This module expects that a method get_regex_for_line_61 exists that returns
+  # a regex that will, if matched, produces the following groups:
+  # $1 - the transaction date
+  # $2 - D for Debit, C for Credit transactions
+  # $3 - The amount of the transaction before the cent mark.
+  # $4 - The cents of the transaction
+  #
+  module DefaultLine61Parser
+    def get_regex_for_line_61
+      raise 'Override this when using this module'
+    end
+
+    def parse_transaction(line_61)
+      if line_61.match(get_regex_for_line_61)
+        type = $2 == 'D' ? -1 : 1
+        transaction = MT940::Transaction.new(amount: type * ($3 + '.' + $4).to_f)
+        transaction.date = parse_date($1)
+        transaction
+      end
+    end
+  end
+end

data/lib/mt940_structured/parsers/iban_support.rb

@@ -0,0 +1,15 @@
+module MT940Structured::Parsers
+  module IbanSupport
+    IBAN_R = /[a-zA-Z]{2}[0-9]{2}[a-zA-Z0-9]{0,30}/
+
+    def iban?(string)
+      !string.nil? and string.match(IBAN_R)
+    end
+
+    def iban_to_account(iban)
+      !iban.nil? ? iban.split(//).last(10).join.gsub(/^0+/, '') : nil
+    end
+
+  end
+
+end

data/lib/mt940_structured/parsers/ing/ing.rb

@@ -0,0 +1,7 @@
+module MT940Structured::Parsers::Ing
+end
+
+require_relative 'types'
+require_relative 'transaction_parser'
+require_relative 'structured_transaction_parser'
+require_relative 'parser'

data/lib/mt940_structured/parsers/ing/parser.rb

@@ -0,0 +1,18 @@
+module MT940Structured::Parsers::Ing
+  class Parser < MT940Structured::Parsers::Base
+    def initialize
+      super "Ing", TransactionParsers.new
+    end
+  end
+
+  class TransactionParsers
+    def structured?(line_61)
+      line_61.match /EREF|PREF|MARF|\d{16}/
+    end
+
+    def for_format(is_structured)
+      is_structured ? StructuredTransactionParser.new : TransactionParser.new
+    end
+  end
+
+end