cmxl 1.5.0 → 2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 661a3296bdc88ce9874f07cb4a1cf4220bf1a60ea559e289edc5c5c34774764b
-  data.tar.gz: 4f0217f656646bd14540e5993f6419354641f30d901e8b7afc884b46fc5ac69e
+  metadata.gz: f7d3b16a51915ab4abdd146b5f213b4957392ace7f27be151fbc253ea820dd6c
+  data.tar.gz: cc398d5bd6fb2e112abfa815de950fd34a300107ba573b50ea8fb9ed98703f23
 SHA512:
-  metadata.gz: 407ecb157601f8cc767ddb7699edbadc66582041cdcd8d1de3b3e169a41865d3f81ffdb794b7b9f8add8a55ecea1e16e4048f30a117cfe2401b05b487d2e1bcf
-  data.tar.gz: 4cbfe3e46cc9a00527bfa1a4d465f5e0ad4311d915449b45a81b172e1a653ee3ec2605778bb1241aca752016651b8f54dad0daf00769349a6a80cb679cff0dcf
+  metadata.gz: 9c9d4c881bc2f1f644471bf24a5c2f0c0fdc0543f828ba82e2960f1a015d397f10cdbc74d5fece4f79f2638625000ba8f8e80d0bbfa7ead01aec2b09992ff940
+  data.tar.gz: 2ba29e61b1060f43ddc7699211ff109064256366e18089a1d46b41b513906fabbb60ca12a74d322595f6ef726ea9068150829d0e0c5c4c3d2c7c690a0aeef309

data/.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        ruby-version:
+          - 3.4
+          - 3.3
+          - 3.2
+          - 3.1
+          - "3.0"
+          - jruby-9.4
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run tests
+        run: bundle exec rake

data/.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Release
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  push:
+    name: Push gem to RubyGems.org
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
+      contents: write # IMPORTANT: this permission is required for `rake release` to push the release tag
+
+    steps:
+      # Set up
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: ruby
+
+      # Release
+      - uses: rubygems/release-gem@v1

data/.rspec

@@ -1 +1,2 @@
 --color
+--require 'pry'

data/CHANGELOG.mdown

@@ -1,5 +1,21 @@
 # NEXT release
 
+# 2.2
+- `[ENHANCEMENT]` enable `strip_headers` option per default.
+- `[BUGFIX]` fix `strip_header` remove greedy regex logic
+- `[BUGFIX]` support details parsing from files on windows os
+
+# 2.1
+- `[REFACTOR]` improve file parser to work with windows line breaks and any known header formats
+- `[BUGFIX]` fix `strip_header` making it work on statements without headers
+- `[HOUSEKEEPING]` use `main` instead of `master` as default branch
+- `[ENHANCEMENT]` add support for ruby `3.4`
+
+# 2.0
+- `[REFACTOR]` **DEPRECATED:** `storno?` and related methods `storno_credit`, `storno_debit`. Use `reversal?` and related methods `reversal_credit?`, `reversal_debit?` instead
+- `[BUGFIX]` **DEPRECATED:** `funds_code` method returns the `credit_debit_indicator` from the SWIFT definition. Therefore the method is deprecated in favor of `credit_debit_indicator` method
+- `[HOUSEKEEPING]` [Replace Travis CI with github actions](https://github.com/railslove/cmxl/pull/57)
+
 # 1.5.0
 
 - `[BUGFIX]` fix potential bug when generation_date is not provided in field 20 and 13 (issue: [#35](https://github.com/railslove/cmxl/issues/35) PR: [#36](https://github.com/railslove/cmxl/pull/36))

data/README.md

@@ -1,5 +1,5 @@
-[![Build Status](https://travis-ci.org/railslove/cmxl.svg?branch=master)](https://travis-ci.org/railslove/cmxl)
-[![Gem Version](https://badge.fury.io/rb/cmxl.svg)](http://badge.fury.io/rb/cmxl)
+[![CI](https://github.com/railslove/cmxl/actions/workflows/ci.yml/badge.svg)](https://github.com/railslove/cmxl/actions/workflows/ci.yml)
+[![Gem Version](https://badge.fury.io/rb/cmxl.svg)](https://rubygems.org/gems/cmxl)
 
 # Cmxl - your friendly ruby MT940 parser
 
@@ -16,7 +16,7 @@ For more information have a look at the different [SWIFT message types](http://e
 
 At some point in the future MT940 file should be exchanged with newer XML documents - but banking institutions are slow, so MT940 will stick around for a while.
 
-## Reqirements
+## Requirements
 
 Cmxl is a pure ruby parser and has no dependency on native extensions.
 
@@ -38,37 +38,23 @@ Or install it yourself as:
 
 ## Usage
 
-Simple usage:
+### Simple usage:
 
 ```ruby
+statements = Cmxl.parse(File.read('mt940.txt'))
 
-# Configuration:
-
-# statement divider regex to split the individual statements in one file - the default is standard and should be good for most files
-Cmxl.config[:statement_separator] = /\n-.\n/m
-
-# do you want an error to be raised when a line can not be parsed? default is true
-Cmxl.config[:raise_line_format_errors] = true
-
-# try to stip the SWIFT header data. This strips everything until the actual first MT940 field. (if parsing fails, try this!)
-Cmxl.config[:strip_headers] = true
-
-
-# Statment parsing:
-
-statements = Cmxl.parse(File.read('mt940.txt'), :encoding => 'ISO-8859-1') # parses the file and returns an array of statement objects. Please note: if no encoding is given Cmxl tries to guess the encoding from the content and converts it to UTF-8.
 statements.each do |s|
   puts s.reference
   puts s.generation_date
   puts s.opening_balance.amount
   puts s.closing_balance.amount
-  puts s.sha # SHA of the statement source - could be used as an identifier (see: https://github.com/railslove/cmxl/blob/master/lib/cmxl/statement.rb#L49-L55)
+  puts s.sha # SHA of the statement source - could be used as an identifier (see: https://github.com/railslove/cmxl/blob/main/lib/cmxl/statement.rb#L49-L55)
 
   s.transactions.each do |t|
     puts t.information
     puts t.description
     puts t.entry_date
-    puts t.funds_code
+    puts t.credit_debit_indicator
     puts t.credit?
     puts t.debit?
     puts t.sign # -1 if it's a debit; 1 if it's a credit
@@ -81,25 +67,58 @@ statements.each do |s|
     # ...
   end
 end
-
 ```
 
 Every object responds to `to_h` and let's you easily convert the data to a hash. Also every object responds to `to_json` which lets you easily represent the statements as JSON with your favorite JSON library.
 
-#### A note about encoding and file weirdnesses
+### File encoding options
 
 You probably will encounter encoding issues (hey, you are building banking applications!).
 We try to handle encoding and format weirdnesses as much as possible. If no encoding is passed we try to guess the encoding of the data and convert it to UTF8.
 In the likely case that you encounter encoding issues you can pass encoding options to `Cmxl.parse(<string>, <options hash>)`. It accepts the same options as [String#encode](http://ruby-doc.org/core-2.1.3/String.html#method-i-encode)
 If that fails, try to modify the file before you pass it to the parser - and please create an issue.
 
-### MT940 SWIFT header data
+```ruby
+Cmxl.parse(File.read('mt940.txt'), :encoding => 'ISO-8859-1')
+```
+
+## Global configurations:
+The gem offers option to adjust behavior for the gem
+
+### `statement_separator`
+statement divider regex to split the individual statements in one file
+
+|type|default|
+|----|-------|
+|regex|[`/\R+-[^\n\r]*\R*/m`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L18)|
 
+```ruby
+Cmxl.config[:statement_separator] = ...
+Cmxl.parse(...)
+```
+
+### `raise_line_format_errors`
+do you want an error to be raised when a line can not be parsed? 
+
+|type|default|
+|----|-------|
+|boolean|[`true`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L19)|
+
+```ruby
+Cmxl.config[:raise_line_format_errors] = ...
+Cmxl.parse(...)
+```
+
+### `strip_headers`
 Cmxl currently does not support parsing of the SWIFT headers (like {1:F01AXISINBBA ....)
-If your file comes with these headers try the `strip_headers` configuration option to strip data execpt the actual MT940 fields.
+If your file comes with these headers try the `strip_headers` configuration option to strip data except the actual MT940 fields.
+
+|type|default|
+|----|-------|
+|boolean|[`false`](https://github.com/railslove/cmxl/blob/main/lib/cmxl.rb#L20)|
 
 ```ruby
-Cmxl.config[:strip_headers] = true
+Cmxl.config[:strip_headers] = ...
 Cmxl.parse(...)
 ```
 
@@ -187,6 +206,6 @@ other parsers:
 
 ---
 built with love by [Railslove](http://railslove.com) and some [amazing people](https://github.com/railslove/cmxl/graphs/contributors).  
-Released under the MIT-Licence. 
+Released under the MIT-License. 
 
 Railslove builds FinTech products, if you need support for your project we are happy to help. Please contact us at team@railslove.com.

data/cmxl.gemspec

@@ -19,11 +19,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'rchardet'
+
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '~>3.0'
   spec.add_development_dependency 'simplecov'
-
-  spec.add_dependency 'rchardet19'
 end

data/lib/cmxl/field.rb

@@ -13,7 +13,7 @@ module Cmxl
 
     # The parser class variable is the registry of all available parser.
     # It is a hash with the tag (MT940 field number/tag) as key and the class as value
-    # When parsing a statment line we look for a matching entry or use the Unknown class as default
+    # When parsing a statement line we look for a matching entry or use the Unknown class as default
     @@parsers = {}
     @@parsers.default = Unknown
     def self.parsers

data/lib/cmxl/fields/statement_details.rb

@@ -7,7 +7,7 @@ module Cmxl
       class << self
         def parse(line)
           # remove line breaks as they are allowed via documentation but not needed for data-parsing
-          super line.delete("\n")
+          super line.gsub(/\R?/, '')
         end
       end
 

data/lib/cmxl/fields/transaction.rb

@@ -2,7 +2,7 @@ module Cmxl
   module Fields
     class Transaction < Field
       self.tag = 61
-      self.parser = %r{^(?<date>\d{6})(?<entry_date>\d{4})?(?<storno_flag>R?)(?<funds_code>[CD]{1})(?<currency_letter>[a-zA-Z])?(?<amount>\d{1,12},\d{0,2})(?<swift_code>(?:N|F|S).{3})(?<reference>NONREF|(.(?!\/\/)){,16}([^\/]){,1})((?:\/\/)(?<bank_reference>[^\n]{,16}))?((?:\n)(?<supplementary>.{,34}))?$}
+      self.parser = %r{^(?<date>\d{6})(?<entry_date>\d{4})?(?<credit_debit_indicator>D|C|RD|RC|ED|EC)(?<currency_letter>[a-zA-Z])?(?<amount>\d{1,12},\d{0,2})(?<swift_code>(?:N|F|S).{3})(?<reference>NONREF|(.(?!\/\/)){,16}([^\/]){,1})((?:\/\/)(?<bank_reference>[^\n]{,16}))?((?:\n)(?<supplementary>.{,34}))?$}
 
       attr_accessor :details
 
@@ -15,31 +15,63 @@ module Cmxl
       end
 
       def credit?
-        data['funds_code'].to_s.casecmp('C').zero?
+        credit_debit_indicator.include?('C')
       end
 
       def debit?
-        data['funds_code'].to_s.casecmp('D').zero?
+        credit_debit_indicator.include?('D')
       end
 
       def storno_credit?
+        warn "[DEPRECATION] `storno_credit?` is deprecated.  Please use `reversal_credit?` instead. It will be removed in version 3.0."
+        reversal_credit?
+      end
+
+      def reversal_credit?
         credit? && storno?
       end
 
       def storno_debit?
+        warn "[DEPRECATION] `storno_debit?` is deprecated.  Please use `reversal_debit?` instead. It will be removed in version 3.0."
+        reversal_debit?
+      end
+
+      def reversal_debit?
         debit? && storno?
       end
 
       def storno?
-        !storno_flag.empty?
+        warn "[DEPRECATION] `storno?` is deprecated.  Please use `reversal?` instead. It will be removed in version 3.0."
+        reversal?
+      end
+
+      def reversal?
+        credit_debit_indicator.include?('R')
+      end
+
+      def expected_credit?
+        credit? && expected?
+      end
+
+      def expected_debit?
+        debit? && expected?
+      end
+
+      def expected?
+        credit_debit_indicator.include?('E')
+      end
+
+      def credit_debit_indicator
+        data['credit_debit_indicator'].to_s
       end
 
       def funds_code
-        data.values_at('storno_flag', 'funds_code').join
+        warn "[DEPRECATION] `funds_code` is deprecated.  Please use `credit_debit_indicator` instead. It will be removed in version 3.0."
+        data['credit_debit_indicator'].to_s
       end
 
       def storno_flag
-        data['storno_flag']
+        reversal? ? 'R' : ''
       end
 
       def sign
@@ -140,8 +172,11 @@ module Cmxl
           'sign' => sign,
           'debit' => debit?,
           'credit' => credit?,
-          'storno' => storno?,
-          'funds_code' => funds_code,
+          'storno' => reversal?,
+          'reversal' => reversal?,
+          'expected' => expected?,
+          'funds_code' => credit_debit_indicator,
+          'credit_debit_indicator' => credit_debit_indicator,
           'swift_code' => swift_code,
           'reference' => reference,
           'bank_reference' => bank_reference,

data/lib/cmxl/statement.rb

@@ -41,8 +41,7 @@ module Cmxl
     end
 
     def strip_headers!
-      source.gsub!(/\A.+?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
-      source.gsub!(/^[^:]+\z/, '') # end: strip every line in the end that does not start with a :
+      source.gsub!(/\A.*?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
       source.strip!
     end
 

data/lib/cmxl/version.rb

@@ -1,3 +1,3 @@
 module Cmxl
-  VERSION = '1.5.0'.freeze
+  VERSION = '2.2'.freeze
 end

data/lib/cmxl.rb

@@ -1,6 +1,6 @@
 require 'cmxl/version'
 
-require 'rchardet19'
+require 'rchardet'
 
 require 'cmxl/field'
 require 'cmxl/statement'
@@ -11,9 +11,13 @@ module Cmxl
     @config
   end
   @config = {
-    statement_separator: /\n-\s*\n/m,
+    # One or more newlines
+    # followed by `-` at the beginning of a line.
+    # "Eats up" but does not require characters until the end of the line and more newlines.
+    # \R is a platform independent newline but in the negated group `[^\n\r]` that did not seem to work.
+    statement_separator: /\R+-[^\n\r]*\R*/m,
     raise_line_format_errors: true,
-    strip_headers: false
+    strip_headers: true
   }
 
   # Public: Parse a MT940 string
@@ -31,8 +35,8 @@ module Cmxl
   def self.parse(data, options = {})
     options[:statement_separator] ||= config[:statement_separator]
     # if no encoding is provided we try to guess using CharDet
-    if options[:encoding].nil? && cd = CharDet.detect(data, silent: true)
-      options[:encoding] = cd.encoding
+    if options[:encoding].nil? && cd = CharDet.detect(data)
+      options[:encoding] = cd['encoding']
     end
 
     if options[:encoding]

data/spec/fields/statement_details_spec.rb

@@ -75,6 +75,47 @@ describe Cmxl::Fields::StatementDetails do
       )
     }
     it { expect(subject.to_hash).to eql(subject.to_h) }
+
+    context 'with newlines in windows format' do
+      it 'removes any newlines' do
+        data = ":86:171?00SEPA LASTSCHRIFT KUNDE?10281?\r\n20KREF+EREF+TRX-0A4A47C3-F846-4729?21-8A1B-5DF620F?22MREF+CAC97D2144174318AC18D9?23BF815BD4FB?24CRED+DE98ZZZ09999999999?25SVWZ+FOO TRX-0A4A47C3-F84?266-4729-8A1B-5DF620F?30HYVEDEMMXXX?31HUkkbbbsssskcccccccccccccccx?\r\n32Peter Pan?99?34171"
+        result = Cmxl::Fields::StatementDetails.parse(data)
+
+        expect(result.to_h).to eql(
+          'bic' => 'HYVEDEMMXXX',
+          'iban' => 'HUkkbbbsssskcccccccccccccccx',
+          'name' => 'Peter Pan',
+          'sepa' => {
+            'KREF' => '',
+            'EREF' => 'TRX-0A4A47C3-F846-4729-8A1B-5DF620F',
+            'MREF' => 'CAC97D2144174318AC18D9BF815BD4FB',
+            'CRED' => 'DE98ZZZ09999999999',
+            'SVWZ' => 'FOO TRX-0A4A47C3-F846-4729-8A1B-5DF620F'
+          },
+          'information' => 'KREF+EREF+TRX-0A4A47C3-F846-4729-8A1B-5DF620FMREF+CAC97D2144174318AC18D9BF815BD4FBCRED+DE98ZZZ09999999999SVWZ+FOO TRX-0A4A47C3-F846-4729-8A1B-5DF620F',
+          'description' => 'SEPA LASTSCHRIFT KUNDE',
+          'sub_fields' => {
+            '00' => 'SEPA LASTSCHRIFT KUNDE',
+            '10' => '281',
+            '20' => 'KREF+EREF+TRX-0A4A47C3-F846-4729',
+            '21' => '-8A1B-5DF620F',
+            '22' => 'MREF+CAC97D2144174318AC18D9',
+            '23' => 'BF815BD4FB',
+            '24' => 'CRED+DE98ZZZ09999999999',
+            '25' => 'SVWZ+FOO TRX-0A4A47C3-F84',
+            '26' => '6-4729-8A1B-5DF620F',
+            '30' => 'HYVEDEMMXXX',
+            '31' => 'HUkkbbbsssskcccccccccccccccx',
+            '32' => 'Peter Pan',
+            '34' => '171',
+            '99' => ''
+          },
+          'transaction_code' => '171',
+          'primanota' => '281',
+          'details' => '?00SEPA LASTSCHRIFT KUNDE?10281?20KREF+EREF+TRX-0A4A47C3-F846-4729?21-8A1B-5DF620F?22MREF+CAC97D2144174318AC18D9?23BF815BD4FB?24CRED+DE98ZZZ09999999999?25SVWZ+FOO TRX-0A4A47C3-F84?266-4729-8A1B-5DF620F?30HYVEDEMMXXX?31HUkkbbbsssskcccccccccccccccx?32Peter Pan?99?34171'
+        )
+      end
+    end
   end
 
   describe 'information parsing with empty fields on the end' do

data/spec/fields/transaction_spec.rb

@@ -7,7 +7,7 @@ describe Cmxl::Fields::Transaction do
   subject(:ocmt_cghs_transaction) { Cmxl::Fields::Transaction.parse(fixture_line(:statement_ocmt_chgs)) }
   subject(:supplementary_transaction) { Cmxl::Fields::Transaction.parse(fixture_line(:statement_supplementary_plain)) }
   subject(:complex_supplementary_transaction) { Cmxl::Fields::Transaction.parse(fixture_line(:statement_supplementary_complex)) }
-  subject(:valuta_after_enty_date) { Cmxl::Fields::Transaction.parse(fixture[3]) }
+  subject(:valuta_after_entry_date) { Cmxl::Fields::Transaction.parse(fixture[3]) }
   subject(:entry_before_valuta_transaction) { Cmxl::Fields::Transaction.parse(fixture[2]) }
   subject(:transaction_type_swift) { Cmxl::Fields::Transaction.parse(fixture[4]) }
 
@@ -70,6 +70,11 @@ describe Cmxl::Fields::Transaction do
   end
 
   context 'statement with complex supplementary' do
+    it 'future reference' do
+      result = Cmxl::Fields::Transaction.parse(':61:2412121212ED162,57NDDTNONREF//950\n')
+      expect(result.amount).to eql(162.57)
+    end
+
     it { expect(complex_supplementary_transaction.initial_amount_in_cents).to eql(nil) }
     it { expect(complex_supplementary_transaction.initial_currency).to eql(nil) }
 
@@ -82,14 +87,14 @@ describe Cmxl::Fields::Transaction do
   end
 
   context 'valuta and entry-date assumptions' do
-    it 'entry_date before valuta is recognized correclty when including year-change' do
+    it 'entry_date before valuta is recognized correctly when including year-change' do
       expect(entry_before_valuta_transaction.date).to eql(Date.new(2014, 1, 10))
       expect(entry_before_valuta_transaction.entry_date).to eql(Date.new(2013, 12, 24))
     end
 
     it 'entry_date after valuta is recognized correctly when including year-change' do
-      expect(valuta_after_enty_date.date).to eql(Date.new(2014, 12, 24))
-      expect(valuta_after_enty_date.entry_date).to eql(Date.new(2015, 1, 2))
+      expect(valuta_after_entry_date.date).to eql(Date.new(2014, 12, 24))
+      expect(valuta_after_entry_date.entry_date).to eql(Date.new(2015, 1, 2))
     end
   end
 
@@ -107,4 +112,77 @@ describe Cmxl::Fields::Transaction do
     it { expect(transaction_type_swift).not_to be_storno }
     it { expect(transaction_type_swift.sign).to eql(1) }
   end
+
+  describe '#credit_debit_indicator' do
+    it 'returns the credit_debit_indicator as debit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902DR000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result.credit_debit_indicator).to eql('D')
+    end
+
+    it 'returns the credit_debit_indicator as credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902CR000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result.credit_debit_indicator).to eql('C')
+    end
+
+    it 'returns the credit_debit_indicator as reversal credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902RC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result.credit_debit_indicator).to eql('RC')
+    end
+
+    it 'returns the credit_debit_indicator as reversal debit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902RD000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result.credit_debit_indicator).to eql('RD')
+    end
+
+    it 'returns the credit_debit_indicator as expected credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902EC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result.credit_debit_indicator).to eql('EC')
+    end
+  end
+
+  describe '#expected?' do
+    it 'returns true if the transaction is expected' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902EC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).to be_expected
+    end
+
+    it 'returns false if the transaction is not expected' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902RD000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).not_to be_expected
+    end
+  end
+
+  describe '#expected_credit?' do
+    it 'returns true if the transaction is expected and credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902EC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).to be_expected_credit
+    end
+
+    it 'returns false if the transaction is not expected and credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902RC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).not_to be_expected_credit
+    end
+
+    it 'returns false if the transaction is expected and debit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902ED000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).not_to be_expected_credit
+    end
+  end
+
+  describe '#expected_debit?' do
+    it 'returns true if the transaction is expected and debit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902ED000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).to be_expected_debit
+    end
+
+    it 'returns false if the transaction is not expected and debit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902RD000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).not_to be_expected_debit
+    end
+
+    it 'returns false if the transaction is expected and credit' do
+      result = Cmxl::Fields::Transaction.parse(':61:1409010902EC000000000001,62NTRF0000549855700010//025498557/000001')
+      expect(result).not_to be_expected_debit
+    end
+  end
 end

data/spec/fixtures/mt940-abnamro.txt

@@ -0,0 +1,44 @@
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19321/1
+:60F:C110522EUR3236,28
+:61:1105240524D9,N192NONREF
+:86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+INCL. 1,44 BTW
+:61:1105210523D11,59N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+:61:1105230523D11,63N426NONREF
+:86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+:61:1105220523D11,8N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+:61:1105210523D13,45N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+:61:1105210523D15,49N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+:61:1105210523D107,N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+:61:1105220523D141,48N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110523EUR876,84
+
+-
+
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19322/1
+:60F:C110523EUR2876,84
+:61:1105240524D9,49N426NONREF
+:86:BEA   NR:XXX1234   24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999
+:61:1105240524D15,N426NONREF
+:86:52.89.39.882 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110524EUR1849,75
+
+-

data/spec/fixtures/mt940-handelsbank.txt

@@ -0,0 +1,16 @@
+ä4:
+:20:5566778899100112
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
+ä4:
+:20:5566778899100169
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å

data/spec/fixtures/mt940-windows-line-breaks.txt

@@ -0,0 +1,16 @@
+{1:D02AASDISLNETAXXXXXXXXXXXXX}
+{2:E623XXXXXXXXAXXXN}
+{4:
+:20:1234567
+:21:9876543210
+:25:10020030/1234567
+:28C:5/1
+:60F:C160314EUR2187,95
+:61:0211011102DR800,NSTONONREF//55555
+:86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339
+:61:0211021102CR3000,NTRFNONREF//55555
+:86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma
+Mustermann GmbH?3050060400?310847564700?32MUELLER?34339
+:62F:C160315EUR4387,95
+:86:Some random data
+-}

data/spec/mt940_parsing_spec.rb

@@ -95,4 +95,115 @@ describe 'parsing a statement' do
       expect(subject[0].transactions.count).to eql(1)
     end
   end
+
+  describe "MT940 abnamro" do
+    it "splits the file into on statement with the headers included" do
+      expected_data = <<~MT940.chomp
+        ABNANL2A
+        940
+        ABNANL2A
+        :20:ABN AMRO BANK NV
+        :25:517852257
+        :28:19321/1
+        :60F:C110522EUR3236,28
+        :61:1105240524D9,N192NONREF
+        :86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+        5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+        INCL. 1,44 BTW
+        :61:1105210523D11,59N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+        :61:1105230523D11,63N426NONREF
+        :86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+        :61:1105220523D11,8N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+        :61:1105210523D13,45N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+        :61:1105210523D15,49N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+        :61:1105210523D107,N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+        :61:1105220523D141,48N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+        :62F:C110523EUR876,84
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe 'MT940 handelsbank' do
+    it 'splits the file with the special characters correctly' do
+      expected_data = <<~MT940.chomp
+        채4:
+        :20:5566778899100112
+        :25:10020030/1234567
+        :28C:188/1
+        :60F:C130928SEK0,
+        :62F:C130930SEK0,
+        :64:C130930SEK0,
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe "MT940 windows line breaks" do
+    it 'splits the file with the special characters correctly' do
+      expected_data =
+        "{1:D02AASDISLNETAXXXXXXXXXXXXX}\r\n"\
+        "{2:E623XXXXXXXXAXXXN}\r\n"\
+        "{4:\r\n"\
+        ":20:1234567\r\n"\
+        ":21:9876543210\r\n"\
+        ":25:10020030/1234567\r\n"\
+        ":28C:5/1\r\n"\
+        ":60F:C160314EUR2187,95\r\n"\
+        ":61:0211011102DR800,NSTONONREF//55555\r\n"\
+        ":86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339\r\n"\
+        ":61:0211021102CR3000,NTRFNONREF//55555\r\n"\
+        ":86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma\r\n"\
+        "Mustermann GmbH?3050060400?310847564700?32MUELLER?34339\r\n"\
+        ":62F:C160315EUR4387,95\r\n"\
+        ":86:Some random data"
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).once
+    end
+  end
 end

data/spec/statement_spec.rb

@@ -27,9 +27,12 @@ describe Cmxl do
           'bank_reference' => '025498557/000001',
           'amount_in_cents' => 162,
           'sign' => -1,
+          'credit_debit_indicator' => 'D',
           'debit' => true,
           'credit' => false,
           'storno' => false,
+          'reversal' => false,
+          'expected' => false,
           'bic' => 'HYVEDEMMXXX',
           'iban' => 'HUkkbbbsssskcccccccccccccccx',
           'name' => 'Peter Pan',
@@ -81,6 +84,7 @@ describe Cmxl do
           'sha' => '3c5e65aa3d3878b06b58b6f1ae2f3693004dfb04e3ab7119a1c1244e612293da',
           'entry_date' => Date.new(2014, 9, 2),
           'funds_code' => 'D',
+          'credit_debit_indicator' => 'D',
           'currency_letter' => 'R',
           'amount' => 1.62,
           'swift_code' => 'NTRF',
@@ -90,12 +94,14 @@ describe Cmxl do
           'sign' => -1,
           'debit' => true,
           'credit' => false,
-          'storno' => false
+          'storno' => false,
+          'reversal' => false,
+          'expected' => false,
         )
       end
     end
 
-    describe 'statement issued over a years boudary' do
+    describe 'statement issued over a years boundary' do
       subject { Cmxl.parse(mt940_file('statement-mt940')).first.transactions.last }
 
       it { expect(subject.mt942?).to be_falsey }
@@ -161,4 +167,95 @@ describe Cmxl do
     it { expect(statement.generation_date).to eql(Date.new(2019, 1, 9)) }
   end
 
+  describe "header parsing" do
+    context "when strip_headers is enabled" do
+      around do |example|
+        existing_value = Cmxl.config[:strip_headers]
+        Cmxl.config[:strip_headers] = true
+        example.run
+        Cmxl.config[:strip_headers] = existing_value
+      end
+
+      it "removes any headers" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+
+      it "does nothing if there are no headers" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+
+    context "when strip_headers is disabled" do
+      around do |example|
+        existing_value = Cmxl.config[:strip_headers]
+        Cmxl.config[:strip_headers] = false
+        example.run
+        Cmxl.config[:strip_headers] = existing_value
+      end
+
+      it "raise an parsing error exception if headers are present" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        expect{
+          Cmxl::Statement.new(data)
+        }.to raise_error(Cmxl::Field::LineFormatError)
+      end
+
+      it "extracts the field" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+  end
 end

metadata

@@ -1,15 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: cmxl
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: '2.2'
 platform: ruby
 authors:
 - Michael Bumann
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-10 00:00:00.000000000 Z
+date: 2025-02-04 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: rchardet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,20 +93,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rchardet19
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Cmxl provides an friendly, extensible and customizable parser for the
   MT940 bank statement format.
 email:
@@ -102,9 +101,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
+- ".github/workflows/release.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CHANGELOG.mdown
 - Gemfile
 - LICENSE.txt
@@ -163,9 +163,12 @@ files:
 - spec/fixtures/lines/statement_supplementary_plain.txt
 - spec/fixtures/lines/vmk_summary_credit.txt
 - spec/fixtures/lines/vmk_summary_debit.txt
+- spec/fixtures/mt940-abnamro.txt
 - spec/fixtures/mt940-deutsche_bank.txt
+- spec/fixtures/mt940-handelsbank.txt
 - spec/fixtures/mt940-headers.txt
 - spec/fixtures/mt940-iso8859-1.txt
+- spec/fixtures/mt940-windows-line-breaks.txt
 - spec/fixtures/mt940-with-colon-after-line-break.txt
 - spec/fixtures/mt940-with-detailed-end-balance.txt
 - spec/fixtures/mt940.txt
@@ -198,8 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Cmxl is your friendly MT940 bank statement parser
 test_files:
@@ -239,9 +241,12 @@ test_files:
 - spec/fixtures/lines/statement_supplementary_plain.txt
 - spec/fixtures/lines/vmk_summary_credit.txt
 - spec/fixtures/lines/vmk_summary_debit.txt
+- spec/fixtures/mt940-abnamro.txt
 - spec/fixtures/mt940-deutsche_bank.txt
+- spec/fixtures/mt940-handelsbank.txt
 - spec/fixtures/mt940-headers.txt
 - spec/fixtures/mt940-iso8859-1.txt
+- spec/fixtures/mt940-windows-line-breaks.txt
 - spec/fixtures/mt940-with-colon-after-line-break.txt
 - spec/fixtures/mt940-with-detailed-end-balance.txt
 - spec/fixtures/mt940.txt

data/.travis.yml

@@ -1,9 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.4.10
-  - 2.5.8
-  - 2.6.6
-  - 2.7.1
-  - jruby-9.1.17.0
-  - jruby-9.2.11.1