cmxl 2.0 → 2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c172d6b56767a8860f57c4b802c7ed2ed2bae3658f6db104f44795a377bf067
-  data.tar.gz: ececb262e0d6a40c97e7ca54dcac08208e1942792c5f493be179ba3c81ece6b7
+  metadata.gz: c6114278dc458c8f3890f6f04e72a33915c98f235fe0694ba8d7b02cc5f394d6
+  data.tar.gz: d47ca765f53acf8f86d9c51b25123975174fc83c86cc68b834f069606ecd8cd4
 SHA512:
-  metadata.gz: f0f673453d6bbbc39313ed7ae86ff574c4c9d60a0ea5543283f38d1f3bf4f7477625e2bc87a1fbdc62714c86461735a7e16796edd54cd361d52d49d2eb05d8db
-  data.tar.gz: a197094a3ccd4235f8811a456dbd54c43bc3b49a47062a72f7b9ee53610ddbb4e79ce97e53e0f90670a0c5e3cb5f08ef490792506dc6cc1202d5442c4f724710
+  metadata.gz: 9ac2f348609babd4cca9df8da0412f965057fed77d210e155f076c489d407541effd6da7012b00ff48f642baec2ae96b028bad0720f08ff50c4f2327d34dc7bf
+  data.tar.gz: cfae7ffcf5ffe4ee50b2afac7828f712a9511e5e1da6493a17a7d94681094fee7729b2687ee5a9fdc9bad7e031778841ed8cda3db44c6741a6810c54f736d25a

data/.github/workflows/ci.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   test:
@@ -13,6 +13,7 @@ jobs:
     strategy:
       matrix:
         ruby-version:
+          - 3.4
           - 3.3
           - 3.2
           - 3.1

data/.rspec

@@ -1 +1,2 @@
---color
+--color
+--require 'pry'

data/CHANGELOG.mdown

@@ -1,5 +1,11 @@
 # NEXT release
 
+# 2.1
+- `[REFACTOR]` improve file parser to work with windows line breaks and any known header formats
+- `[BUGFIX]` fix `strip_header` making it work on statements without headers
+- `[HOUSEKEEPING]` use `main` instead of `master` as default branch
+- `[ENHANCEMENT]` add support for ruby `3.4`
+
 # 2.0
 - `[REFACTOR]` **DEPRECATED:** `storno?` and related methods `storno_credit`, `storno_debit`. Use `reversal?` and related methods `reversal_credit?`, `reversal_debit?` instead
 - `[BUGFIX]` **DEPRECATED:** `funds_code` method returns the `credit_debit_indicator` from the SWIFT definition. Therefore the method is deprecated in favor of `credit_debit_indicator` method

data/README.md

@@ -62,7 +62,7 @@ statements.each do |s|
   puts s.generation_date
   puts s.opening_balance.amount
   puts s.closing_balance.amount
-  puts s.sha # SHA of the statement source - could be used as an identifier (see: https://github.com/railslove/cmxl/blob/master/lib/cmxl/statement.rb#L49-L55)
+  puts s.sha # SHA of the statement source - could be used as an identifier (see: https://github.com/railslove/cmxl/blob/main/lib/cmxl/statement.rb#L49-L55)
 
   s.transactions.each do |t|
     puts t.information

data/lib/cmxl/statement.rb

@@ -41,8 +41,8 @@ module Cmxl
     end
 
     def strip_headers!
-      source.gsub!(/\A.+?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
-      source.gsub!(/^[^:]+\z/, '') # end: strip every line in the end that does not start with a :
+      source.gsub!(/\A.*?(?=^:)/m, '') # beginning: strip every line in the beginning that does not start with a :
+      source.gsub!(/^[^:]*\z/, '') # end: strip every line in the end that does not start with a :
       source.strip!
     end
 

data/lib/cmxl/version.rb

@@ -1,3 +1,3 @@
 module Cmxl
-  VERSION = '2.0'.freeze
+  VERSION = '2.1'.freeze
 end

data/lib/cmxl.rb

@@ -11,7 +11,11 @@ module Cmxl
     @config
   end
   @config = {
-    statement_separator: /\n-\s*\n/m,
+    # One or more newlines
+    # followed by `-` at the beginning of a line.
+    # "Eats up" but does not require characters until the end of the line and more newlines.
+    # \R is a platform independent newline but in the negated group `[^\n\r]` that did not seem to work.
+    statement_separator: /\R+-[^\n\r]*\R*/m,
     raise_line_format_errors: true,
     strip_headers: false
   }

data/spec/fixtures/mt940-abnamro.txt

@@ -0,0 +1,44 @@
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19321/1
+:60F:C110522EUR3236,28
+:61:1105240524D9,N192NONREF
+:86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+INCL. 1,44 BTW
+:61:1105210523D11,59N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+:61:1105230523D11,63N426NONREF
+:86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+:61:1105220523D11,8N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+:61:1105210523D13,45N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+:61:1105210523D15,49N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+:61:1105210523D107,N426NONREF
+:86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+:61:1105220523D141,48N426NONREF
+:86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110523EUR876,84
+
+-
+
+ABNANL2A
+940
+ABNANL2A
+:20:ABN AMRO BANK NV
+:25:517852257
+:28:19322/1
+:60F:C110523EUR2876,84
+:61:1105240524D9,49N426NONREF
+:86:BEA   NR:XXX1234   24.05.11/09.18 PETS PLACE KATWIJK KATWI,PAS999
+:61:1105240524D15,N426NONREF
+:86:52.89.39.882 MYCOM DEN HAAG  S-GRAVEN,PAS999
+:62F:C110524EUR1849,75
+
+-

data/spec/fixtures/mt940-handelsbank.txt

@@ -0,0 +1,16 @@
+ä4:
+:20:5566778899100112
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å
+ä4:
+:20:5566778899100169
+:25:10020030/1234567
+:28C:188/1
+:60F:C130928SEK0,
+:62F:C130930SEK0,
+:64:C130930SEK0,
+-å

data/spec/fixtures/mt940-windows-line-breaks.txt

@@ -0,0 +1,16 @@
+{1:D02AASDISLNETAXXXXXXXXXXXXX}
+{2:E623XXXXXXXXAXXXN}
+{4:
+:20:1234567
+:21:9876543210
+:25:10020030/1234567
+:28C:5/1
+:60F:C160314EUR2187,95
+:61:0211011102DR800,NSTONONREF//55555
+:86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339
+:61:0211021102CR3000,NTRFNONREF//55555
+:86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma
+Mustermann GmbH?3050060400?310847564700?32MUELLER?34339
+:62F:C160315EUR4387,95
+:86:Some random data
+-}

data/spec/mt940_parsing_spec.rb

@@ -95,4 +95,115 @@ describe 'parsing a statement' do
       expect(subject[0].transactions.count).to eql(1)
     end
   end
+
+  describe "MT940 abnamro" do
+    it "splits the file into on statement with the headers included" do
+      expected_data = <<~MT940.chomp
+        ABNANL2A
+        940
+        ABNANL2A
+        :20:ABN AMRO BANK NV
+        :25:517852257
+        :28:19321/1
+        :60F:C110522EUR3236,28
+        :61:1105240524D9,N192NONREF
+        :86:GIRO   428428 KPN - DIGITENNE    BETALINGSKENM.  000000042188659
+        5314606715                       BETREFT FACTUUR D.D. 20-05-2011
+        INCL. 1,44 BTW
+        :61:1105210523D11,59N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.54 DIRCKIII FIL2500 KATWIJK,PAS999
+        :61:1105230523D11,63N426NONREF
+        :86:BEA   NR:XXX1234   23.05.11/09.08 DIGROS FIL1015 KATWIJK Z,PAS999
+        :61:1105220523D11,8N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/14.25 MC DONALDS A44 LEIDEN,PAS999
+        :61:1105210523D13,45N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.09 PRINCE FIL. 55 KATWIJK Z,PAS999
+        :61:1105210523D15,49N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.55 DIRX FIL6017 KATWIJK ZH ,PAS999
+
+        :61:1105210523D107,N426NONREF
+        :86:BEA   NR:XXX1234   21.05.11/12.04 HANS ANDERS OPT./056 KAT,PAS999
+        :61:1105220523D141,48N426NONREF
+        :86:BEA   NR:XXX1234   22.05.11/13.45 MYCOM DEN HAAG  S-GRAVEN,PAS999
+        :62F:C110523EUR876,84
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-abnamro'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe 'MT940 handelsbank' do
+    it 'splits the file with the special characters correctly' do
+      expected_data = <<~MT940.chomp
+        채4:
+        :20:5566778899100112
+        :25:10020030/1234567
+        :28C:188/1
+        :60F:C130928SEK0,
+        :62F:C130930SEK0,
+        :64:C130930SEK0,
+      MT940
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-handelsbank'))
+
+      expect(Cmxl::Statement).to have_received(:new).twice
+    end
+  end
+
+  describe "MT940 windows line breaks" do
+    it 'splits the file with the special characters correctly' do
+      expected_data =
+        "{1:D02AASDISLNETAXXXXXXXXXXXXX}\r\n"\
+        "{2:E623XXXXXXXXAXXXN}\r\n"\
+        "{4:\r\n"\
+        ":20:1234567\r\n"\
+        ":21:9876543210\r\n"\
+        ":25:10020030/1234567\r\n"\
+        ":28C:5/1\r\n"\
+        ":60F:C160314EUR2187,95\r\n"\
+        ":61:0211011102DR800,NSTONONREF//55555\r\n"\
+        ":86:008?00DAUERAUFTRAG?100599?20Miete November?3010020030?31234567?32MUELLER?34339\r\n"\
+        ":61:0211021102CR3000,NTRFNONREF//55555\r\n"\
+        ":86:051?00UEBERWEISUNG?100599?20Gehalt Oktober?21Firma\r\n"\
+        "Mustermann GmbH?3050060400?310847564700?32MUELLER?34339\r\n"\
+        ":62F:C160315EUR4387,95\r\n"\
+        ":86:Some random data"
+      stub = instance_double(Cmxl::Statement)
+      allow(Cmxl::Statement).to receive(:new).and_return(stub)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).with(expected_data)
+    end
+
+    it 'splits the file into two statements' do
+      allow(Cmxl::Statement).to receive(:new)
+
+      Cmxl.parse(mt940_file('mt940-windows-line-breaks'))
+
+      expect(Cmxl::Statement).to have_received(:new).once
+    end
+  end
 end

data/spec/statement_spec.rb

@@ -101,7 +101,7 @@ describe Cmxl do
       end
     end
 
-    describe 'statement issued over a years boudary' do
+    describe 'statement issued over a years boundary' do
       subject { Cmxl.parse(mt940_file('statement-mt940')).first.transactions.last }
 
       it { expect(subject.mt942?).to be_falsey }
@@ -167,4 +167,88 @@ describe Cmxl do
     it { expect(statement.generation_date).to eql(Date.new(2019, 1, 9)) }
   end
 
+  describe "header parsing" do
+    context "when strip_headers is enabled" do
+      around do |example|
+        existing_value = Cmxl.config[:strip_headers]
+        Cmxl.config[:strip_headers] = true
+        example.run
+        Cmxl.config[:strip_headers] = existing_value
+      end
+
+      it "removes any headers" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+
+      it "does nothing if there are no headers" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+
+    context "when strip_headers is disabled" do
+      it "raise an parsing error exception if headers are present" do
+        data = <<~MT940.chomp
+          {1:D02AASDISLNETAXXXXXXXXXXXXX}
+          {2:E623XXXXXXXXAXXXN}
+          {4:
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+          -}
+        MT940
+
+        expect{
+          Cmxl::Statement.new(data)
+        }.to raise_error(Cmxl::Field::LineFormatError)
+      end
+
+      it "extracts the field" do
+        data = <<~MT940.chomp
+          :20:MT940/78374
+          :25:xxxxxxxxxxxxxx
+          :28C:3/1
+          :60F:C160201INR0,00
+          :61:3622687806CR1368378,92NMSC37935531
+          :86:-TX TRN-REF NO.1156ADS5601187 EUR 13456/TSV
+          :62F:C141387INR11 27421,94
+        MT940
+
+        result = Cmxl::Statement.new(data)
+
+        expect(result.fields.count).to eq(6)
+      end
+    end
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: cmxl
 version: !ruby/object:Gem::Version
-  version: '2.0'
+  version: '2.1'
 platform: ruby
 authors:
 - Michael Bumann
 bindir: bin
 cert_chain: []
-date: 2025-01-10 00:00:00.000000000 Z
+date: 2025-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rchardet
@@ -163,9 +163,12 @@ files:
 - spec/fixtures/lines/statement_supplementary_plain.txt
 - spec/fixtures/lines/vmk_summary_credit.txt
 - spec/fixtures/lines/vmk_summary_debit.txt
+- spec/fixtures/mt940-abnamro.txt
 - spec/fixtures/mt940-deutsche_bank.txt
+- spec/fixtures/mt940-handelsbank.txt
 - spec/fixtures/mt940-headers.txt
 - spec/fixtures/mt940-iso8859-1.txt
+- spec/fixtures/mt940-windows-line-breaks.txt
 - spec/fixtures/mt940-with-colon-after-line-break.txt
 - spec/fixtures/mt940-with-detailed-end-balance.txt
 - spec/fixtures/mt940.txt
@@ -238,9 +241,12 @@ test_files:
 - spec/fixtures/lines/statement_supplementary_plain.txt
 - spec/fixtures/lines/vmk_summary_credit.txt
 - spec/fixtures/lines/vmk_summary_debit.txt
+- spec/fixtures/mt940-abnamro.txt
 - spec/fixtures/mt940-deutsche_bank.txt
+- spec/fixtures/mt940-handelsbank.txt
 - spec/fixtures/mt940-headers.txt
 - spec/fixtures/mt940-iso8859-1.txt
+- spec/fixtures/mt940-windows-line-breaks.txt
 - spec/fixtures/mt940-with-colon-after-line-break.txt
 - spec/fixtures/mt940-with-detailed-end-balance.txt
 - spec/fixtures/mt940.txt