hsbc_pdf_statement_parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d9737c59472032f49b8bc7539d08856feb6e5703a5a43966b8a80340566e8de6
+  data.tar.gz: 989e26dddd81066f6b2d931067acc47e6b23d60adf35daef7a30bdf27830317e
+SHA512:
+  metadata.gz: ec9375eeee5b241bfc50fa13a1f39b99587e25fde0c5bde829bcd97d166124727837f788686218b5e46ad3aaaa54dc1707733f6902bd9e400447f93a35eaadf6
+  data.tar.gz: 3db670821e2d4b6868ace32aefe4688645e4766d4e68a28c3c5d64e320fa7d68764799101c050ab1b9b1c2ca2bab907db6c9e6cc950ae3789edc09fbfb0cff65

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in hsbc-pdf-statement-parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Jon Pearse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.textile

@@ -0,0 +1,62 @@
+h1. HSBC PDF Statement Parser
+
+This is a _very_ quick and dirty gem that swallows downloaded PDF files from HSBC (UK) and parses them into an array of hashes containing each transaction.
+
+It exists soley because HSBC doesn’t seem to offer any way of exporting old statements as anything other than PDFs, which makes it a pain in the backside to import anything into any kind of finance packages.
+You probably shouldn’t use it (see warnings below)
+
+h2. Installation
+
+Using bundler on the command line:
+
+<pre>
+$ bundle add hsbc_pdf_statement_parser
+$ bundle
+</pre>
+
+h2. Usage
+
+<pre>
+require 'hsbc_pdf_statement_parser'
+
+parser = HsbcPdfStatementParser::Parser.new( 'path/to/statement.pdf' )
+parser.transactions.each do |tx|
+  
+  printf( "%s: %-40s %7.02f\n", tx[:date], tx[:details].lines.first.strip, tx[:change] )
+
+end
+</pre>
+
+h3. Methods
+
+- @statements@ := returns transactions as an array of hashes (see hash keys section, below) _(Array Hash)_
+- @opening_balance@ := returns the opening balance of the statement _(Float)_
+- @closing_balance@ := returns the closing balance of the statement _(Float)_
+- @payments_in@ := returns the total of all payments in _(Float)_
+- @payments_out@ := returns the total of all payments out _(Float)_
+
+Note that @payments_in@ and @payments_out@ are read from the header section at the top of the statement, and are not calculated from the transactions in the statement.
+
+h3. Transaction hash keys
+
+- @:date@ := a `Date` object describing the date of the transaction _(Date)_
+- @:type@ := the type of the description, eg `DD` for a direct debit, `VIS` for visa, `(((` for contactless _(String)_
+- @:details@ := the text descrption of the transaction. This may span multiple lines _(String)_
+- @:in@ := the amount entering your account, or nil if an outbound transaction _(Float, nil)_
+- @:out@ := the amount leaving your account, or nil if an inbound transaction _(Float, nil)_
+- @:change@ := a calculated field showing the change to your bank balance: negative for debits, positive for credits _(Float)_
+- @:balance@ := the balance of your account after the transaction, if present in the PDF _(Float, nil)_
+
+*Note:* that the @:balance@ key is pulled straight from the PDF and will only be present for the last transaction on a particular day. I’m not doing anything even remotely clever here :)
+
+h2. ⚠️ Warnings
+
+This gem has been thrown together for my own needs, and is offered to the world just in case someone else might want to play around with it. 
+It seems to work pretty well with statements from my Advance account here in the UK, and may also work with other flavours of accounts from elsewhere in the world, but comes with absolutely zero guarantees or assurances.
+
+That is to say: it seems to work OK for mucking around, but I’d recommend not using it for anything mission-critical, or in a situation that might lead you or others into making any kind of financial decisions.
+Any dumb financial decisions made are entirely on you =)
+
+Also, this gem contains a patch for "pdf-reader":https://github.com/yob/pdf-reader to help it better cope with the weird way in which HSBC seems to generate PDFs. This is unapologetically a massive hack, and really could do with someone far smarter than me to come up with a better solution.
+For more information, see "pdf-reader issue #169":https://github.com/yob/pdf-reader/issues/169 which goes into a little more detail about what’s going on (my files seem to terminate the image data with @0xE0@, per my patch)
+

data/hsbc_pdf_statement_parser.gemspec

@@ -0,0 +1,20 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "hsbc_pdf_statement_parser/version"
+
+Gem::Specification.new do |spec|
+  
+  spec.name     = 'hsbc_pdf_statement_parser'
+  spec.summary  = 'Quick and dirty RubyGem to parse HSBC’s statement PDFs'
+  spec.license  = 'MIT'
+  
+  spec.authors  = 'Jon Pearse'
+  spec.email    = 'hello@jonpearse.net'
+  spec.homepage = 'https://github.com/jonpearse/hsbc-pdf-statement-parser'
+  
+  spec.version  = HsbcPdfStatementParser::VERSION
+  spec.files    = `git ls-files`.split($\)
+  
+  spec.add_dependency 'pdf-reader', '~> 2.4'
+  
+end

data/lib/hsbc_pdf_statement_parser.rb

@@ -0,0 +1,7 @@
+require 'date'
+require 'pdf-reader'
+require 'hsbc_pdf_statement_parser/pdf_reader_patch'
+require 'hsbc_pdf_statement_parser/parser'
+
+module HsbcPdfStatementParser
+end

data/lib/hsbc_pdf_statement_parser/parser.rb

@@ -0,0 +1,162 @@
+class HsbcPdfStatementParser::Parser 
+  
+  # Creates a new parser from a PDF file.
+  #
+  # === Parameters
+  #
+  # [filename] the filename to parse
+  def initialize( filename )
+    
+    @reader = PDF::Reader.new( filename )
+    
+  end
+  
+  # Returns an array of the transactions in the document as hashes.
+  #
+  # === Hash keys
+  #
+  # [:date] the date of the transaction _(Date)_
+  # [:type] the type of the transaction, eg ‘VISA’, ‘DD’, ‘ATM’, etc _(String)_
+  # [:details] the details of the transaction. This can span multiple lines _(String)_
+  # [:out] the amount of the transaction, if a debit _(Float, nil)_
+  # [:in] the amount of the transaction, if a credit _(Float, nil)_
+  # [:change] the amount of the transacation: negative if a debit, positive if a credit _(Float)_
+  def transactions
+    
+    @_transactions ||= begin
+      
+      current_transaction = nil
+      current_date = nil
+      transactions = []
+      
+      document_text
+        .scan( /BALANCE\s?BROUGHT\s?FORWARD(?:.*?)\n(.*?)BALANCE\s?CARRIED\s?FORWARD/im )
+        .map{ |text| parse_page( text[0] )}
+        .flatten
+        .each do |line|
+        
+          # store the current date
+          current_date = line[:date] unless line[:date].nil?
+        
+          # if we have a type, start a new transaction
+          unless line[:type].nil?
+            transactions << current_transaction unless current_transaction.nil?
+            current_transaction = line.merge( date: current_date )
+            next
+          end
+        
+          # merge things in
+          current_transaction.merge!( line.select{ |k,v| v }, { details: "#{current_transaction[:details]}\n#{line[:details]}" })
+        
+        end
+      
+      # dump the final transaction + return
+      transactions << current_transaction unless current_transaction.nil?
+      transactions
+    end
+    
+  end
+  
+  # Returns the opening balance of the statement read from the table on the first page.
+  def opening_balance
+    
+    @_opening_balance ||= scan_figure( 'Opening Balance' )
+    
+  end
+  
+  # Returns the closing balance of the statement read from the table on the first page.
+  def closing_balance
+    
+    @_closing_balance ||= scan_figure( 'Closing Balance' )
+    
+  end
+  
+  # Returns the total value of payments in during the statement read from the table on the first page (ie: not calculated)
+  def payments_in
+    
+    @_payments_in ||= scan_figure( 'Payments In' )
+    
+  end
+  
+  # Returns the total value of payments out during the statement read from the table on the first page (ie: not calculated)
+  def payments_out
+    
+    @_payments_out ||= scan_figure( 'Payments Out' )
+    
+  end
+  
+  private
+  
+  def document_text
+    
+    @text ||= begin
+      
+      @reader.pages.map( &:text ).join
+      
+    end
+    
+  end
+  
+  def scan_figure( search_string )
+    
+    @_first_page ||= @reader.pages.first.text
+    
+    match = Regexp.new( "#{search_string}(?:.*?)([0-9\.\,]{4,})", Regexp::IGNORECASE ).match( @_first_page )
+    return nil if match.nil?
+    
+    match[1].gsub( ',', '' ).to_f
+    
+  end
+  
+  def parse_page( page_str )
+    
+    # grab lines + get the longest
+    lines = page_str.lines
+    max_length = lines.map( &:length ).max
+    
+    lines.map{ |line| parse_line( line.rstrip.ljust( max_length ))}.compact
+      
+  end
+  
+  def parse_line( line_str )
+    
+    # if we’re blank…
+    return nil if line_str.strip.empty?
+    
+    # start cutting things up
+    row = {
+      date:    empty_to_nil( line_str[0..12] ),
+      type:    empty_to_nil( line_str[12..20] ),
+      details: empty_to_nil( line_str[20..70] ),
+      out:     empty_to_nil( line_str[70..90] ),
+      in:      empty_to_nil( line_str[90..110] ),
+      balance: empty_to_nil( line_str[110..130] )
+    }
+    
+    # munge things further
+    row[:date]    = Date.strptime( row[:date], '%d %b %y' ) unless row[:date].nil?
+    row[:out]     = row[:out].gsub( ',', '' ).to_f unless row[:out].nil?
+    row[:in]      = row[:in].gsub( ',', '' ).to_f unless row[:in].nil?
+    row[:balance] = row[:balance].gsub( ',', '' ).to_f unless row[:balance].nil?
+    
+    # set a change amount
+    row[:change]  = if !row[:out].nil?
+      0 - row[:out]
+    elsif !row[:in].nil?
+      row[:in]
+    else
+      nil
+    end
+    
+    # return
+    row
+      
+  end
+  
+  def empty_to_nil( str )
+    
+    ( str.strip!.empty? ) ? nil : str
+    
+  end
+  
+end

data/lib/hsbc_pdf_statement_parser/pdf_reader_patch.rb

@@ -0,0 +1,44 @@
+# This is a horrendous patch to work around issue #169 in the pdf-reader repo (https://github.com/yob/pdf-reader/issues/169).
+#
+# Short version is that whatever HSBC is using to generate PDFs doesn’t seem to null-/whitespace-terminate inline image
+# data. Thus, when PdfReader tries to find the ‘EI’ token when parsing inline media, it can’t and simply runs off the end
+# of the document, causing a TypeError to be thrown.
+#
+# The PDF files I’m getting all seem to end some of the images with xE0, so I’ve simply monkey-patched this into the
+# library for use with my files.
+# This may not be the case for anyone else, in which case maybe add whatever your problem character is to the regex and
+# open a PR should you feel the need.
+# 
+# Or, y’know, look at the PdfReader source and see if you can work out something better, because this is horrendous :/
+
+module PdfReaderPatch
+  def self.included( base )
+    base.class_eval do      
+      def prepare_inline_token
+        str = "".dup
+
+        buffer = []
+        to_rewind = -3
+
+        until buffer[0] =~ /\s|\0|\xE0/n && buffer[1, 2] == ['E', 'I']
+          chr = @io.read(1)
+          buffer << chr
+          
+          if buffer.length > 3
+            str << buffer.shift
+          end
+          
+          to_rewind = -2 if buffer.first =~ /\xE0/n
+        end
+
+        str << '\0' if buffer.first == '\0'
+
+        @tokens << string_token(str)
+              
+        @io.seek(to_rewind, IO::SEEK_CUR) unless chr.nil?      
+      end
+    end
+  end
+end
+
+PDF::Reader::Buffer.send( :include, PdfReaderPatch )

data/lib/hsbc_pdf_statement_parser/version.rb

@@ -0,0 +1,3 @@
+module HsbcPdfStatementParser
+  VERSION = "1.0.0"
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: hsbc_pdf_statement_parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Jon Pearse
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+description: 
+email: hello@jonpearse.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE.txt
+- README.textile
+- hsbc_pdf_statement_parser.gemspec
+- lib/hsbc_pdf_statement_parser.rb
+- lib/hsbc_pdf_statement_parser/parser.rb
+- lib/hsbc_pdf_statement_parser/pdf_reader_patch.rb
+- lib/hsbc_pdf_statement_parser/version.rb
+homepage: https://github.com/jonpearse/hsbc-pdf-statement-parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Quick and dirty RubyGem to parse HSBC’s statement PDFs
+test_files: []