bank_statement_parser 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8b4ac87d92f4024ad763ee53e76547c6677c01de
+  data.tar.gz: 0549a669283aa82da48fa7c54fc1d6e2cffcd8a0
+SHA512:
+  metadata.gz: 60756a4ea07f81fb4bb0e8ed98335f68cd68bbde0152d95865c0457d2853d79af88d7f15359c243b68c821b891e62ec61e210d33c05f37cf0e65de784639686b
+  data.tar.gz: 2283b2bafe0d69d8125ced615be1833d0ac12c0ebeadfdf9884155fc3b525dfb7caa4d0f9fc12271bc17b12d90d0a4d3647d2a78606e6e08c303b0ff7213eed5

data/lib/bank_statement_parser/base.rb

@@ -0,0 +1,56 @@
+module BankStatementParser
+
+  # Base class for statement parsers
+  #
+  # Subclasses must implement the following instance methods
+  #
+  # * void reset()
+  # * bool handle_line(String line)
+  class Base
+
+    require 'fileutils.rb'
+
+    attr_accessor :sort_code, :account_number, :statement_date, :records
+
+    # Constructor
+    def initialize
+      reset
+    end
+
+    # Parse the specified text file
+    def parse path
+      raise "Expected a text file path" unless path =~ /\.txt\z/
+
+      reset
+
+      # Grab the full text file content
+      full_text = File.read(path)
+
+      # Process each line in turn
+      full_text.split("\n").each do |line|
+        break unless handle_line(line)
+      end
+
+      # Sanity checking
+      raise "Failed to find sort code" if @sort_code.nil?
+      raise "Failed to find account number" if @account_number.nil?
+      raise "Failed to find statement date" if @statement_date.nil?
+    end
+
+    protected
+
+    # Convenience method to access the logger
+    def logger
+      BankStatementParser.logger
+    end
+
+    # Reset the parser
+    def reset
+      @sort_code = nil
+      @account_number = nil
+      @statement_date = nil
+      @records = []
+    end
+
+  end
+end

data/lib/bank_statement_parser/hsbc.rb

@@ -0,0 +1,354 @@
+module BankStatementParser
+
+  # Parser for HSBC bank statements
+  class HSBC < Base
+
+    # Handle the specified line
+    #
+    # Returns true if parsing should continue; false to terminate the parser
+    def handle_line line
+
+      # Skip blank lines
+      return true if line =~ /\A\s*\z/
+
+      # Sanity checking
+      raise "line contains TAB characters" if line =~ /\t/
+
+      # Stop line
+      if line =~ /\A\s+AER\s+EAR\s*\z/
+        logger.debug { "Found stop line (2nd form)" }
+        return false
+      end
+      if line =~ /\AStatements produced from \d{1,2} (?:#{MONTHS.join('|')}) \d{4} are available in PDF format\.\s*\z/
+        logger.debug { "Found stop line (1st form)" }
+        return false
+      end
+
+      # Look for sort code and account number lines, if we haven't found
+      # one yet
+      if @sort_code.nil? && @account_number.nil?
+        if line =~ /(?:\A[A-Z][\w\s]+|,)\s+(?<sort_code>\d{2}-\d{2}-\d{2})\s+(?<account_number>\d{8})(?:\s*|\s+\d+)\z/
+          logger.debug { "Found sort code and account number" }
+          @sort_code = Regexp.last_match(:sort_code)
+          @account_number = Regexp.last_match(:account_number)
+        end
+      end
+
+      # Look for statement date lines, if we haven't found one yet
+      if @statement_date.nil?
+        if line =~ /\A\s*(?<statement_date>\d{2} (?:#{MONTHS.map{|m| m[0,3]}.join('|')}) \d{4})\s*\z/
+          logger.debug { "Found statement date (1st form)" }
+          @statement_format = StatementFormat::FORMAT_1ST
+
+          # Parse statement date
+          date_string = Regexp.last_match(:statement_date)
+          @statement_date = DateTime.parse(date_string)
+        elsif line =~ /\A(?<date_range_start>\d+\s+(?:#{MONTHS.join('|')})(?:\s+\d{4})?)\s+to\s+(?<date_range_end>\d+\s+(?:#{MONTHS.join('|')})\s+\d{4})\b/
+          logger.debug { "Found statement date (2nd form)" }
+          @statement_format = StatementFormat::FORMAT_2ND
+
+          date_range_start = Regexp.last_match(:date_range_start)
+          date_range_end = Regexp.last_match(:date_range_end)
+          logger.debug { "Found statement date range #{date_range_start}-#{date_range_end}" }
+
+          # Parse range end date
+          @statement_date = DateTime.parse(date_range_end)
+        end
+      end
+
+      if !@sort_code.nil? && !@account_number.nil? && !@statement_date.nil?
+
+        # Look for statement records proper
+        headings = nil
+        case @statement_format
+        when StatementFormat::FORMAT_UNKNOWN
+          raise "Failed to detect statement format before start of records"
+        when StatementFormat::FORMAT_1ST
+          headings = COLUMN_HEADINGS_1ST
+        when StatementFormat::FORMAT_2ND
+          headings = COLUMN_HEADINGS_2ND
+        end
+        logger.debug { "Parsing potential record line (format #{@statement_format})" }
+        parse_record_line_format(line, headings)
+
+      end
+
+      return true
+    end
+
+    private
+
+    TYPES = ["ATM", "BP", "CHQ", "CIR", "CR", "DD", "DIV", "DR",
+             "MAE", "PIM", "SO", "TFR", "VIS", ")))"]
+
+    MONTHS = Date::MONTHNAMES[1..12]
+
+    # N.B. Unicode pound symbol deleted from brackets in balance column heading
+    COLUMN_HEADINGS_1ST = ["Date",
+                           "Type\\s+Description",
+                           "Paid out",
+                           "Paid in",
+                           "Balance \\(\\)"]
+
+    COLUMN_HEADINGS_2ND = ["Date",
+                           "Payment type and details",
+                           "Paid out",
+                           "Paid in",
+                           "Balance"]
+
+    # Enumerate statement formats
+    module StatementFormat
+      FORMAT_UNKNOWN = 0
+      FORMAT_1ST = 1 # "old" style, browser-printed PDF
+      FORMAT_2ND = 2 # "new", pre-formatted PDF
+    end
+
+    # Reset the parser
+    def reset
+      super
+
+      @statement_format = StatementFormat::FORMAT_UNKNOWN
+
+      # Somewhere to cache the most-recent statement record date
+      @cached_statement_date = nil
+
+      # Somewhere to cached the most-recent statement record type
+      @cached_payment_type = nil
+
+      # Somewhere to cache details for the ongoing statement record
+      @cached_details = []
+
+      # Somewhere to cache column alignments
+      @cols = []
+
+      # Flag to temporarily pause the parser
+      @parser_paused = false
+    end
+
+    # Fix the year of the specified record date
+    #
+    # Returns the record date, with the year fixed
+    def fix_record_date_year record_date
+      # Sanity checking
+      if Date.today.year != record_date.year
+        logger.info { "No need to fix year for statement record date" }
+        return record_date
+      end
+
+      # The date we have parsed will have the year set to the current year.
+      #
+      # We need to figure out the correct year, from the statement date.
+      raise "No statement date" unless @statement_date
+      record_date = record_date.change(year: @statement_date.year)
+      logger.debug { "record date #{record_date}" }
+      if @statement_date.month != record_date.month
+        logger.debug { "record month differs from statement month" }
+        if 1 == @statement_date.month
+          # Assume that the statement crosses a year boundary: the record
+          # must be from the end of the previous year
+          raise "Expected a record from December" unless
+            12 == record_date.month
+          record_date = record_date.prev_year
+        end
+      end
+
+      record_date
+    end
+
+    # If the specified line is a headings line, use it to update our column
+    # alignments
+    #
+    # Returns true if column alignments were updated; false otherwise
+    def update_columns line, headings
+      # Look for lines that allow us to match column alignments
+      raise "Expected a five-column layout" unless 5 == headings.size
+
+      # Build a regexp for matching the column header line
+      column_heading_regexp_str = '\A'
+      headings.each_with_index do |item,index|
+        pre_space_match = ''
+        post_space_quantifier = '{2,}'
+        if 0 == index
+          pre_space_match = '\s*'
+        elsif (headings.length - 1) == index
+          post_space_quantifier = '*'
+        end
+        column_heading_regexp_str +=
+          '(?<col' + index.to_s + '>' + pre_space_match + item + '\s' + post_space_quantifier + ')'
+      end
+      column_heading_regexp_str += '\z'
+      column_heading_regexp = Regexp.new(column_heading_regexp_str)
+
+      if line =~ column_heading_regexp
+        if @cols.empty?
+          logger.debug { "Setting column alignments from line #{line}" }
+        else
+          logger.debug { "Updating column alignments from line #{line}" }
+        end
+        (0...headings.size).each do |i|
+          str_i = "col" + i.to_s
+          sym_i = str_i.to_sym
+          @cols[i] = Regexp.last_match.offset(sym_i)[0]
+        end
+
+        return true
+      end
+
+      return false
+    end
+
+    # Split the specified line into an array of column fragments
+    def get_column_fragments line
+      col_fragments = []
+
+      @cols.reverse.each_with_index do |i,index|
+        # We need to be flexible here, because the columns can (and do)
+        # fail to line up with the heading alignments
+        #
+        # Check whether the supposed column boundary has whitespace on
+        # at least one side:
+        #
+        # * If so, then this is a correct column boundary
+        # * If not, then (somewhat arbitrarily, based on cases that have
+        #   been seen) opt to move the column left until we hit whitespace
+        if (i > 0) && (i < line.length)
+          char_before_boundary = line[i-1]
+          char_after_boundary = line[i]
+          unless char_before_boundary =~ /\A\s\z/ ||
+              char_after_boundary =~ /\A\s\z/
+            logger.warn { "Column boundary failure: #{char_before_boundary}|#{char_after_boundary}" }
+
+            # Shift down until we hit whitespace before the boundary
+            boundary_limit =
+              ((index + 1) < @cols.reverse.size) ? @cols.reverse[index + 1] : -1
+            logger.debug { "Boundary adjust limit #{boundary_limit}" }
+            new_boundary = i
+            while new_boundary > boundary_limit
+              left = line[new_boundary]
+              if left =~ /\A\s\z/
+                logger.debug { "Adjusting column boundary from #{i} to #{new_boundary}" }
+                i = new_boundary
+                break
+              end
+              new_boundary -= 1
+            end
+
+            raise "Failed to adjust column boundary" if 0 == new_boundary
+
+          end
+        end
+
+        fragment_i = line[i...(line.length)]
+        unless fragment_i.nil?
+          fragment_i.strip!
+          if fragment_i.empty?
+            fragment_i = nil
+          end
+        end
+        col_fragments.unshift(fragment_i)
+        line = line[0...i]
+      end
+
+      return col_fragments
+    end
+
+    # Parse the specified line, looking for records
+    def parse_record_line_format line, headings
+
+      if update_columns(line, headings)
+        if @parser_paused
+          logger.debug { "Resuming parser: set/updated columns" }
+          @parser_paused = false
+        end
+        return
+      end
+
+      return if @cols.empty?
+
+      return if @parser_paused
+
+      col_fragments = get_column_fragments(line)
+
+      # N.B. Detect and fix up failed column splitting
+      date_string = col_fragments[0]
+      unless date_string.nil?
+        if date_string =~ /(?<date_proper>.+)\s+(?<spurious_tail>[A-Z]+)\z/
+          date_proper = Regexp.last_match(:date_proper)
+          spurious_tail = Regexp.last_match(:spurious_tail)
+          logger.warn { "Must fix date string #{date_string}|#{date_proper}|#{spurious_tail}" }
+          col_fragments[0] = date_proper
+          col_fragments[1] = spurious_tail + " " + col_fragments[1]
+        end
+      end
+
+      date_string = col_fragments[0]
+      unless date_string.nil?
+        begin
+          @cached_statement_date = DateTime.parse(date_string)
+          @cached_statement_date =
+            fix_record_date_year(@cached_statement_date)
+        rescue ArgumentError => e
+          raise "Failed to parse date/time #{date_string}: #{e}"
+        end
+      end
+
+      payment_type_and_details = col_fragments[1]
+
+      if payment_type_and_details =~ /\ABALANCE CARRIED FORWARD\z/i
+        logger.debug { "Pausing parser" }
+        @parser_paused = true
+        return
+      elsif payment_type_and_details =~ /\ABALANCE BROUGHT FORWARD\z/i
+        if @parser_paused
+          logger.debug { "Resuming parser" }
+          @parser_paused = false
+        else
+          logger.debug { "Skipping parser resume line" }
+        end
+        return
+      end
+      if @parser_paused
+        logger.debug { "Skipping line: parser paused" }
+        return
+      end
+
+      payment_details = nil
+      if payment_type_and_details =~ /\A(?<payment_type>#{TYPES.map{ |t| Regexp.quote(t) }.join('|')})\s+(?<payment_details>.*)\z/
+        logger.debug { "Found the start of a record (group)" }
+        @cached_payment_type = Regexp.last_match(:payment_type)
+        payment_details = Regexp.last_match(:payment_details)
+      else
+        payment_details = payment_type_and_details
+      end
+      @cached_details << payment_details
+
+      paid_out = col_fragments[2]
+      paid_in = col_fragments[3]
+      paid_out.delete!(",") unless paid_out.nil?
+      paid_in.delete!(",") unless paid_in.nil?
+      balance = col_fragments[4]
+
+      if !paid_out.nil? || !paid_in.nil?
+        logger.debug { "Found the end of a record (group)" }
+        full_details = @cached_details.join("\n")
+
+        record_credit = !paid_in.nil?
+        record_amount = record_credit ? paid_in.to_f : paid_out.to_f
+
+        # Create statement record
+        record = StatementRecord.new(date: @cached_statement_date,
+                                     type: @cached_payment_type,
+                                     credit: record_credit,
+                                     amount: record_amount,
+                                     detail: full_details)
+        logger.debug { "Created statement record: #{record}" }
+        @records << record
+
+        @cached_payment_type = nil
+        @cached_details = []
+      end
+
+    end
+
+  end
+end

data/lib/bank_statement_parser/statement_record.rb

@@ -0,0 +1,31 @@
+module BankStatementParser
+
+  # A bank statement record
+  class StatementRecord
+    attr_accessor :date, :type, :credit, :amount, :detail
+
+    # Constructor
+    def initialize date: nil, type: nil, credit: nil, amount: nil, detail: nil
+      @date = date
+      @type = type
+      @credit = credit
+      @amount = amount
+      @detail = detail
+    end
+
+    # Stringify
+    def to_s
+      "%s:%s:%s:%f:%s" % [date, type, credit.to_s, amount, detail]
+    end
+
+    # Equality test
+    def ==(other)
+      super || (date == other.date &&
+                type == other.type &&
+                credit == other.credit &&
+                amount == other.amount &&
+                detail = other.detail)
+    end
+  end
+
+end

data/lib/bank_statement_parser.rb

@@ -0,0 +1,11 @@
+module BankStatementParser
+
+  @@logger = Logger.new(STDERR)
+  def self.logger
+    @@logger
+  end
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: bank_statement_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Simon Dawson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-24 00:00:00.000000000 Z
+dependencies: []
+description: A gem for parsing bank statements
+email: spdawson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/bank_statement_parser.rb
+- lib/bank_statement_parser/base.rb
+- lib/bank_statement_parser/hsbc.rb
+- lib/bank_statement_parser/statement_record.rb
+homepage: http://rubygems.org/gems/bank_statement_parser
+licenses:
+- GPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Bank statement parser
+test_files: []