statements 0.1.0

data/lib/statements/cli.rb

@@ -0,0 +1,17 @@
+require 'thin'
+
+module Statements
+  class Cli
+
+    def self.main(pwd, argv)
+      Statements::Database.new "#{pwd}/statements.sqlite3"
+      Statements::Reader.read_dir pwd
+      Thin::Server.start '0.0.0.0', 57473 do
+        map('/q') { run Server.new }
+        use Rack::Static, urls: [''], root: "#{ROOT}/lib/html", index: 'index.html'
+        run Server
+      end
+    end
+
+  end
+end

data/lib/statements/database.rb

@@ -0,0 +1,22 @@
+require 'active_record'
+
+module Statements
+  class Database
+
+    def initialize(path = nil)
+      @path = ENV['DB_PATH'] || path
+      ActiveRecord::Base.establish_connection(
+          adapter: 'sqlite3',
+          database: @path
+      )
+      ActiveRecord::Base.logger = Logger.new(ENV['DB_LOG']) if ENV['DB_LOG']
+      ActiveRecord::Migrator.migrate migrations_dir
+    end
+
+    private
+
+    def migrations_dir
+      @migrations_dir ||= Statements::ROOT.join('lib/statements/migrations').to_s
+    end
+  end
+end

data/lib/statements/migrations/00_alpha.rb

@@ -0,0 +1,43 @@
+class Alpha < ActiveRecord::Migration
+  def change
+
+    create_table :documents do |t|
+      t.string :path
+      t.string :md5, limit: 32
+
+      t.timestamps null: true
+
+      t.index :md5, unique: true
+    end
+
+    create_table :accounts do |t|
+      t.string :name
+      t.string :number
+
+      t.timestamps null: true
+
+      t.index [:name, :number], unique: true
+    end
+
+    create_table :transactions do |t|
+      t.references :document
+      t.references :account
+      t.integer :document_line
+      t.datetime :transacted_at, null: true
+      t.datetime :posted_at, null: true
+      t.string :description
+      t.decimal :amount, precision: 2, scale: 13
+      t.decimal :balance, precision: 2, scale: 13
+      t.decimal :foreign_amount, precision: 2, scale: 13
+      t.string :foreign_currency, limit: 3
+      t.string :colour, limit: 20, default: 'white'
+
+      t.string :checksum, limit: 40
+
+      t.timestamps null: true
+
+      t.index [:document_id, :document_line], unique: true
+    end
+
+  end
+end

data/lib/statements/models/account.rb

@@ -0,0 +1,9 @@
+class Account < ActiveRecord::Base
+  def self.as_json(options = nil)
+    order('name asc').map &:as_json
+  end
+
+  def as_json(options = nil)
+    slice :id, :name, :number
+  end
+end

data/lib/statements/models/document.rb

@@ -0,0 +1,29 @@
+require 'digest'
+
+class Document < ActiveRecord::Base
+  has_many :transactions
+
+  def scan(base: nil)
+    path = base + self.path
+    md5 = Digest::MD5.file(path).hexdigest.downcase
+    print "Scanning #{self.path} ... "
+    if md5 == self.md5
+      puts 'skipping (unchanged)'
+    else
+      reader = Statements::Reader.for_file(path)
+      if reader
+        Transaction.delete_all document: self if persisted?
+        reader.transactions.each do |t|
+          t.document = self
+          t.save! unless Transaction.find_by('checksum = ? AND document_id != ?', t.checksum!, id || 0)
+        end
+        puts "added #{transactions.count} transactions(s)"
+      else
+        puts 'skipping (unknown format)'
+      end
+    end
+    self.md5 = md5
+    save!
+  end
+
+end

data/lib/statements/models/transaction.rb

@@ -0,0 +1,43 @@
+require 'active_record'
+require 'digest'
+
+class Transaction < ActiveRecord::Base
+  belongs_to :account
+  belongs_to :document
+
+  before_save :checksum!
+
+  def set_account(name, number)
+    self.account = Account.find_or_create_by(
+        name: name,
+        number: number
+    )
+  end
+
+  def checksum!
+    self.checksum = calculate_checksum
+  end
+
+  def credit?
+    amount > 0
+  end
+
+  def debit?
+    amount < 0
+  end
+
+  private
+
+  def calculate_checksum
+    parts = [
+        account_id.to_s,
+        transacted_at.strftime('%F'),
+        posted_at.strftime('%F'),
+        description,
+        amount.to_s,
+        balance.to_s
+    ]
+    Digest::SHA1.hexdigest parts.join "\0"
+  end
+
+end

data/lib/statements/pdf_reader.rb

@@ -0,0 +1,34 @@
+require 'shellwords'
+
+module Statements
+  module PdfReader
+    class << self
+
+      def read(path)
+        x = 0
+        result = []
+        loop do
+          page = read_page(path, x += 1)
+          break if page.chomp.empty?
+          result << page
+        end
+        result
+      end
+
+      private
+
+      def read_page(path, page)
+        `#{pdftotext_path} -enc UTF-8 -table -q -f #{page} -l #{page} #{Shellwords.escape path} /dev/stdout`
+      end
+
+      def pdftotext_path
+        unless @pdftotext_path
+          @pdftotext_path = `which pdftotext`.chomp
+          raise 'Could not find `pdftotext`. Please install Xpdf from http://www.foolabs.com/xpdf/download.html' if @pdftotext_path.empty?
+        end
+        @pdftotext_path
+      end
+
+    end
+  end
+end

data/lib/statements/reader/common/st_george.rb

@@ -0,0 +1,31 @@
+module StGeorge
+  ABNS = %w[33007457141 92055513070]
+
+  def st_george?
+    document =~ /\bSt\. *George\b/ && document =~ /\bABN *((?:\d *){11})/ && ABNS.include?($1.delete ' ')
+  end
+
+  def years
+    @years ||= period.map(&:year)
+  end
+
+  def parse_date(str)
+    date = change_year(Time.parse(str.to_s.strip), years.first)
+    date = change_year(date, years.last) if date < period.first
+    date
+  end
+
+  def period
+    @period ||= (pages.first =~ %r`Statement Period\s+(\d\d/\d\d/\d{4})\s+to\s+(\d\d/\d\d/\d{4})` && [Time.parse($1), Time.parse($2)])
+  end
+
+  def account_number
+    @account_number ||= document[/Account Number ([\d ]+)/, 1].strip.gsub(/\s+/, ' ')
+  end
+
+  private
+
+  def change_year(time, year)
+    Time.new year, time.month, time.day
+  end
+end

data/lib/statements/reader/st_george_credit_card.rb

@@ -0,0 +1,53 @@
+require_relative 'common/st_george'
+require 'time'
+require 'bigdecimal'
+
+module Statements
+  class Reader
+    class StGeorgeCreditCard < self
+      include StGeorge
+
+      def valid?
+        st_george? && pages.first.include?('Credit Limit')
+      end
+
+      def self.cell_pattern
+        @cell_pattern ||= %r`^
+          \s* (\d{1,2}\s+[A-Z][a-z]{2})
+          \s* (\d{1,2}\s+[A-Z][a-z]{2})
+          \s* (.+?)
+          \s* (\$[\d,]+\.\d\d(?:\s+CR)?)
+          \s* (\$[\d,]+\.\d\d(?:\s+CR)?)
+          \s* (\n\s+\d+\.\d\d\s+[A-Z]{3})?
+          \s*$
+        `x
+      end
+
+      def parse_cells(cells, tx)
+        [:posted_at, :transacted_at].each.with_index do |attr, index|
+          date = Time.parse(cells[index])
+          tx[attr] = Time.new((date.month == 12 ? years.first : years.last), date.month, date.day)
+        end
+        tx.description = cells[2]
+        {amount: 3, balance: 4}.each do |attr, index|
+          number = BigDecimal cells[index][/[\d,]+\.\d+/].delete(',')
+          credit = cells[index].end_with? 'CR'
+          number *= -1 unless credit
+          tx[attr] = number
+        end
+        foreign = cells[5]
+        if foreign
+          tx.foreign_amount = BigDecimal foreign[0..-5]
+          tx.foreign_amount *= -1 if tx.amount < 0
+          tx.foreign_currency = foreign[-3..-1]
+        end
+        tx.set_account account_name, account_number
+      end
+
+      def account_name
+        @account_name ||= document[/^\s*(.+)\s*Statement\s*$/, 1].strip
+      end
+
+    end
+  end
+end

data/lib/statements/reader/st_george_savings.rb

@@ -0,0 +1,95 @@
+require_relative 'common/st_george'
+require 'time'
+require 'bigdecimal'
+
+module Statements
+  class Reader
+    class StGeorgeSavings < self
+      include StGeorge
+
+      DATED_DESCRIPTIONS = [
+          'INTERNET WITHDRAWAL',
+          'INTERNET DEPOSIT',
+          'ATM DEPOSIT',
+          'ATM WITHDRAWAL',
+          'VISA PURCHASE',
+          'VISA PURCHASE O/SEAS',
+          'EFTPOS PURCHASE'
+      ]
+
+      def valid?
+        st_george? && pages.first =~ /BSB\s+Number\s+112-879/
+      end
+
+      def self.cell_pattern
+        @cell_pattern ||= %r`^
+           (\d\d\s+)
+           ([A-Z]{3}\s+)
+           (.+?)
+           ([\d,]+\.\d\d) \s+
+           ([\d,]+\.\d\d) \s*
+           $
+           ((?:
+             \n+\040[^\r\n]+
+           )*)
+        `x
+      end
+
+      def parse_cells(cells, tx, page_index)
+
+        # Examples cells:
+        # ['17 ', 'NOV  ', 'LINE 1  ', '12,345.67', '50.00', "\n  LINE 2\n  Line 3"]
+
+        tx.posted_at = parse_date(cells[0..1].join)
+
+        is_debit = is_debit?(cells, page_index)
+        debit_factor = is_debit ? -1 : 1
+
+        tx.amount  = BigDecimal(cells[3].delete ',') * debit_factor
+        tx.balance = BigDecimal(cells[4].delete ',')
+
+        # TODO: negative balances
+
+        lines = [cells[2].strip]
+        lines += cells[5].strip.split(/\s*\n\s*/) if cells[5]
+        lines.each { |line| line.gsub! /\s+/, ' ' }
+        lines.reject! { |line| line.delete(' ').start_with? 'SUBTOTALCARRIEDFORWARDTONEXTPAGE' }
+
+        tx.description = lines.join("\n")
+
+        if lines.first =~ %r`^(.+?) (?:(\d\d)/(\d\d)/\d\d|(\d\d\w{3}) \d\d:\d\d)$` && DATED_DESCRIPTIONS.include?($1)
+          description = $1
+          tx.transacted_at = $2 ? parse_date("2000-#{$3}-#{$2}") : parse_date($4)
+          if description.end_with?('O/SEAS') && lines.last =~ /\b([A-Z]{3}) ([\d,]+\.\d\d)$/
+            tx.foreign_currency = $1
+            tx.foreign_amount = BigDecimal($2.delete ',') * debit_factor
+          end
+        end
+
+        tx.transacted_at ||= tx.posted_at
+        
+        tx.set_account account_name, account_number
+      end
+
+      def is_debit?(cells, page_index)
+        amount_offset = cells[0..2].map(&:length).inject(:+)
+        amount_center = amount_offset + cells[3].length / 2
+        amount_center < credit_threshold(page_index)
+      end
+
+      def credit_threshold(page)
+        (@credit_thresholds ||= {})[page] ||= find_credit_threshold(page)
+      end
+
+      def find_credit_threshold(page)
+        lengths = pages[page].scan(/^(Date\s+Transaction\s+De\w+\s+)(Debit\s+Credit)/).first.map(&:length)
+        lengths.first + lengths.last / 2
+      end
+
+      def account_name
+        @account_name ||= document[/^\s*Statement\s+of\s+Account\s*((?:\S\s{0,4})+)/, 1].strip
+      end
+
+    end
+  end
+end

data/lib/statements/reader.rb

@@ -0,0 +1,76 @@
+require 'forwardable'
+
+module Statements
+  class Reader
+    extend Forwardable
+
+    class << self
+      attr_reader :classes
+    end
+
+    unless classes
+      Dir[Statements::ROOT.join('lib/statements/reader/*.rb')].each { |p| require p }
+      @classes = constants.map { |n| const_get n }.select { |c| Class === c && c < Reader }
+    end
+
+    def self.for_file(file)
+      file = file.to_s
+      pages = (file =~ /\.pdf$/i) ? PdfReader.read(file) : File.read(file).split(/-{5,}/)
+      classes.each do |klass|
+        reader = klass.new(pages)
+        return reader if reader.valid?
+      end
+      nil
+    end
+
+    def self.read_dir(dir)
+      base = Pathname(dir).realpath
+      Dir[base.join('**/*.{pdf,txt}')].each do |path|
+        rel_path = Pathname(path).relative_path_from(base)
+        begin
+          doc = Document.find_or_initialize_by(path: rel_path.to_s)
+          doc.scan base: base
+        rescue => e
+          puts "error: #{e.class.name} #{e.message}\n  #{e.backtrace.join "\n  "}"
+        end
+      end
+    end
+
+    attr_reader :pages, :document
+
+    delegate [:include?, :scan] => :document
+
+    def initialize(pages)
+      @pages = pages
+      @document = pages.join("\n").freeze
+    end
+
+    def cell_pattern
+      self.class.cell_pattern
+    end
+
+    def self.cell_pattern
+      raise NotImplementedError
+    end
+
+    def transactions
+      @transactions ||= search_for_transactions
+    end
+
+    def search_for_transactions
+      index = 0
+      result = []
+      pages.each.with_index do |page, page_index|
+        page.scan(cell_pattern).each do |cells|
+          result << Transaction.new(document_line: index += 1).tap do |transaction|
+            args = [cells, transaction, page_index]
+            arity = method(:parse_cells).arity
+            parse_cells *args[0..(arity - 1)]
+          end
+        end
+      end
+      result
+    end
+
+  end
+end

data/lib/statements/search.rb

@@ -0,0 +1,48 @@
+require 'time'
+
+module Statements
+  class Search
+
+    attr_reader :input
+
+    def initialize(input)
+      @input = input
+    end
+
+    def transactions
+      @transactions ||= query.all
+    end
+
+    def debits
+      transactions.select { |t| t.amount < 0 }
+    end
+
+    def credits
+      transactions.select { |t| t.amount > 0 }
+    end
+
+    def total(collection = :transactions)
+      __send__(collection).inject(0) { |sum, t| sum + t.amount }
+    end
+
+    private
+
+    def query
+      query = Transaction.order(input['order'])
+      query = query.where(account_id: input['accounts'])
+      query = query.where('posted_at > ? and posted_at < ?',
+                          Time.parse(input['date_start']),
+                          Time.parse(input['date_end']))
+      query = query.where('amount < 0') if input['type'] == 'debits'
+      query = query.where('amount > 0') if input['type'] == 'credits'
+      query = query.where(colour: input['colours'])
+      text = input['search'].strip.downcase
+      unless text.empty?
+        words = text.split(/\s+/)
+        query = query.where('lower(description) like ?', "%#{words.join '%'}%")
+      end
+      query
+    end
+
+  end
+end

data/lib/statements/server.rb

@@ -0,0 +1,69 @@
+require 'json'
+require 'erb'
+
+module Statements
+  class Server
+
+    def initialize
+
+    end
+
+    def call(env)
+      request = Rack::Request.new(env)
+      verb = request.request_method.downcase
+      path = request.path_info[1..-1].split('/')
+      handler_name = "#{verb}_#{path.first || 'index'}".gsub('.', '_')
+      args = [request] + path[1..-1]
+      method = respond_to?(handler_name) && method(handler_name)
+      if method && method.arity == args.length
+        __send__ handler_name, *args
+      else
+        [404, {}, ['Not found']]
+      end
+    end
+
+    # noinspection RubyStringKeysInHashInspection
+    def serve(type, str)
+      [200, {'Content-Type' => type, 'Content-Length' => str.length.to_s}, [str]]
+    end
+
+    def json(data)
+      serve 'application/json', JSON.generate(data, quirks_mode: true)
+    end
+
+    def js(script)
+      serve 'application/x-javascript', script
+    end
+
+    def html(html)
+      serve 'text/html; charset=UTF-8', html
+    end
+
+    def get_accounts_js(request)
+      js "window.accounts = #{Account.to_json}"
+    end
+
+    def self.render(template, obj = nil)
+      @templates ||= {}
+      @templates[template] ||= ERB.new(File.read File.expand_path("../views/#{template}.erb", __FILE__))
+      @templates[template].result (obj || self).instance_eval { binding }
+    end
+
+    def post_search_html(request)
+      html self.class.render 'search', Search.new(JSON.parse request.body.read)
+    end
+
+    def post_colour_json(request)
+      input = JSON.parse(request.body.read)
+      transaction = Transaction.find(input['id']) rescue false
+      if transaction
+        transaction.colour = input['colour']
+        transaction.save
+        json success: true
+      else
+        400
+      end
+    end
+
+  end
+end

data/lib/statements/version.rb

@@ -0,0 +1,3 @@
+module Statements
+  VERSION = '0.1.0'
+end

data/lib/statements/views/footer.erb

@@ -0,0 +1,6 @@
+<tr>
+    <th></th>
+    <th><%= "#{ActiveSupport::NumberHelper.number_to_delimited count} #{name.capitalize[0..-2]}#{count == 1 ? '' : 's'}" %></th>
+    <td><%= ActiveSupport::NumberHelper.number_to_currency total.abs, unit: '' %></td>
+    <th></th>
+</tr>

data/lib/statements/views/row.erb

@@ -0,0 +1,16 @@
+<tr class="<%= credit? ? 'credit' : 'debit' %> colour-<%= colour %>" data-id="<%= id %>">
+    <td class="date">
+        <%= posted_at.strftime '%-d %b %Y' %>
+    </td>
+    <td class="description">
+        <% desc = description.lines %>
+        <p><%= desc[0] %></p>
+        <% if desc.length > 1 %>
+            <p class="small"><%= desc[1..-1].join '<br/>' %></p>
+        <% end %>
+    </td>
+    <td class="amount"><%= ActiveSupport::NumberHelper.number_to_currency amount.abs, unit: '' %></td>
+    <td class="colour">
+        <a href="javascript:" class="picker"></a>
+    </td>
+</tr>

data/lib/statements/views/search.erb

@@ -0,0 +1,25 @@
+<table class="table table-condensed">
+    <thead>
+    <tr>
+        <th>Posted</th>
+        <th>Description</th>
+        <th class="amount">Amount</th>
+        <th><!-- Tags --></th>
+    </tr>
+    </thead>
+    <tbody>
+    <%= transactions.map { |x| Statements::Server.render 'row', x }.join %>
+    </tbody>
+    <tfoot>
+    <%= %w[debits credits transactions].map do |n|
+      list = __send__(n)
+      data = {
+        name: n,
+        count: list.count,
+        total: total(n)
+      }
+      Statements::Server.render 'footer', OpenStruct.new(data)
+    end.join %>
+    </tfoot>
+</table>
+<!-- fix for long response bug -->

data/lib/statements.rb

@@ -0,0 +1,17 @@
+require 'pathname'
+
+module Statements
+  ROOT = Pathname File.expand_path('../..', __FILE__)
+end
+
+require 'statements/version'
+require 'statements/cli'
+require 'statements/reader'
+require 'statements/database'
+require 'statements/pdf_reader'
+require 'statements/server'
+require 'statements/search'
+
+require 'statements/models/transaction'
+require 'statements/models/account'
+require 'statements/models/document'