coop_to_ofx 1.0.1

data/README.rdoc

@@ -0,0 +1,38 @@
+Convert statement HTML from the Co-operative bank's online banking system to OFX for import into financial apps.
+
+= Usage
+
+For a Current Account:
+
+1. Save the HTML source of the statement page.
+
+  coop_to_ofx --current /path/to/statement.html
+  
+Will produce /path/to/statement.ofx
+
+For a Credit Card:
+
+1. Save the HTML source of the statement page
+
+  coop_to_ofx /path/to/statement.html
+
+Or
+
+  coop_to_ofx --credit /path/to/statement.html
+  
+Will produce /path/to/statement.ofx
+
+
+To produce OFX 1 SGML (rather than OFX 2 XML):
+
+  coop_to_ofx --ofx1 /path/to/statement.html
+  coop_to_ofx --ofx1 --current /path/to/statement.html
+
+To show all the options:
+
+  coop_to_ofx --help 
+
+
+== To do
+
+XML / SGML validation of output against the specs

data/Rakefile

@@ -0,0 +1,141 @@
+require 'rake'
+require 'rake/rdoctask'
+gem 'rspec'
+require 'spec/rake/spectask'
+require 'lib/coop_scraper/version.rb'
+
+desc 'Generate documentation for Co-op-to-OFX.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Co-op to OFX'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+task :default => :spec
+
+desc "Run all specs in spec directory (excluding plugin specs)"
+Spec::Rake::SpecTask.new(:spec) do |t|
+  t.spec_opts = ['--options', "\"#{Rake.original_dir}/spec/spec.opts\""]
+  t.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+namespace :spec do
+  desc "Run all specs in spec directory with RCov (excluding plugin specs)"
+  Spec::Rake::SpecTask.new(:rcov) do |t|
+    t.spec_opts = ['--options', "\"#{Rake.original_dir}/spec/spec.opts\""]
+    t.spec_files = FileList['spec/**/*_spec.rb']
+    t.rcov = true
+    t.rcov_opts = lambda do
+      IO.readlines("#{Rake.original_dir}/spec/rcov.opts").map {|l| l.chomp.split " "}.flatten
+    end
+  end
+  
+  desc "Print Specdoc for all specs (excluding plugin specs)"
+  Spec::Rake::SpecTask.new(:doc) do |t|
+    t.spec_opts = ["--format", "specdoc", "--dry-run"]
+    t.spec_files = FileList['spec/**/*_spec.rb']
+  end
+end
+
+require "rubygems"
+require "rake/gempackagetask"
+
+# This builds the actual gem. For details of what all these options
+# mean, and other ones you can add, check the documentation here:
+#
+#   http://rubygems.org/read/chapter/20
+#
+spec = Gem::Specification.new do |s|
+  
+  # Change these as appropriate
+  s.name              = "coop_to_ofx"
+  s.version           = CoopScraper::Version()
+  s.summary           = "Convert Co-operative bank HTML statements into OFX"
+  s.description       = File.read('README.rdoc')
+  s.author            = "Matt Patterson"
+  s.email             = "matt@reprocessed.org"
+  s.homepage          = "http://reprocessed.org/"
+
+  s.has_rdoc          = true
+  s.extra_rdoc_files  = %w(README.rdoc)
+  s.rdoc_options      = %w(--main README.rdoc)
+
+  # Add any extra files to include in the gem
+  s.files             = %w(Rakefile README.rdoc) + Dir.glob("{bin,spec,lib}/**/*")
+  s.executables       = FileList["bin/**"].map { |f| File.basename(f) }
+   
+  s.require_paths     = ["lib"]
+  
+  # If you want to depend on other gems, add them here, along with any
+  # relevant versions
+  s.add_dependency("hpricot", "~> 0.6.0")
+  s.add_dependency("builder", "~> 2.1.0")
+  
+  s.add_development_dependency("rspec") # add any other gems for testing/development
+
+  # If you want to publish automatically to rubyforge, you'll may need
+  # to tweak this, and the publishing task below too.
+  s.rubyforge_project = "coop_to_ofx"
+end
+
+# This task actually builds the gem. We also regenerate a static 
+# .gemspec file, which is useful if something (i.e. GitHub) will
+# be automatically building a gem for this project. If you're not
+# using GitHub, edit as appropriate.
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+  
+  # Generate the gemspec file for github.
+  file = File.dirname(__FILE__) + "/#{spec.name}.gemspec"
+  File.open(file, "w") {|f| f << spec.to_ruby }
+end
+
+desc 'Clear out RDoc and generated packages'
+task :clean => [:clobber_rdoc, :clobber_package] do
+  rm "#{spec.name}.gemspec"
+end
+
+# If you want to publish to RubyForge automatically, here's a simple 
+# task to help do that. If you don't, just get rid of this.
+# Be sure to set up your Rubyforge account details with the Rubyforge
+# gem; you'll need to run `rubyforge setup` and `rubyforge config` at
+# the very least.
+begin
+  require "rake/contrib/sshpublisher"
+  namespace :rubyforge do
+    
+    desc "Release gem and RDoc documentation to RubyForge"
+    task :release => ["rubyforge:release:gem", "rubyforge:release:docs"]
+    
+    namespace :release do
+      desc "Release a new version of this gem"
+      task :gem => [:package] do
+        require 'rubyforge'
+        rubyforge = RubyForge.new
+        rubyforge.configure
+        rubyforge.login
+        rubyforge.userconfig['release_notes'] = spec.summary
+        path_to_gem = File.join(File.dirname(__FILE__), "pkg", "#{spec.name}-#{spec.version}.gem")
+        puts "Publishing #{spec.name}-#{spec.version.to_s} to Rubyforge..."
+        rubyforge.add_release(spec.rubyforge_project, spec.name, spec.version.to_s, path_to_gem)
+      end
+    
+      desc "Publish RDoc to RubyForge."
+      task :docs => [:rdoc] do
+        config = YAML.load(
+            File.read(File.expand_path('~/.rubyforge/user-config.yml'))
+        )
+ 
+        host = "#{config['username']}@rubyforge.org"
+        remote_dir = "/var/www/gforge-projects/coop_to_ofx/" # Should be the same as the rubyforge project name
+        local_dir = 'rdoc'
+ 
+        Rake::SshDirPublisher.new(host, remote_dir, local_dir).upload
+      end
+    end
+  end
+rescue LoadError
+  puts "Rake SshDirPublisher is unavailable or your rubyforge environment is not configured."
+end

data/bin/coop_to_ofx

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+$:.push(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+require 'optparse'
+require 'coop_scraper/version'
+
+options = {:format => :ofx2, :statement_type => :credit}
+OptionParser.new do |opts|
+  opts.banner = "Usage: coop_to_ofx [[--ofx1 | --ofx2] [--current | --credit]] /path/to/statement.html"
+  
+  opts.on("--ofx1", "Generate OFX 1 SGML output") do |format|
+    options[:format] = :ofx1
+  end
+  
+  opts.on("--ofx2", "Generate OFX 2 XML output (the default)") do |format|
+    options[:format] = :ofx1
+  end
+  
+  opts.on("--current", "Process Current account statements") do |statement_type|
+    options[:statement_type] = :current
+  end
+  
+  opts.on("--credit", "Process Credit Card account statements (the default)") do |statement_type|
+    options[:statement_type] = :credit
+  end
+  
+  opts.separator ""
+  opts.separator "Common options:"
+  
+  # No argument, shows at tail.  This will print an options summary.
+  # Try it and see!
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+  
+  # Another typical switch to print the version.
+  opts.on_tail("--version", "Show version") do
+    puts CoopScraper::Version()
+    puts "Copyright (c) 2009, Matt Patterson. Released under the MIT license"
+    exit
+  end
+end.parse!
+
+input_path = ARGV[0]
+raise ArgumentError, "Argument must be a real file!" unless File.file?(ARGV[0])
+create_time = File.stat(input_path).ctime
+
+output_file_path = File.dirname(input_path) + "/#{File.basename(input_path, '.html')}.ofx"
+
+if options[:statement_type] == :credit
+  require 'coop_scraper/credit_card'
+  scraper_class = CoopScraper::CreditCard
+else
+  require 'coop_scraper/current_account'
+  scraper_class = CoopScraper::CurrentAccount
+end
+
+statement = scraper_class.generate_statement(File.open(input_path), create_time)
+File.open(output_file_path, 'w') { |f| f.write(statement.serialise(options[:format])) }

data/lib/coop_scraper/base.rb

@@ -0,0 +1,8 @@
+module CoopScraper
+  module Base
+    def coop_date_to_time(coop_date)
+      day, month, year = coop_date.match(/([0-9]{2})\/([0-9]{2})\/([0-9]{4})/).captures
+      Time.utc(year, month, day)
+    end
+  end
+end

data/lib/coop_scraper/credit_card.rb

@@ -0,0 +1,94 @@
+require 'rubygems'
+require 'hpricot'
+
+require 'coop_scraper/base'
+require 'ofx/statement'
+
+module CoopScraper
+  class CreditCard
+    class << self
+      include CoopScraper::Base
+      
+      def extract_statement_date(doc)
+        coop_date_to_time(doc.at("td[text()='Statement Date'] ~ td").inner_text)
+      end
+      
+      def extract_account_number(doc)
+        doc.at("h4[text()*='TRAVEL CARD']").inner_text.match(/([0-9]{16})/)[1]
+      end
+      
+      def extract_statement_balance(doc)
+        amount, sign = doc.at("td[text()='Statement Balance'] ~ td").inner_text.match(/([0-9.]+) *(DR)?/).captures
+        amount = "-#{amount}" if sign == "DR"
+        amount
+      end
+      
+      def extract_available_credit(doc)
+        doc.at("td[text()='Available Credit'] ~ td").inner_text.match(/[0-9.]+/).to_s
+      end
+      
+      def determine_trntype(details)
+        case details
+        when /^OVERLIMIT CHARGE$/
+          :service_charge
+        when /^MERCHANDISE INTEREST AT [0-9]+\.[0-9]+% PER MTH$/
+          :interest
+        else
+          nil
+        end
+      end
+      
+      def extract_transactions(doc, statement)
+        transactions = []
+        current_transaction = {}
+        doc.search('tbody.contents tr').each do |statement_row|
+          date = statement_row.at('td.dataRowL').inner_text
+          unless date == "?" || date[0] == 194
+            current_transaction = extract_transaction(statement_row, coop_date_to_time(date))
+            transactions << current_transaction
+          else
+            transaction = extract_transaction(statement_row, statement.date)
+            if transaction[:details].match(/OVERLIMIT CHARGE/)
+              transactions << transaction
+            else
+              conversion_details = transaction[:details]
+              current_transaction[:conversion] = conversion_details unless conversion_details.match(/ESTIMATED INTEREST/)
+            end
+          end
+        end
+        transactions.collect do |t| 
+          options = {}
+          options[:memo] = t[:conversion] if t[:conversion]
+          trntype = determine_trntype(t[:details])
+          options[:trntype] = trntype unless trntype.nil?
+          OFX::Statement::Transaction.new(t[:amount], t[:date], t[:details], options)
+        end
+      end
+      
+      def extract_transaction(statement_row, date)
+        details = statement_row.at('td.transData').inner_text.strip
+        credit = statement_row.at('td.moneyData:first').inner_text.match(/[0-9.]+/)
+        debit = statement_row.at('td.moneyData:last').inner_text.match(/[0-9.]+/)
+        amount = credit.nil? ? "-#{debit}" : credit.to_s
+        {:date => date, :amount => amount, :details => details}
+      end
+      
+      def generate_statement(html_statement_io, server_response_time)
+        doc = Hpricot(html_statement_io)
+        statement = OFX::Statement::CreditCard.new
+        
+        statement.server_response_time = server_response_time
+        statement.account_number = extract_account_number(doc)
+        statement.date = extract_statement_date(doc)
+        statement.ledger_balance = extract_statement_balance(doc)
+        statement.available_credit = extract_available_credit(doc)
+
+        extract_transactions(doc, statement).each { |transaction| statement << transaction }
+
+        statement.start_date = statement.transactions.first.date
+        statement.end_date = statement.date
+        statement
+      end
+    end
+  end
+end

data/lib/coop_scraper/current_account.rb

@@ -0,0 +1,92 @@
+require 'rubygems'
+require 'hpricot'
+
+require 'coop_scraper/base'
+require 'ofx/statement'
+
+module CoopScraper
+  module CurrentAccount
+    class << self
+      include CoopScraper::Base
+      
+      def extract_account_number(doc)
+        doc.at("h4[text()*='CURRENT ACCOUNT']").inner_text.match(/([0-9]{8})/)[1]
+      end
+      
+      def extract_sort_code(doc)
+        doc.at("h4[text()*='CURRENT ACCOUNT']").inner_text.match(/([0-9]{2}-[0-9]{2}-[0-9]{2})/)[1].tr('-', '')
+      end
+      
+      def extract_statement_date(doc)
+        coop_date_to_time(doc.at("td[text()^='Date']").inner_text)
+      end
+      
+      def extract_transaction_rows(doc)
+        a_td = doc.at('td.transData')
+        a_td.parent.parent.search('tr')
+      end
+      
+      def determine_trntype(details)
+        case details
+        when /^DEBIT INTEREST$/
+          :interest
+        when /^LINK +[0-9]{2}:[0-9]{2}[A-Z]{3}[0-9]{2}$/
+          :atm
+        when /^SERVICE CHARGE$/
+          :service_charge
+        when /^TFR [0-9]{14}$/
+          :transfer
+        else
+          nil
+        end
+      end
+      
+      def extract_transactions(doc)
+        transactions = []
+        a_td = doc.at('td.transData')
+        transaction_rows = extract_transaction_rows(doc)
+        first_row = transaction_rows.shift
+        transaction_rows.each do |statement_row|
+          date = statement_row.at('td.dataRowL').inner_text
+          details = statement_row.at('td.transData').inner_text.strip
+          credit = statement_row.at('td.moneyData:first').inner_text.match(/[0-9.]+/)
+          debit = statement_row.search('td.moneyData')[1].inner_text.match(/[0-9.]+/)
+          amount = credit.nil? ? "-#{debit}" : credit.to_s
+          options = {}
+          trntype = determine_trntype(details)
+          options[:trntype] = trntype unless trntype.nil?
+          transactions << OFX::Statement::Transaction.new(amount, coop_date_to_time(date), details, options)
+        end
+        transactions
+      end
+      
+      def extract_closing_balance(doc)
+        final_transaction = extract_transaction_rows(doc).last.at('td.moneyData:last').inner_text
+        amount = final_transaction.match(/[0-9.]+/).to_s
+        sign = final_transaction.match(/[CD]R/).to_s
+        sign == "CR" ? amount : "-#{amount}"
+      end
+      
+      def extract_statement_start_date(doc)
+        coop_date_to_time(extract_transaction_rows(doc).first.at('td.dataRowL').inner_text)
+      end
+      
+      def generate_statement(html_statement_io, server_response_time)
+        doc = Hpricot(html_statement_io)
+        statement = OFX::Statement::CurrentAccount.new
+        
+        statement.server_response_time = server_response_time
+        statement.account_number = extract_account_number(doc)
+        statement.sort_code = extract_sort_code(doc)
+        statement.date = extract_statement_date(doc)
+        statement.ledger_balance = extract_closing_balance(doc)
+        
+        extract_transactions(doc).each { |transaction| statement << transaction }
+        
+        statement.start_date = extract_statement_start_date(doc)
+        statement.end_date = extract_statement_date(doc)
+        statement
+      end
+    end
+  end
+end

data/lib/coop_scraper/version.rb

@@ -0,0 +1,12 @@
+module CoopScraper
+  def self.Version
+    CoopScraper::Version::FULL
+  end
+  
+  module Version
+    MAJOR = 1
+    MINOR = 0
+    POINT = 1
+    FULL = [CoopScraper::Version::MAJOR, CoopScraper::Version::MINOR, CoopScraper::Version::POINT].join('.')
+  end
+end

data/lib/coop_scraper.rb

@@ -0,0 +1,2 @@
+require 'coop_scraper/credit_card'
+require 'coop_scraper/current_account'

data/lib/ofx/statement/base.rb

@@ -0,0 +1,53 @@
+require 'rubygems'
+require 'builder'
+require 'digest/sha1'
+
+module OFX
+  module Statement
+    class Base
+      attr_reader   :builder_class
+      attr_writer   :currency, :server_response_time, :language
+      attr_accessor :account_number, :start_date, :end_date, 
+                    :date, :ledger_balance
+      
+      def initialize(format = :ofx2)
+        case format
+        when :ofx1
+          @builder_class = OFX::Statement::Output::Builder::OFX1
+        when :ofx2
+          @builder_class = OFX::Statement::Output::Builder::OFX2
+        end
+      end
+      
+      def currency
+        @currency ||= 'GBP'
+      end
+      
+      def language
+        @language ||= 'ENG'
+      end
+      
+      def server_response_time
+        @server_response_time ||= Time.now
+      end
+      
+      def <<(transaction)
+        transaction.statement = self
+        transactions << transaction
+      end
+      
+      def transactions
+        @transactions ||= []
+      end
+      
+      def fitid_for(transaction)
+        index = transactions.index(transaction)
+        Digest::SHA1.hexdigest(self.date.strftime('%Y%m%d') + transaction.date.strftime('%Y%m%d') + index.to_s)
+      end
+      
+      def serialise(format = :ofx2)
+        output.new.serialise(self, format)
+      end
+    end
+  end
+end

data/lib/ofx/statement/credit_card.rb

@@ -0,0 +1,15 @@
+require 'ofx/statement/base'
+require 'ofx/statement/output/credit_card'
+
+
+module OFX
+  module Statement
+    class CreditCard < OFX::Statement::Base
+      attr_accessor :available_credit
+      
+      def output
+        OFX::Statement::Output::CreditCard
+      end
+    end
+  end
+end

data/lib/ofx/statement/current_account.rb

@@ -0,0 +1,14 @@
+require 'ofx/statement/base'
+require 'ofx/statement/output/current_account'
+
+module OFX
+  module Statement
+    class CurrentAccount < OFX::Statement::Base
+      attr_accessor :sort_code
+      
+      def output
+        OFX::Statement::Output::CurrentAccount
+      end
+    end
+  end
+end

data/lib/ofx/statement/output/base.rb

@@ -0,0 +1,131 @@
+require 'ofx/statement/output/builder'
+
+module OFX
+  module Statement
+    module Output
+      class Base
+        class << self
+          # See OFX 2.0.3 spec, section 11.4.3.1 "Transaction types used in <TRNTYPE>"
+          def trntype_hash
+            @trntype_hash ||= {
+              :debit => "DEBIT",
+              :credit => "CREDIT",
+              :interest => "INT",
+              :dividend => "DIV",
+              :fee => "FEE", 
+              :service_charge => "SRVCHG", 
+              :deposit => "DEP", 
+              :atm => "ATM", 
+              :point_of_sale => "POS", 
+              :transfer => "XFER", 
+              :cheque => "CHECK", 
+              :check => "CHECK",
+              :payment => "PAYMENT", 
+              :cash => "CASH", 
+              :direct_deposit => "DIRECTDEP",
+              :direct_debit => "DIRECTDEBIT", 
+              :repeating_payment => "REPEATPMT",
+              :standing_order => "REPEATPMT", 
+              :other => "OTHER"
+            }
+          end
+        end
+        
+        def serialise(statement, format = :ofx2)
+          builder = create_builder(format)
+          builder.ofx_stanza!
+          ofx_block(builder) do |ofx|
+            signon_block(ofx, statement) do |signon|
+              message_set_block(signon) do |message_set|
+                statement_block(message_set, statement) do |stmnt|
+                  statement.transactions.each do |transaction|
+                    transaction_block(stmnt, transaction)
+                  end
+                end
+              end
+            end
+          end
+          builder.target!
+        end
+        
+        def fitid_hash
+          @fitid_hash ||= {}
+        end
+        
+        def time_to_ofx_dta(timeobj, extended=false)
+          fmt = '%Y%m%d' + (extended ? '%H%M%S' : '')
+          timeobj.strftime(fmt)
+        end
+        
+        def generate_fitid(time)
+          date = time_to_ofx_dta(time)
+          suffix = fitid_hash[date].nil? ? 1 : fitid_hash[date] + 1
+          fitid_hash[date] = suffix
+          "#{date}#{suffix}"
+        end
+        
+        def generate_trntype(trntype)
+          output = OFX::Statement::Output::Base.trntype_hash[trntype]
+          raise UnknownTrntype, trntype unless output
+          output
+        end
+        
+        def ofx_block(node)
+          return node.OFX unless block_given?
+          
+          node.OFX { |child| yield(child) }
+        end
+        
+        def signon_block(node, statement)
+          node.SIGNONMSGSRSV1 do |signonmsgrsv1|
+            signonmsgrsv1.SONRS do |sonrs|
+              sonrs.STATUS do |status|
+                status.CODE "0"
+                status.SEVERITY "INFO"
+              end
+              sonrs.DTSERVER time_to_ofx_dta(statement.server_response_time, true)
+              sonrs.LANGUAGE statement.language
+            end
+            yield(signonmsgrsv1) if block_given?
+          end
+        end
+        
+        def ledger_balance_block(node, statement)
+          node.LEDGERBAL do |ledgerbal|
+            ledgerbal.BALAMT statement.ledger_balance
+            ledgerbal.DTASOF time_to_ofx_dta(statement.date)
+          end
+        end
+        
+        def transaction_list(node, statement)
+          node.BANKTRANLIST do |banktranlist|
+            banktranlist.DTSTART time_to_ofx_dta(statement.start_date)
+            banktranlist.DTEND time_to_ofx_dta(statement.end_date)
+            yield(banktranlist) if block_given?
+          end
+        end
+        
+        def transaction_block(node, transaction)
+          node.STMTTRN do |stmttrn|
+            stmttrn.TRNTYPE generate_trntype(transaction.trntype)
+            stmttrn.DTPOSTED time_to_ofx_dta(transaction.date)
+            stmttrn.TRNAMT transaction.amount
+            stmttrn.FITID transaction.fitid
+            stmttrn.NAME transaction.name
+            stmttrn.MEMO transaction.memo if transaction.has_memo?
+          end
+        end
+        
+        def create_builder(format = :ofx2)
+          case format
+          when :ofx1
+            OFX::Statement::Output::Builder::OFX1.new(:indent => 2)
+          when :ofx2
+            OFX::Statement::Output::Builder::OFX2.new(:indent => 2)
+          end
+        end
+      end
+    end
+  end
+end
+