bankjob 0.5.0

data/scrapers/bpi_scraper.rb

@@ -0,0 +1,190 @@
+
+require 'rubygems'
+require 'bankjob'      # this require will pull in all the classes we need
+require 'base_scraper' # this defines scraper that BpiScraper extends
+
+include Bankjob        # access the namespace of Bankjob
+
+##
+# BpiScraper is a scraper tailored to the BPI bank in Portugal (www.bpinet.pt).
+# It takes advantage of the BaseScraper to create the mechanize agent,
+# then followins the basic recipe there of first loading the tranasctions page
+# then parsing it.
+#
+# In addition to actually working for the BPI online banking, this class serves
+# as an example of how to build your own scraper.
+#
+# BpiScraper expects the user name and password to be passed on the command line
+# using -scraper_args "user password" (with a space between them).
+#
+class BpiScraper < BaseScraper
+
+  currency  "EUR" # Set the currency as euros
+  decimal   ","    # BPI statements use commas as separators - this is used by the real_amount method
+  account_number "1234567" # override this with a real accoun number
+  account_type Statement::CHECKING # this is the default anyway
+
+  # This rule detects ATM withdrawals and modifies
+  # the description and sets the the type
+  transaction_rule do |tx|
+    if (tx.real_amount < 0)
+      if tx.raw_description =~ /LEV.*ATM ELEC\s+\d+\/\d+\s+/i
+        tx.description = "Multibanco withdrawal at #{$'}"
+        tx.type = Transaction::ATM
+      end
+    end
+  end
+
+  # This rule detects checque payments and modifies the description
+  # and sets the type
+  transaction_rule do |tx|
+    if tx.raw_description =~ /CHEQUE\s+(\d+)/i
+      cheque_number = $+   # $+ holds the last group of the match which is (\d+)
+      # change the description but append $' in case there was trailing text after the cheque no
+      tx.description = "Cheque ##{cheque_number} withdrawn #{$'}"
+      tx.type = Transaction::CHECK
+      tx.check_number = cheque_number
+    end
+  end
+
+  # This rule goes last and sets the description of transactions 
+  # that haven't had their description to the raw description after
+  # changing the words to have capital letters only on the first word.
+  # (Note that +description+ will default to being the same as +raw_description+
+  #  anyway - this rule is only for making the all uppercase output less ugly)
+  # The payee is also fixed in this way
+  transaction_rule(-999) do |tx|
+    if (tx.description == tx.raw_description)
+      tx.description = Bankjob.capitalize_words(tx.raw_description)
+    end
+  end
+
+  # Some constants for the URLs and main elements in the BPI bank app
+  LOGIN_URL = 'https://www.bpinet.pt/'
+  TRANSACTIONS_URL = 'https://www.bpinet.pt/areaInf/consultas/Movimentos/Movimentos.asp'
+
+  ##
+  # Uses the mechanize web +agent+ to fetch the page holding the most recent
+  # bank transactions and returns it.
+  # This overrides (implements) +fetch_transactions_page+ in BaseScraper
+  #
+  def fetch_transactions_page(agent)
+    login(agent)
+    logger.info("Logged in, now navigating to transactions on #{TRANSACTIONS_URL}.")
+    transactions_page = agent.get(TRANSACTIONS_URL)
+    if (transactions_page.nil?)
+      raise "BPI Scraper failed to load the transactions page at #{TRANSACTIONS_URL}"
+    end
+    return transactions_page
+  end
+
+  
+  ##
+  # Parses the BPI page listing about a weeks worth of transactions
+  # and creates a Transaction for each one, putting them together
+  # in a Statement.
+  # Overrides (implements) +parse_transactions_page+ in BaseScraper.
+  #
+  def parse_transactions_page(transactions_page)
+    begin
+      statement = create_statement
+
+      # Find the closing balance avaliable and accountable
+      # Get from this:
+      #    <td valign="middle" width="135" ALIGN="left" class="TextoAzulBold">Saldo Disponível:</td>
+      #    <td valign="middle" width="110" ALIGN="right">1.751,31&nbsp;EUR</td>
+      # to 1751,31
+      available_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Dispon/ }.first.next_sibling
+      statement.closing_available = available_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
+      account_balance_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Contab/ }.first.next_sibling
+      statement.closing_balance = account_balance_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
+
+      transactions = []
+
+      # find the first header with the CSS class "Laranja" as this will be the first
+      # header in the transactions table
+      header = (transactions_page/"td.Laranja").first
+
+      # the table element is the grandparent element of this header (the row is the parent)
+      table = header.parent.parent 
+
+      # each row with the valign attribute set to "top" holds a transaction
+      rows = (table/"tr[@valign=top]")
+      rows.each do |row|
+        transaction = create_transaction # use the support method because it sets the separator
+
+        # collect all of the table cells' inner html in an array (stripping leading/trailing spaces)
+        data = (row/"td").collect{ |cell| cell.inner_html.strip }
+
+        # the first (0th) column holds the date 
+        transaction.date = data[0]
+
+        # the 2nd column holds the value date - but it's often empty 
+        # in which case we set it to nil
+        vdate = data[1]
+        if vdate.nil? or vdate.length == 0 or vdate.strip == "&nbsp;"
+          transaction.value_date = nil
+        else
+          transaction.value_date = vdate
+        end
+
+        # the transaction raw_description is in the 3rd column
+        transaction.raw_description = data[2]
+
+        # the 4th column holds the transaction amount (with comma as decimal place)
+        transaction.amount = data[3]
+
+        # the new balance is in the last column
+        transaction.new_balance=data[4]
+
+        # add thew new transaction to the array
+        transactions << transaction
+        #	break if $debug
+      end
+    rescue => exception
+      msg = "Failed to parse the transactions page at due to exception: #{exception.message}\nCheck your user name and password."
+      logger.fatal(msg);
+      logger.debug(exception)
+      logger.debug("Failed parsing transactions page:")
+      logger.debug("--------------------------------")
+      logger.debug(transactions_page) #.body
+      logger.debug("--------------------------------")
+      abort(msg)
+    end
+
+    # set the transactions on the statement
+    statement.transactions = transactions
+    return statement
+  end
+
+  ##
+  # Logs into the BPI banking app by finding the form
+  # setting the name and password and submitting it then
+  # waits a bit.
+  #
+  def login(agent)
+    logger.info("Logging in to #{LOGIN_URL}.")
+    if (scraper_args)
+      username, password = *scraper_args
+    end
+    raise "Login failed for BPI Scraper - pass user name and password using -scraper_args \"user <space> pass\"" unless (username and password)
+
+    # navigate to the login page
+    login_page = agent.get(LOGIN_URL)
+
+    # find login form - it's called 'signOn' - fill it out and submit it
+    form  = login_page.form('signOn')
+
+    # username and password are taken from the commandline args, set them
+    # on USERID and PASSWORD which are the element names that the web page
+    # form uses to identify the form fields
+    form.USERID = username
+    form.PASSWORD = password
+
+    # submit the form - same as the user hitting the Login button
+    agent.submit(form)
+    sleep 3  # wait while the login takes effect
+  end
+end # class BpiScraper
+
+

data/spec/bankjob_cli_spec.rb

@@ -0,0 +1,15 @@
+equire File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'bankjob/cli'
+
+describe Bankjob::CLI, "execute" do
+  before(:each) do
+    @stdout_io = StringIO.new
+    Bankjob::CLI.execute(@stdout_io, [])
+    @stdout_io.rewind
+    @stdout = @stdout_io.read
+  end
+  
+  it "should do something" do
+    @stdout.should_not =~ /To update this executable/
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+begin
+  require 'spec'
+rescue LoadError
+  require 'rubygems'
+  gem 'rspec'
+  require 'spec'
+end
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'bankjob.rb'

data/spec/statement_spec.rb

@@ -0,0 +1,121 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+#require File.expand_path(File.dirname(__FILE__) + '/../lib/bankjob.rb')
+ 
+include Bankjob
+
+# Test the Statement merging in particular
+describe Statement do
+  before(:each) do
+
+    @tx1 = Transaction.new(",")
+    @tx1.date = "20080730000000"
+    @tx1.value_date = "20080731145906"
+    @tx1.raw_description = "1 Stamp duty 001"
+    @tx1.amount = "-2,40"
+    @tx1.new_balance = "1.087,43"
+
+
+    @tx2 = Transaction.new(",")
+    @tx2.date = "0080729000000"
+    @tx2.value_date = "20080731145906"
+    @tx2.raw_description =  "2 Interest payment 001"
+    @tx2.amount = "-59,94"
+    @tx2.new_balance = "1.089,83"
+
+
+    @tx3 = Transaction.new(",")
+    @tx3.date = "20080208000000"
+    @tx3.value_date = "20080731145906"
+    @tx3.raw_description =  "3 Load payment 001"
+    @tx3.amount = "-256,13"
+    @tx3.new_balance = "1.149,77"
+
+
+    @tx4 = Transaction.new(",")
+    @tx4.date = "20080207000000"
+    @tx4.value_date = "20080731145906"
+    @tx4.raw_description =  "4 Transfer to bank 2"
+    @tx4.amount = "-1.000,00"
+    @tx4.new_balance = "1.405,90"
+
+
+    @tx5 = Transaction.new(",")
+    @tx5.date = "20080209000000"
+    @tx5.value_date = "20080731145906"
+    @tx5.raw_description =  "5 Internet payment 838"
+    @tx5.amount = "-32,07"
+    @tx5.new_balance = "1.405,90"
+
+    # the lot
+    @s12345 = Statement.new
+    @s12345.transactions = [ @tx1.dup, @tx2.dup, @tx3.dup, @tx4.dup, @tx5.dup]
+
+    # first 2
+    @s12 = Statement.new
+    @s12.transactions = [ @tx1.dup, @tx2.dup]
+
+    # middle 1
+    @s3 = Statement.new
+    @s3.transactions = [ @tx3.dup]
+
+    # last 2
+    @s45 = Statement.new
+    @s45.transactions = [ @tx4.dup, @tx5.dup]
+    
+    # first 3
+    @s123 = Statement.new
+    @s123.transactions = [ @tx1.dup, @tx2.dup, @tx3.dup]
+
+    # last 4, overlaps with 23 of s123
+    @s2345 = Statement.new
+    @s2345.transactions = [ @tx2.dup, @tx3.dup, @tx4.dup, @tx5.dup]
+
+    # 2nd and last - overlaps non-contiguously with s123
+    @s25 = Statement.new
+    @s25.transactions = [ @tx2.dup, @tx5.dup]
+    
+  end
+
+  it "should merge consecutive satements properly" do
+    @s123.merge(@s45).should == @s12345
+  end
+
+  it "should merge overlapping statments properly" do
+    #@s123.merge(@s2345).transactions.each { |tx| print "#{tx.to_s}, "}
+    @s123.merge(@s2345).should == @s12345
+  end
+
+  it "should merge a statement with a duplicate of itself without changing it" do
+    @s123.merge(@s123.dup).should == @s123
+  end
+
+
+  it "should merge non-contiguous with an error" do
+    m =  @s123.merge(@s25)
+    m.transactions.each { |tx| print "#{tx.to_s}, "}
+  end
+
+  it "should read back a satement from csv as it was written" do
+    csv = @s123.to_csv
+    statement = Statement.new()
+    statement.from_csv(csv, ",")
+    statement.should == @s123
+  end
+
+  it "should read back and merge a statement with itself without change" do
+    csv = @s123.to_csv
+    statement = Statement.new()
+    statement.from_csv(csv, ",")
+    m = @s123.merge(statement)
+    m.should == @s123
+  end
+
+  it "should write, read, merge and write a statement without changing it" do
+    csv = @s123.to_csv
+    statement = Statement.new()
+    m = @s123.merge(statement)
+    m_csv = m.to_csv
+    m_csv.should == csv
+  end
+end
+

data/spec/transaction_spec.rb

@@ -0,0 +1,81 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+require 'bankjob.rb'
+include Bankjob
+
+describe Transaction do
+  before(:each) do
+    @tx1 = Transaction.new()
+    @tx1.date = "30-7-2008"
+    @tx1.value_date = "20080731145906" 
+    @tx1.raw_description = "Some tax thing 10493"
+    @tx1.amount = "-2,40"
+    @tx1.new_balance = "1.087,43"
+
+    @tx1_copy = Transaction.new()
+    @tx1_copy.date = "30-7-2008"
+    @tx1_copy.value_date = "20080731145906" 
+    @tx1_copy.raw_description = "Some tax thing 10493"
+    @tx1_copy.amount = "-2,40"
+    @tx1_copy.new_balance = "1.087,43"
+  
+    @tx1_dup = @tx1.dup
+
+    @tx2 = Transaction.new()
+    @tx2.date = "0080729000000"
+    @tx2.value_date = "20080731145906" 
+    @tx2.raw_description = "Interest payment"
+    @tx2.amount = "-59,94"
+    @tx2.new_balance = "1.089,83"
+  end
+
+  it "should generate the same ofx_id as its copy" do
+    puts "tx1: #{@tx1.to_s}\n-----"
+    puts "tx1_copy: #{@tx1.to_s}"
+    @tx1.ofx_id.should == @tx1_copy.ofx_id
+    puts "#{@tx1.ofx_id} == #{@tx1_copy.ofx_id}"
+  end
+
+  it "should generate the same ofx_id as its duplicate" do
+    @tx1.ofx_id.should == @tx1_dup.ofx_id
+  end
+
+
+  it "should be == to its duplicate" do
+    @tx1.should == @tx1_dup
+  end
+
+  it "should be == to its identical copy" do
+    @tx1.should == @tx1_copy
+  end
+
+  it "should not == a different transaction" do
+    @tx1.should_not == @tx2
+  end
+
+  it "should be eql to its duplicate (necessary for merging)"  do
+    @tx1.should eql(@tx1_dup)
+  end
+
+  it "should not be equal to its duplicate" do
+    @tx1.should_not equal(@tx1_dup)
+  end
+
+  it "should be === to its duplicate" do
+    @tx1.should === @tx1_dup
+  end
+
+  it "should have the same hash as its duplicate" do
+    @tx1.hash.should == @tx1_dup.hash
+  end
+
+  it "should convert 1,000,000.32 to 1000000.32 when decimal separator is ." do
+   Bankjob.string_to_float("1,000,000.32", ".").should == 1000000.32
+  end
+
+  it "should convert 1.000.000,32 to 1000000.32 when decimal separator is ," do
+   Bankjob.string_to_float("1.000.000,32", ",").should == 1000000.32
+  end
+
+end
+

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification 
+name: bankjob
+version: !ruby/object:Gem::Version 
+  version: 0.5.0
+platform: ruby
+authors: 
+- rhubarb
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-11 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0.6"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: mechanize
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.7.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: builder
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.2
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.3
+    version: 
+description: Bankjob is a command-line ruby program for scraping online banking sites and producing statements in OFX (Open Fincancial Exchange) or CSV (Comma Separated Values) formats.  Bankjob was created for people like me who want to get their bank data into a 3rd party application but whose bank does not support downloads in OFX format. It's also useful for keeping a permanent store of bank statements on your computer for reading in Excel (vs filing paper statements)
+email: 
+- rhubarb.bankjob@gmail.com
+executables: 
+- bankjob
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- PostInstall.txt
+- README.rdoc
+files: 
+- History.txt
+- PostInstall.txt
+- README.rdoc
+- bin/bankjob
+- lib/bankjob.rb
+- lib/bankjob/bankjob_runner.rb
+- lib/bankjob/cli.rb
+- lib/bankjob/payee.rb
+- lib/bankjob/scraper.rb
+- lib/bankjob/statement.rb
+- lib/bankjob/support.rb
+- lib/bankjob/transaction.rb
+- scrapers/base_scraper.rb
+- scrapers/bpi_scraper.rb
+- spec/bankjob_cli_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/statement_spec.rb
+- spec/transaction_spec.rb
+has_rdoc: true
+homepage: http://bankjob.rubyforge.org/
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: bankjob
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Bankjob is a command-line ruby program for scraping online banking sites and producing statements in OFX (Open Fincancial Exchange) or CSV (Comma Separated Values) formats
+test_files: []
+