bankjob 0.5.1 → 0.5.2

data/History.txt

@@ -1,4 +1,11 @@
-== 0.5.1 2009-04-20
+== 0.5.2 2009-05-18
+* 3 minor enhancements:
+  * Added the ability to set fake timestamps on transactions so that transactions that have 00:00 for their time stamp that occur on the same day will be in the correct order in Wesabe
+    To use this in your scraper, call statement#finish(true, true)
+  * bpi_scraper.rb now sets the account number on the statement after scraping it. It turns out this is important for uploading multiple accounts to Wesabe - the accounts get mixed if the account numbers are wrong
+  * the Statement#finish method also updates the balance and start and end dates of the statement after the last transaction is scraped. This used to be done automatically in the attribute getters but did not support statements with most-recent transaction last (only first).
+
+=== 0.5.1 2009-04-20
 * 1 minor enhancement:
   * bpi_scraper.rb now accepts an optional third argument for the account number. Entering a number will cause the scraper to scrape that account rather than the default account.
 

data/lib/bankjob.rb

@@ -8,5 +8,5 @@ require 'bankjob/scraper.rb'
 require 'bankjob/payee.rb'
 
 module Bankjob
-  BANKJOB_VERSION = '0.5.1' unless defined?(BANKJOB_VERSION)
+  BANKJOB_VERSION = '0.5.2' unless defined?(BANKJOB_VERSION)
 end

data/lib/bankjob/bankjob_runner.rb

@@ -12,7 +12,7 @@ module Bankjob
       logger = options.logger
 
       if options.wesabe_help
-        Bankjob.wesabe_help(options.wesabe_args)
+        Bankjob.wesabe_help(options.wesabe_args, logger)
         exit(0) # Wesabe help describes to the user how to use the wesabe options then quits
       end
 
@@ -181,4 +181,4 @@ NEWFILEUID:NONE
       return output_file
     end
   end # class BankjobRunner
-end # module Bankjob
+end # module Bankjob

data/lib/bankjob/scraper.rb

@@ -69,6 +69,13 @@ module Bankjob
   #                      organized Transaction objects with clearer descriptions of the 
   #                      transaction, etc.
   #
+  # +finish+ :: finishes a transaction by setting the balances and to and from dates
+  #             based on the first and last transactions. Also, optionally, generates
+  #             fake timestamps for transactions that have no time component in their
+  #             dates. This is important for clients that use the timestamps to order
+  #             the transactions correctly, and would otherwise mess up the order
+  #             if all transactions on the same day were at the same time (E.g. Wesabe)
+  #
   # Here is an example of a simple (but incomplete) scraper.
   # Note that all of the scraping and parsing is in the +scrape_statement+ method, although
   # a lot of the details of Hpricot parsing are left up to the imagination of the reader.

data/lib/bankjob/statement.rb

@@ -67,6 +67,14 @@ module Bankjob
     # Use a constant to set this - defaults to CHECKING
     attr_accessor :account_type
 
+    # the last date of the period the statement covers
+    # Translates to the OFX element DTEND
+    attr_accessor :to_date
+
+    # the first date of the period the statement covers
+    # Translates to the OFX element DTSTART
+    attr_accessor :from_date
+
     ##
     # Creates a new empty Statement with no transactions.
     # The +account_number+ must be specified as a 1-22 character string.
@@ -77,6 +85,8 @@ module Bankjob
       @currency = currency
       @transactions = []
       @account_type = CHECKING
+      @closing_balance = nil
+      @closing_available = nil
     end
     
     ##
@@ -148,45 +158,6 @@ module Bankjob
       @transactions = merge_transactions(other)
     end
 
-    ##
-    # Returns the statement's start date.
-    # The +from_date+ is taken from the date of the last transaction in the statement
-    #
-    def from_date()
-      return nil if @transactions.empty?
-      @transactions.last.date
-    end
-
-    ##
-    # Returns the statement's end date.
-    # The +to_date+ is taken from the date of the first transaction in the statement
-    #
-    def to_date()
-      return nil if @transactions.empty?
-      @transactions.first.date
-    end
-
-    ##
-    # Returns the closing balance by looking at the
-    # new balance of the first transaction.
-    # If there are no transactions, +nil+ is returned.
-    #
-    def closing_balance()
-      return nil if @closing_balance.nil? and @transactions.empty?
-      @closing_balance ||= @transactions.first.new_balance
-    end
-
-    ##
-    # Returns the closing available balance by looking at the
-    # new balance of the first transaction.
-    # If there are no transactions, +nil+ is returned.
-    # Note that this is the same value returned as +closing_balance+.
-    #
-    def closing_available()
-      return nil if @closing_available.nil? and @transactions.empty?
-      @closing_available ||= @transactions.first.new_balance
-    end
-
     ##
     # Generates a CSV (comma separated values) string with a single
     # row for each transaction.
@@ -342,6 +313,112 @@ module Bankjob
       }
       return buf
     end
+    
+    ONE_MINUTE = 60
+    ELEVEN_59_PM = 23 * 60 * 60 + 59 * 60  # seconds at 23:59
+    MIDDAY = 12 * 60 * 60
+
+    ##
+    # Finishes the statement after scraping in two ways depending on the information
+    # that the scraper was able to obtain. Optionally have your scraper class call
+    # this after scraping is finished.
+    #
+    # This method:
+    # 
+    # 1. Sets the closing balance and available_balance and the to_ and from_dates
+    #    by using the first and last transactions in the list. Which transaction is
+    #    used depends on whether +most_recent_first+ is true or false.
+    #    The scraper may just set these directly in which case this may not be necessary.
+    #
+    # 2. If +fake_times+ is true time-stamps are invented and added to the transaction
+    #    date attributes. This is useful if the website beings scraped shows dates, but
+    #    not times, but has transactions listed in chronoligical arder. 
+    #    Without this process, the ofx generated has no proper no indication of the order of
+    #    transactions that occurred in the same day other than the order in the statement
+    #    and this may be ignored by the client. (Specifically, Wesabe will reorder transactions
+    #    in the same day if they all appear to occur at the same time).
+    #    
+    #    Note that the algorithm to set the fake times is a little tricky. Assuming
+    #    the transactionsa are most-recent-first, the first last transaction on each 
+    #    day is set at 11:59pm each transaction prior to that is one minute earlier.
+    #    
+    #    But for the first transactions in the statement, the first is set at a few
+    #    minutes after midnight, then we count backward. (The actual number of minutes
+    #    is based on the number of transactions + 1 to be sure it doesnt pass midnight)
+    #    
+    #    This is crucial because transactions for a given day will often span 2 or more
+    #    statement. By starting just after midnight and going back to just before midnight
+    #    we reduce the chance of overlap.
+    #
+    #    If the to-date is the same as the from-date for a transaction, then we start at
+    #    midday, so that prior and subsequent statements don't overlap.
+    #
+    #   This simple algorithm basically guarantees no overlaps so long as:
+    #   i.  The number of transactions is small compared to the number of minutes in a day
+    #   ii. A single day will not span more than 3 statements
+    #
+    #   If the statement is most-recent-last (+most_recent_first = false+) the same
+    #   algorithm is applied, only in reverse
+    #
+    def finish(most_recent_first, fake_times=false)
+      if !@transactions.empty? then
+        # if the user hasn't set the balances, set them to the first or last
+        # transaction balance depending on the order
+        if most_recent_first then
+          @closing_balance ||= transactions.first.new_balance
+          @closing_available ||= transactions.first.new_balance
+          @to_date ||= transactions.first.date
+          @from_date ||= transactions.last.date
+        else
+          @closing_balance ||= transactions.last.new_balance
+          @closing_available ||= transactions.last.new_balance
+          @to_date ||= transactions.last.date
+          @from_date ||= transactions.first.date
+        end
+
+        if fake_times and to_date.hour == 0 then
+          # the statement was unable to scrape times to go with the dates, but the
+          # client (say wesabe) will get the transaction order wrong if there are no
+          # times, so here we add times that order the transactions according to the
+          # order of the array of transactions
+
+          # the delta is 1 minute forward or backward fr
+          if to_date == from_date then
+            # all of the statement's transactions occur in the same day - to try to
+            # avoid overlap with subsequent or previous transacitons we group order them
+            # from 11am onward
+            seconds = MIDDAY
+          else
+            seconds = (transactions.length + 1) * 60
+          end
+
+          if most_recent_first then
+            yday = transactions.first.date.yday
+            start = 0
+            delta = 1
+            finish = transactions.length
+          else
+            yday = transactions.last.date.yday
+            start = transactions.length - 1
+            finish = -1
+            delta = -1
+          end
+
+          i = start
+          until i == finish
+            tx = transactions[i]
+            if tx.date.yday != yday
+              # starting a new day, begin the countdown from 23:59 again
+              yday = tx.date.yday
+              seconds = ELEVEN_59_PM
+            end
+            tx.date += seconds unless tx.date.hour > 0
+            seconds -= ONE_MINUTE
+            i += delta
+          end
+        end
+      end
+    end
 
     def to_s
       buf = "#{self.class}: close_bal = #{closing_balance}, avail = #{closing_available}, curr = #{currency}, transactions:"
@@ -352,4 +429,4 @@ module Bankjob
       return buf
     end
   end # class Statement
-end # module
+end # module

data/lib/bankjob/support.rb

@@ -6,7 +6,7 @@ module Bankjob
 
   ##
   # Takes a date-time as a string or as a Time or DateTime object and returns
-  # it as either a Time or a DateTime object.
+  # it as either a Time object.
   #
   # This is useful in the setter method of a date attribute allowing the date
   # to be set as any type but stored internally as an object compatible with
@@ -14,8 +14,8 @@ module Bankjob
   # (Bankjob::Transaction uses this internally in the setter for +date+ for example
   #
   def self.create_date_time(date_time_raw)
-    if (date_time_raw.respond_to?(:rfc822)) then
-      # It's already a Time or DateTime
+    if (date_time_raw.is_a?(Time)) then
+      # It's already a Time 
       return date_time_raw
     elsif (date_time_raw.to_s.strip.empty?)
       # Nil or non dates are returned as nil
@@ -167,7 +167,7 @@ module Bankjob
   # When used with Wesabe account and password, will log into Wesabe and list
   # the users accounts, and suggest command line args to upload to each account.
   #
-  def self.wesabe_help(wesabe_args)
+  def self.wesabe_help(wesabe_args, logger)
     if (wesabe_args.nil? or wesabe_args.length != 2)
       puts <<-EOF
 Wesabe (http://www.wesabe.com) is an online bank account management tool (like Mint)
@@ -240,12 +240,15 @@ Troubleshooting:
           end
         end
       rescue Exception => e
-        raise <<-EOF
+        msg =<<-EOF
 Failed to get Wesabe account information due to: #{e.message}.
 Check your username and password or use:
   bankjob --wesabe-help
 with no arguments for more details.
         EOF
+	logger.debug(msg)
+	logger.debug(e)
+	raise msg
       end
     end
   end # wesabe_help

data/scrapers/bpi_scraper.rb

@@ -1,205 +1,234 @@
-
-require 'rubygems'
-require 'bankjob'      # this require will pull in all the classes we need
-require 'base_scraper' # this defines scraper that BpiScraper extends
-
-include Bankjob        # access the namespace of Bankjob
-
-##
-# BpiScraper is a scraper tailored to the BPI bank in Portugal (www.bpinet.pt).
-# It takes advantage of the BaseScraper to create the mechanize agent,
-# then followins the basic recipe there of first loading the tranasctions page
-# then parsing it.
-#
-# In addition to actually working for the BPI online banking, this class serves
-# as an example of how to build your own scraper.
-#
-# BpiScraper expects the user name and password to be passed on the command line
-# using --scraper-args "user password" (with a space between them).
-# Optionally, the account number can also be specified with the 3rd argument so:
-# --scraper-args "user password 803030000001" causing that account to be selected
-# before scraping the statement
-#
-class BpiScraper < BaseScraper
-
-  currency  "EUR" # Set the currency as euros
-  decimal   ","    # BPI statements use commas as separators - this is used by the real_amount method
-  account_number "1234567" # override this with a real accoun number
-  account_type Statement::CHECKING # this is the default anyway
-
-  # This rule detects ATM withdrawals and modifies
-  # the description and sets the the type
-  transaction_rule do |tx|
-    if (tx.real_amount < 0)
-      if tx.raw_description =~ /LEV.*ATM ELEC\s+\d+\/\d+\s+/i
-        tx.description = "Multibanco withdrawal at #{$'}"
-        tx.type = Transaction::ATM
-      end
-    end
-  end
-
-  # This rule detects checque payments and modifies the description
-  # and sets the type
-  transaction_rule do |tx|
-    if tx.raw_description =~ /CHEQUE\s+(\d+)/i
-      cheque_number = $+   # $+ holds the last group of the match which is (\d+)
-      # change the description but append $' in case there was trailing text after the cheque no
-      tx.description = "Cheque ##{cheque_number} withdrawn #{$'}"
-      tx.type = Transaction::CHECK
-      tx.check_number = cheque_number
-    end
-  end
-
-  # This rule goes last and sets the description of transactions 
-  # that haven't had their description to the raw description after
-  # changing the words to have capital letters only on the first word.
-  # (Note that +description+ will default to being the same as +raw_description+
-  #  anyway - this rule is only for making the all uppercase output less ugly)
-  # The payee is also fixed in this way
-  transaction_rule(-999) do |tx|
-    if (tx.description == tx.raw_description)
-      tx.description = Bankjob.capitalize_words(tx.raw_description)
-    end
-  end
-
-  # Some constants for the URLs and main elements in the BPI bank app
-  LOGIN_URL = 'https://www.bpinet.pt/'
-  TRANSACTIONS_URL = 'https://www.bpinet.pt/areaInf/consultas/Movimentos/Movimentos.asp'
-
-  ##
-  # Uses the mechanize web +agent+ to fetch the page holding the most recent
-  # bank transactions and returns it.
-  # This overrides (implements) +fetch_transactions_page+ in BaseScraper
-  #
-  def fetch_transactions_page(agent)
-    login(agent)
-    logger.info("Logged in, now navigating to transactions on #{TRANSACTIONS_URL}.")
-    transactions_page = agent.get(TRANSACTIONS_URL)
-    if (transactions_page.nil?)
-      raise "BPI Scraper failed to load the transactions page at #{TRANSACTIONS_URL}"
-    end
-
-    # If there is a third scraper arg, it is the account number and we use it
-    # to select the account on the transactions page
-    if (scraper_args and scraper_args.length > 2)
-      account = scraper_args[2]
-      # the account selector is the field 'contaCorrente' in the form 'form_mov'
-      Bankjob.select_and_submit(transactions_page, 'form_mov', 'contaCorrente', account)
-      sleep 1
-      # refetch the transactions page after selecting the account
-      transactions_page = agent.get(TRANSACTIONS_URL)
-    end
-
-    return transactions_page
-  end
-
-  
-  ##
-  # Parses the BPI page listing about a weeks worth of transactions
-  # and creates a Transaction for each one, putting them together
-  # in a Statement.
-  # Overrides (implements) +parse_transactions_page+ in BaseScraper.
-  #
-  def parse_transactions_page(transactions_page)
-    begin
-      statement = create_statement
-
-      # Find the closing balance avaliable and accountable
-      # Get from this:
-      #    <td valign="middle" width="135" ALIGN="left" class="TextoAzulBold">Saldo Disponível:</td>
-      #    <td valign="middle" width="110" ALIGN="right">1.751,31&nbsp;EUR</td>
-      # to 1751,31
-      available_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Dispon/ }.first.next_sibling
-      statement.closing_available = available_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
-      account_balance_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Contab/ }.first.next_sibling
-      statement.closing_balance = account_balance_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
-
-      transactions = []
-
-      # find the first header with the CSS class "Laranja" as this will be the first
-      # header in the transactions table
-      header = (transactions_page/"td.Laranja").first
-
-      # the table element is the grandparent element of this header (the row is the parent)
-      table = header.parent.parent 
-
-      # each row with the valign attribute set to "top" holds a transaction
-      rows = (table/"tr[@valign=top]")
-      rows.each do |row|
-        transaction = create_transaction # use the support method because it sets the separator
-
-        # collect all of the table cells' inner html in an array (stripping leading/trailing spaces)
-        data = (row/"td").collect{ |cell| cell.inner_html.strip }
-
-        # the first (0th) column holds the date 
-        transaction.date = data[0]
-
-        # the 2nd column holds the value date - but it's often empty 
-        # in which case we set it to nil
-        vdate = data[1]
-        if vdate.nil? or vdate.length == 0 or vdate.strip == "&nbsp;"
-          transaction.value_date = nil
-        else
-          transaction.value_date = vdate
-        end
-
-        # the transaction raw_description is in the 3rd column
-        transaction.raw_description = data[2]
-
-        # the 4th column holds the transaction amount (with comma as decimal place)
-        transaction.amount = data[3]
-
-        # the new balance is in the last column
-        transaction.new_balance=data[4]
-
-        # add thew new transaction to the array
-        transactions << transaction
-        #	break if $debug
-      end
-    rescue => exception
-      msg = "Failed to parse the transactions page at due to exception: #{exception.message}\nCheck your user name and password."
-      logger.fatal(msg);
-      logger.debug(exception)
-      logger.debug("Failed parsing transactions page:")
-      logger.debug("--------------------------------")
-      logger.debug(transactions_page) #.body
-      logger.debug("--------------------------------")
-      abort(msg)
-    end
-
-    # set the transactions on the statement
-    statement.transactions = transactions
-    return statement
-  end
-
-  ##
-  # Logs into the BPI banking app by finding the form
-  # setting the name and password and submitting it then
-  # waits a bit.
-  #
-  def login(agent)
-    logger.info("Logging in to #{LOGIN_URL}.")
-    if (scraper_args)
-      username, password = *scraper_args
-    end
-    raise "Login failed for BPI Scraper - pass user name and password using -scraper_args \"user <space> pass\"" unless (username and password)
-
-    # navigate to the login page
-    login_page = agent.get(LOGIN_URL)
-
-    # find login form - it's called 'signOn' - fill it out and submit it
-    form  = login_page.form('signOn')
-
-    # username and password are taken from the commandline args, set them
-    # on USERID and PASSWORD which are the element names that the web page
-    # form uses to identify the form fields
-    form.USERID = username
-    form.PASSWORD = password
-
-    # submit the form - same as the user hitting the Login button
-    agent.submit(form)
-    sleep 3  # wait while the login takes effect
-  end
-end # class BpiScraper
-
-
+
+require 'rubygems'
+require 'bankjob'      # this require will pull in all the classes we need
+require 'base_scraper' # this defines scraper that BpiScraper extends
+
+include Bankjob        # access the namespace of Bankjob
+
+##
+# BpiScraper is a scraper tailored to the BPI bank in Portugal (www.bpinet.pt).
+# It takes advantage of the BaseScraper to create the mechanize agent,
+# then followins the basic recipe there of first loading the tranasctions page
+# then parsing it.
+#
+# In addition to actually working for the BPI online banking, this class serves
+# as an example of how to build your own scraper.
+#
+# BpiScraper expects the user name and password to be passed on the command line
+# using --scraper-args "user password" (with a space between them).
+# Optionally, the account number can also be specified with the 3rd argument so:
+# --scraper-args "user password 803030000001" causing that account to be selected
+# before scraping the statement
+#
+class BpiScraper < BaseScraper
+
+  currency  "EUR" # Set the currency as euros
+  decimal   ","    # BPI statements use commas as separators - this is used by the real_amount method
+  account_number "1234567" # override this with a real account number
+  account_type Statement::CHECKING # this is the default anyway
+
+  # This rule detects ATM withdrawals and modifies
+  # the description and sets the the type
+  transaction_rule do |tx|
+    if (tx.real_amount < 0)
+      if tx.raw_description =~ /LEV.*ATM ELEC\s+\d+\/\d+\s+/i
+        tx.description = "Multibanco withdrawal at #{$'}"
+        tx.type = Transaction::ATM
+      end
+    end
+  end
+
+  # This rule detects checque payments and modifies the description
+  # and sets the type
+  transaction_rule do |tx|
+    if tx.raw_description =~ /CHEQUE\s+(\d+)/i
+      cheque_number = $+   # $+ holds the last group of the match which is (\d+)
+      # change the description but append $' in case there was trailing text after the cheque no
+      tx.description = "Cheque ##{cheque_number} withdrawn #{$'}"
+      tx.type = Transaction::CHECK
+      tx.check_number = cheque_number
+    end
+  end
+
+  # This rule goes last and sets the description of transactions 
+  # that haven't had their description to the raw description after
+  # changing the words to have capital letters only on the first word.
+  # (Note that +description+ will default to being the same as +raw_description+
+  #  anyway - this rule is only for making the all uppercase output less ugly)
+  # The payee is also fixed in this way
+  transaction_rule(-999) do |tx|
+    if (tx.description == tx.raw_description)
+      tx.description = Bankjob.capitalize_words(tx.raw_description)
+    end
+  end
+
+  # Some constants for the URLs and main elements in the BPI bank app
+  LOGIN_URL = 'https://www.bpinet.pt/'
+  TRANSACTIONS_URL = 'https://www.bpinet.pt/areaInf/consultas/Movimentos/Movimentos.asp'
+
+  ##
+  # Uses the mechanize web +agent+ to fetch the page holding the most recent
+  # bank transactions and returns it.
+  # This overrides (implements) +fetch_transactions_page+ in BaseScraper
+  #
+  def fetch_transactions_page(agent)
+    login(agent)
+    logger.info("Logged in, now navigating to transactions on #{TRANSACTIONS_URL}.")
+    transactions_page = agent.get(TRANSACTIONS_URL)
+    if (transactions_page.nil?)
+      raise "BPI Scraper failed to load the transactions page at #{TRANSACTIONS_URL}"
+    end
+
+    # If there is a third scraper arg, it is the account number and we use it
+    # to select the account on the transactions page
+    if (scraper_args and scraper_args.length > 2)
+      account = scraper_args[2]
+      # the account selector is the field 'contaCorrente' in the form 'form_mov'
+      Bankjob.select_and_submit(transactions_page, 'form_mov', 'contaCorrente', account)
+      sleep 1
+      # refetch the transactions page after selecting the account
+      transactions_page = agent.get(TRANSACTIONS_URL)
+    end
+
+    return transactions_page
+  end
+
+  
+  ##
+  # Parses the BPI page listing about a weeks worth of transactions
+  # and creates a Transaction for each one, putting them together
+  # in a Statement.
+  # Overrides (implements) +parse_transactions_page+ in BaseScraper.
+  #
+  def parse_transactions_page(transactions_page)
+    begin
+      statement = create_statement
+
+      account_number = get_account_number(transactions_page)
+      statement.account_number = account_number unless account_number.nil?
+      
+      # Find the closing balance avaliable and accountable
+      # Get from this:
+      #    <td valign="middle" width="135" ALIGN="left" class="TextoAzulBold">Saldo Disponível:</td>
+      #    <td valign="middle" width="110" ALIGN="right">1.751,31&nbsp;EUR</td>
+      # to 1751,31
+      # Commenting out balances for now to let the balance be taken from the 
+      # top-most transaction - this keeps balances in synch with actual transactions
+      # and allows for statements created for past dates (the balance at the top of the
+      # page is always the current one, not the one for the last transaction on that page)
+      #available_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Dispon/ }.first.next_sibling
+      #statement.closing_available = available_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
+      #account_balance_cell = (transactions_page/"td").select { |ele| ele.inner_text =~ /^Saldo Contab/ }.first.next_sibling
+      #statement.closing_balance = account_balance_cell.inner_text.scan(/[\d.,]+/)[0].gsub(/\./,"")
+
+      #transactions = []
+
+      # find the first header with the CSS class "Laranja" as this will be the first
+      # header in the transactions table
+      header = (transactions_page/"td.Laranja").first
+
+      # the table element is the grandparent element of this header (the row is the parent)
+      table = header.parent.parent 
+
+      # each row with the valign attribute set to "top" holds a transaction
+      rows = (table/"tr[@valign=top]")
+      rows.each do |row|
+        transaction = create_transaction # use the support method because it sets the separator
+
+        # collect all of the table cells' inner html in an array (stripping leading/trailing spaces)
+        data = (row/"td").collect{ |cell| cell.inner_html.strip }
+
+        # the first (0th) column holds the date 
+        transaction.date = data[0]
+
+        # the 2nd column holds the value date - but it's often empty 
+        # in which case we set it to nil
+        vdate = data[1]
+        if vdate.nil? or vdate.length == 0 or vdate.strip == "&nbsp;"
+          transaction.value_date = nil
+        else
+          transaction.value_date = vdate
+        end
+
+        # the transaction raw_description is in the 3rd column
+        transaction.raw_description = data[2]
+
+        # the 4th column holds the transaction amount (with comma as decimal place)
+        transaction.amount = data[3]
+
+        # the new balance is in the last column
+        transaction.new_balance=data[4]
+
+        # add thew new transaction to the array
+        statement.add_transaction(transaction)
+        #	break if $debug
+      end
+    rescue => exception
+      msg = "Failed to parse the transactions page at due to exception: #{exception.message}\nCheck your user name and password."
+      logger.fatal(msg);
+      logger.debug(exception)
+      logger.debug("Failed parsing transactions page:")
+      logger.debug("--------------------------------")
+      logger.debug(transactions_page) #.body
+      logger.debug("--------------------------------")
+      abort(msg)
+    end
+
+    # finish the statement to set the balances and dates
+    # and to fake the times since the bpi web pages
+    # don't hold the transaction times
+    statement.finish(true, true) # most_recent_first, fake_times
+
+    return statement
+  end
+
+  def get_account_number(transactions_page)
+    # make sure the page is a mechanize page, not hpricot
+    if transactions_page.kind_of?(Hpricot::Doc) then 
+      page = WWW::Mechanize::Page.new(nil, {'content-type'=>'text/html'},
+        transactions_page.html, nil, nil)
+    else
+      page = transactions_page
+    end
+
+    # find the form for selecting an account  -it's called 'form_mov' 
+    form_mov  = page.form('form_mov')
+    # the field for selecting the current account is in this form
+    account_selector = form_mov.field('contaCorrente')
+    # the selected account value is the account number but it has "|NR|" on the end so strip
+    # everything that's not a number
+    account_number = account_selector.value.gsub(/[^0-9]/,"")
+    return account_number
+  end
+
+  ##
+  # Logs into the BPI banking app by finding the form
+  # setting the name and password and submitting it then
+  # waits a bit.
+  #
+  def login(agent)
+    logger.info("Logging in to #{LOGIN_URL}.")
+    if (scraper_args)
+      username, password = *scraper_args
+    end
+    raise "Login failed for BPI Scraper - pass user name and password using -scraper_args \"user <space> pass\"" unless (username and password)
+
+    # navigate to the login page
+    login_page = agent.get(LOGIN_URL)
+
+    # find login form - it's called 'signOn' - fill it out and submit it
+    form  = login_page.form('signOn')
+
+    # username and password are taken from the commandline args, set them
+    # on USERID and PASSWORD which are the element names that the web page
+    # form uses to identify the form fields
+    form.USERID = username
+    form.PASSWORD = password
+
+    # submit the form - same as the user hitting the Login button
+    agent.submit(form)
+    sleep 3  # wait while the login takes effect
+  end
+end # class BpiScraper
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: bankjob
 version: !ruby/object:Gem::Version 
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors: 
 - rhubarb
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-20 00:00:00 +01:00
+date: 2009-05-18 00:00:00 +01:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency