bankjob 0.5.0

data/lib/bankjob/statement.rb

@@ -0,0 +1,355 @@
+require 'rubygems'
+require 'builder'
+require 'fastercsv'
+require 'bankjob'
+
+module Bankjob
+
+  ##
+  # A Statement object represents a bank statement and is generally the result of running a Bankjob scraper.
+  # The Statement holds an array of Transaction objects and specifies the closing balance and the currency in use.
+  #
+  # A Scraper will create a Statement by scraping web pages in an online banking site.
+  # The Statement can then be stored as a file in CSV (Comma Separated Values) format
+  # using +to_csv+ or in OFX (Open Financial eXchange http://www.ofx.net) format
+  # using +to_ofx+.
+  #
+  # One special ability of Statement is the ability to merge with an existing statement,
+  # automatically eliminating overlapping transactions.
+  # This means that when writing subsequent statements to the same CSV file
+  # <em>(note well: CSV only)</em> a continous transaction record can be built up
+  # over a long period.
+  #
+  class Statement
+
+    # OFX value for the ACCTTYPE of a checking account
+    CHECKING   = "CHECKING"
+
+    # OFX value for the ACCTTYPE of a savings account
+    SAVINGS    = "SAVINGS"
+
+    # OFX value for the ACCTTYPE of a money market account
+    MONEYMRKT  = "MONEYMRKT"
+
+    # OFX value for the ACCTTYPE of a loan account
+    CREDITLINE = "CREDITLINE"
+
+    # the account balance after the last transaction in the statement
+    # Translates to the OFX element BALAMT in LEDGERBAL
+    attr_accessor :closing_balance
+
+    # the avaliable funds in the account after the last transaction in the statement (generally the same as closing_balance)
+    # Translates to the OFX element BALAMT in AVAILBAL
+    attr_accessor :closing_available
+
+    # the array of Transaction objects that comprise the statement
+    attr_accessor :transactions
+
+    # the three-letter currency symbol generated into the OFX output (defaults to EUR)
+    # This is passed into the initializer (usually by the Scraper - see Scraper#currency)
+    attr_reader :currency 
+
+    # the identifier of the bank - a 1-9 char string (may be empty)
+    # Translates to the OFX element BANKID
+    attr_accessor :bank_id
+
+    # the account number of the statement - a 1-22 char string that must be passed
+    # into the initalizer of the Statement
+    # Translates to the OFX element ACCTID
+    attr_accessor :account_number
+
+    # the type of bank account the statement is for
+    # Tranlsates to the OFX type ACCTTYPE and must be one of
+    # * CHECKING
+    # * SAVINGS
+    # * MONEYMRKT
+    # * CREDITLINE
+    # Use a constant to set this - defaults to CHECKING
+    attr_accessor :account_type
+
+    ##
+    # Creates a new empty Statement with no transactions.
+    # The +account_number+ must be specified as a 1-22 character string.
+    # The specified +currency+ defaults to EUR if nothing is passed in.
+    #
+    def initialize(account_number, currency = "EUR")
+      @account_number = account_number
+      @currency = currency
+      @transactions = []
+      @account_type = CHECKING
+    end
+    
+    ##
+    # Appends a new Transaction to the end of this Statement
+    #
+    def add_transaction(transaction)
+      @transactions << transaction
+    end
+
+    ##
+    # Overrides == to allow comparison of Statement objects.
+    # Two Statements are considered equal (that is, ==) if
+    # and only iff they have the same values for:
+    # * +to_date+
+    # * +from_date+
+    # * +closing_balance+
+    # * +closing_available+
+    # * each and every transaction.
+    # Note that the transactions are compared with Transaction.==
+    #
+    def ==(other) # :nodoc:
+      if other.kind_of?(Statement) 
+        return (from_date == other.from_date and
+            to_date == other.to_date and
+            closing_balance == other.closing_balance and
+            closing_available == other.closing_available and
+            transactions == other.transactions)
+      end
+      return false
+    end
+   
+    ##
+    # Merges the transactions of +other+ into the transactions of this statement
+    # and returns the resulting array of transactions
+    # Raises an exception if the two statements overlap in a discontiguous fashion.
+    #
+    def merge_transactions(other)
+      if (other.kind_of?(Statement))
+        union = transactions | other.transactions # the set union of both
+        # now check that the union contains all of the originals, otherwise
+        # we have merged some sort of non-contiguous range
+        raise "Failed to merge transactions properly." unless union.first(@transactions.length) == @transactions
+        return union
+      end
+    end
+
+    ##
+    # Merges the transactions of +other+ into the transactions of this statement
+    # and returns the result.
+    # Neither statement is changed. See #merge! if you want to modify the statement.
+    # Raises an exception if the two statements overlap in a discontiguous fashion.
+    #
+    def merge(other)
+      union = merge_transactions(other)
+      merged = self.dup
+      merged.closing_balance = nil
+      merged.closing_available = nil
+      merged.transactions = union
+      return merged
+    end
+
+    ##
+    # Merges the transactions of +other+ into the transactions of this statement.
+    # Causes this statement to be changed. See #merge for details.
+    #
+    def merge!(other)
+      @closing_balance = nil
+      @closing_available = nil
+      @transactions = merge_transactions(other)
+    end
+
+    ##
+    # Returns the statement's start date.
+    # The +from_date+ is taken from the date of the last transaction in the statement
+    #
+    def from_date()
+      return nil if @transactions.empty?
+      @transactions.last.date
+    end
+
+    ##
+    # Returns the statement's end date.
+    # The +to_date+ is taken from the date of the first transaction in the statement
+    #
+    def to_date()
+      return nil if @transactions.empty?
+      @transactions.first.date
+    end
+
+    ##
+    # Returns the closing balance by looking at the
+    # new balance of the first transaction.
+    # If there are no transactions, +nil+ is returned.
+    #
+    def closing_balance()
+      return nil if @closing_balance.nil? and @transactions.empty?
+      @closing_balance ||= @transactions.first.new_balance
+    end
+
+    ##
+    # Returns the closing available balance by looking at the
+    # new balance of the first transaction.
+    # If there are no transactions, +nil+ is returned.
+    # Note that this is the same value returned as +closing_balance+.
+    #
+    def closing_available()
+      return nil if @closing_available.nil? and @transactions.empty?
+      @closing_available ||= @transactions.first.new_balance
+    end
+
+    ##
+    # Generates a CSV (comma separated values) string with a single
+    # row for each transaction.
+    # Note that no header row is generated as it would make it
+    # difficult to concatenate and merge subsequent CSV strings
+    # (but we should consider it as a user option in the future)
+    #
+    def to_csv
+      buf = ""
+      transactions.each do |transaction|
+        buf << transaction.to_csv
+      end
+      return buf
+    end
+
+    ##
+    # Generates a string for use as a header in a CSV file for a statement.
+    #
+    # Delegates to Transaction#csv_header
+    #
+    def self.csv_header
+      return Transaction.csv_header
+    end
+
+    ##
+    # Reads in transactions from a CSV file or string specified by +source+
+    # and adds them to this statement.
+    # 
+    # Uses a simple (dumb) heuristic to determine if the +source+ is a file
+    # or a string: if it contains a comma (,) then it is a string
+    # otherwise it is treated as a file path.
+    #
+    def from_csv(source, decimal = ".")
+      if (source =~ /,/)
+        # assume source is a string
+        FasterCSV.parse(source) do |row|
+          add_transaction(Transaction.from_csv(row, decimal))
+        end
+      else
+        # assume source is a filepath
+        FasterCSV.foreach(source) do |row|
+          add_transaction(Transaction.from_csv(row, decimal))
+        end
+      end
+    end
+
+    ##
+    # Generates an XML string adhering to the OFX standard
+    # (see Open Financial eXchange http://www.ofx.net)
+    # representing a single bank statement holding a list
+    # of transactions.
+    # The XML for the individual transactions is generated
+    # by the Transaction class itself.
+    #
+    # The OFX 2 schema for a statement response (STMTRS) is:
+    #
+    #  <xsd:complexType name="StatementResponse">
+    #    <xsd:annotation>
+    #      <xsd:documentation>
+    #        The OFX element "STMTRS" is of type "StatementResponse"
+    #      </xsd:documentation>
+    #    </xsd:annotation>
+    #
+    #    <xsd:sequence>
+    #      <xsd:element name="CURDEF" type="ofx:CurrencyEnum"/>
+    #      <xsd:element name="BANKACCTFROM" type="ofx:BankAccount"/>
+    #      <xsd:element name="BANKTRANLIST" type="ofx:BankTransactionList" minOccurs="0"/>
+    #      <xsd:element name="LEDGERBAL" type="ofx:LedgerBalance"/>
+    #      <xsd:element name="AVAILBAL" type="ofx:AvailableBalance" minOccurs="0"/>
+    #      <xsd:element name="BALLIST" type="ofx:BalanceList" minOccurs="0"/>
+    #      <xsd:element name="MKTGINFO" type="ofx:InfoType" minOccurs="0"/>
+    #    </xsd:sequence>
+    #  </xsd:complexType>
+    #
+    # Where the BANKTRANLIST (Bank Transaction List) is defined as:
+    #
+    #  <xsd:complexType name="BankTransactionList">
+    #    <xsd:annotation>
+    #      <xsd:documentation>
+    #        The OFX element "BANKTRANLIST" is of type "BankTransactionList"
+    #      </xsd:documentation>
+    #    </xsd:annotation>
+    #    <xsd:sequence>
+    #      <xsd:element name="DTSTART" type="ofx:DateTimeType"/>
+    #      <xsd:element name="DTEND" type="ofx:DateTimeType"/>
+    #      <xsd:element name="STMTTRN" type="ofx:StatementTransaction" minOccurs="0" maxOccurs="unbounded"/>
+    #    </xsd:sequence>
+    #  </xsd:complexType>
+    #
+    # And this is the definition of the type BankAccount.
+    #
+    #  <xsd:complexType name="BankAccount">
+		#    <xsd:annotation>
+		#      <xsd:documentation>
+    #        The OFX elements BANKACCTFROM and BANKACCTTO are of type "BankAccount"
+    #      </xsd:documentation>
+		#    </xsd:annotation>
+		#    <xsd:complexContent>
+		#      <xsd:extension base="ofx:AbstractAccount">
+    #        <xsd:sequence>
+    #          <xsd:element name="BANKID" type="ofx:BankIdType"/>
+    #          <xsd:element name="BRANCHID" type="ofx:AccountIdType" minOccurs="0"/>
+    #          <xsd:element name="ACCTID" type="ofx:AccountIdType"/>
+    #          <xsd:element name="ACCTTYPE" type="ofx:AccountEnum"/>
+    #          <xsd:element name="ACCTKEY" type="ofx:AccountIdType" minOccurs="0"/>
+    #        </xsd:sequence>
+    #      </xsd:extension>
+    #    </xsd:complexContent>
+    #  </xsd:complexType>
+    #
+    # The to_ofx method will only generate the essential elements which are 
+    # * BANKID - the bank identifier (a 1-9 char string - may be empty)
+    # * ACCTID - the account number (a 1-22 char string - may not be empty!)
+    # * ACCTTYPE - the type of account - must be one of:
+    #              "CHECKING", "SAVINGS", "MONEYMRKT", "CREDITLINE"
+    #
+    # (See Transaction for a definition of STMTTRN)
+    #
+    def to_ofx
+      buf = ""
+      # Use Builder to generate XML. Builder works by catching missing_method
+      # calls and generating an XML element with the name of the missing method,
+      # nesting according to the nesting of the calls and using arguments for content
+      x = Builder::XmlMarkup.new(:target => buf, :indent => 2)
+      x.OFX {
+        x.BANKMSGSRSV1 { #Bank Message Response
+          x.STMTTRNRS {		#Statement-transaction aggregate response
+            x.STMTRS {		#Statement response
+              x.CURDEF currency	#Currency
+              x.BANKACCTFROM {
+                x.BANKID bank_id # bank identifier
+                x.ACCTID account_number
+                x.ACCTTYPE account_type # acct type: checking/savings/...
+              }
+              x.BANKTRANLIST {	#Transactions
+                x.DTSTART Bankjob.date_time_to_ofx(from_date)
+                x.DTEND Bankjob.date_time_to_ofx(to_date)
+                transactions.each { |transaction|
+                  buf << transaction.to_ofx
+                }
+              }
+              x.LEDGERBAL {	# the final balance at the end of the statement
+                x.BALAMT closing_balance # balance amount
+                x.DTASOF Bankjob.date_time_to_ofx(to_date)		# balance date
+              }
+              x.AVAILBAL {	# the final Available balance
+                x.BALAMT closing_available
+                x.DTASOF Bankjob.date_time_to_ofx(to_date)
+              }
+            }
+          }
+        }
+      }
+      return buf
+    end
+
+    def to_s
+      buf = "#{self.class}: close_bal = #{closing_balance}, avail = #{closing_available}, curr = #{currency}, transactions:"
+      transactions.each do |tx|
+        buf << "\n\t\t#{tx.to_s}"
+      end
+      buf << "\n---\n"
+      return buf
+    end
+  end # class Statement
+end # module

data/lib/bankjob/support.rb

@@ -0,0 +1,217 @@
+
+require 'rubygems'
+require 'bankjob'
+
+module Bankjob
+
+  ##
+  # Takes a date-time as a string or as a Time or DateTime object and returns
+  # it as either a Time or a DateTime object.
+  #
+  # This is useful in the setter method of a date attribute allowing the date
+  # to be set as any type but stored internally as an object compatible with
+  # conversion through +strftime()+
+  # (Bankjob::Transaction uses this internally in the setter for +date+ for example
+  #
+  def self.create_date_time(date_time_raw)
+    if (date_time_raw.respond_to?(:rfc822)) then
+      # It's already a Time or DateTime
+      return date_time_raw
+    elsif (date_time_raw.to_s.strip.empty?)
+      # Nil or non dates are returned as nil
+      return nil
+    else
+      # Assume it can be converted to a time
+      return Time.parse(date_time_raw.to_s)
+    end
+  end
+
+  ##
+  # Takes a Time or DateTime and formats it in the correct format for OFX date elements.
+  #
+  # The OFX format is a string of digits in the format "YYMMDDHHMMSS".
+  # For example, the 1st of February 2009 at 2:34PM and 56 second becomes "20090201143456"
+  #
+  # Note must use a Time, or DateTime, not a String, nor a Date.
+  #
+  def self.date_time_to_ofx(time)
+    time.nil? ? "" : "#{time.strftime( '%Y%m%d%H%M%S' )}"
+  end
+
+  ##
+  # Takes a Time or DateTime and formats in a suitable format for comma separated values files.
+  # The format produced is suitable for loading into an Excel-like spreadsheet program
+  # being automatically treated as a date.
+  #
+  # A string is returned with the format "YY-MM-DD HH:MM:SS".
+  # For example, the 1st of February 2009 at 2:34PM and 56 second becomes "2009-02-01 14:34:56"
+  #
+  # Note must use a Time, or DateTime, not a String, nor a Date.
+  #
+  def self.date_time_to_csv(time)
+    time.nil? ? "" : "#{time.strftime( '%Y-%m-%d %H:%M:%S' )}"
+  end
+
+  ##
+  # Takes a string and capitalizes the first letter of every word
+  # and forces the rest of the word to be lowercase.
+  #
+  # This is a utility method for use in scrapers to make descriptions
+  # more readable.
+  #
+  def self.capitalize_words(message)
+    message.downcase.gsub(/\b\w/){$&.upcase}
+  end
+
+  ##
+  # converts a numeric +string+ to a float given the specified +decimal+
+  # separator.
+  #
+  def self.string_to_float(string, decimal)
+    return nil if string.nil?
+    amt = string.gsub(/\s/, '')
+    if (decimal == ',') # E.g.  "1.000.030,99"
+      amt.gsub!(/\./, '')  # strip out . 1000s separator
+      amt.gsub!(/,/, '.')  # replace decimal , with .
+    elsif (decimal == '.')
+      amt.gsub!(/,/, '')  # strip out comma 1000s separator
+    end
+    return amt.to_f
+  end
+
+
+  def self.wesabe_upload(wesabe_args, ofx_doc, logger)
+    if (wesabe_args.nil? or (wesabe_args.length < 2 and wesabe_args.length > 3))
+      raise "Incorrect number of args for Wesabe (#{wesabe_args}), should be 2 or 3."
+    else
+      load_wesabe
+      wuser, wpass, windex = *wesabe_args
+      wesabe = Wesabe.new(wuser, wpass)
+      num_accounts = wesabe.accounts.length
+      if num_accounts == 0
+        raise "The user \"#{wuser}\" has no Wesabe accounts. Create one at www.wesabe.com before attempting to upload a statement."
+      elsif (not windex.nil? and (num_accounts < windex.to_i))
+        raise "The user \"#{wuser}\" has only #{num_accounts} Wesabe accounts, but the account index #{windex} was specified."
+      elsif windex.nil?
+        if num_accounts > 1
+          raise "The user \"#{wuser}\" has #{num_accounts} Wesabe accounts, so the account index must be specified in the WESABE_ARGS."
+        else
+          # we have only one account, no need to specify the index
+          windex = 1
+        end
+      elsif windex.to_i == 0
+        raise "The Wesabe account index must be between 1 and #{num_accounts}. #{windex} is not acceptable"
+      end
+      logger.debug("Attempting to upload statement to the ##{windex} Wesabe account for user #{wuser}...")
+      # Get the account at the index (which is not necessarily the index in the array 
+      # so we use the account(index) method to get it
+      account = wesabe.account(windex.to_i)
+      uploader = account.new_upload
+      uploader.statement = ofx_doc
+      uploader.upload!
+      logger.info("Uploaded statement to Wesabe account #{account.name}, the ##{windex} account for user #{wuser}, with the result: #{uploader.status}")
+    end
+  end
+
+  def self.wesabe_help(wesabe_args)
+    if (wesabe_args.nil? or wesabe_args.length != 2)
+      puts <<-EOF
+Wesabe (http://www.wesabe.com) is an online bank account management tool (like Mint)
+that allows you to upload (in some cases automatically) your bank statements and
+automatically convert them into a more readable format to allow you to track
+your spending and much more. Wesabe comes with its own community attached.
+
+Bankjob has no affiliation with Wesabe, but allows you to upload the statements it
+generates to your Wesabe account automatically.
+
+To use Wesabe you need the Wesabe Ruby gem installed:
+See the gem at http://github.com/wesabe/wesabe-rubygem
+Install the gem with:
+  $ sudo gem install -r --source http://gems.github.com/ wesabe-wesabe
+(on Windows, omit the "sudo")
+
+You also need your Wesabe login name and password, and, if you have
+more than one account on Wesabe, the id number of the account.
+This is not a real account number - it's simply a counter that Wesabe uses.
+If you have a single account it will be '1', if you have two accounts the
+second account will be '2', etc.
+
+Bankjob will help you find this number by listing your Wesabe accounts for you.
+Simply use:
+  bankjob -wesabe_help "username password"
+(The quotes are important - this is a single argument to Bankjob with two words)
+
+If you already know the number of the account and you want to start uploading use:
+
+  bankjob [other bankjob args] --wesabe "username password id"
+
+E.g.
+  bankjob --scraper bpi_scraper.rb --wesabe "bloggsy pw123 2"
+
+If you only have a single account, you don't need to specify the id number
+(but Bankjob will check and will fail with an error if you have more than one account)
+
+  bankjob [other bankjob args] --wesabe "username password"
+
+If in any doubt --wesabe-help "username password" will set you straight.
+
+Troubleshooting:
+- If you see an error like Wesabe::Request::Unauthorized, then chances
+  are your username or password for Wesabe is incorrect.
+
+- If you see an error "end of file reached" then it may be that you are logged
+  into the Wesabe account to which you are trying to upload - perhaps in a browser.
+  In this case, log out from Wesabe in the browser, _wait a minute_, then try again.
+  EOF
+    else
+      load_wesabe
+      begin
+        puts "Connecting to Wesabe...\n"
+        wuser, wpass = *wesabe_args
+        wesabe = Wesabe.new(wuser, wpass)
+        puts "You have #{wesabe.accounts.length} Wesabe accounts:"
+        wesabe.accounts.each do |account|
+          puts " Account Name: #{account.name}"
+          puts "    wesabe id: #{account.id}"
+          puts "   account no: #{account.number}"
+          puts "         type: #{account.type}"
+          puts "      balance: #{account.balance}"
+          puts "         bank: #{account.financial_institution.name}"
+          puts "To upload to this account use:"
+          puts "  bankjob [other bankjob args] --wesabe \"#{wuser} password #{account.id}\""
+          puts ""
+          if wesabe.accounts.length == 1
+            puts "Since you have one account you do not need to specify the id number, use:"
+            puts "  bankjob [other bankjob args] --wesabe \"#{wuser} password\""
+          end
+        end
+      rescue Exception => e
+        raise <<-EOF
+Failed to get Wesabe account information due to: #{e.message}.
+Check your username and password or use:
+  bankjob --wesabe-help
+with no arguments for more details.
+        EOF
+      end
+    end
+  end # wesabe_help
+
+  private
+
+  def self.load_wesabe(logger = nil)
+    begin
+      require 'wesabe'
+    rescue LoadError => error
+      msg = <<-EOF
+Failed to load the Wesabe gem due to #{error.module}
+See the gem at http://github.com/wesabe/wesabe-rubygem
+Install the gem with:
+  $ sudo gem install -r --source http://gems.github.com/ wesabe-wesabe
+EOF
+      logger.fatal(msg) unless logger.nil?
+      raise msg
+    end
+  end
+end # module Bankjob
+
+