sps_bill 0.1.0

data/lib/sps_bill/bill_collection.rb

@@ -0,0 +1,102 @@
+# SpsBill::BillCollection is an Array-like class that represents a collection
+# of SP Services PDF bills.
+#
+# The <tt>load</tt> method is used to initialise the collection given a path specification.
+#
+# A range of collection methods are provided to extract sets of data
+# from the entire collection (e.g. <tt>electricity_usages</tt>).
+#
+class SpsBill::BillCollection < Array
+
+  class << self
+
+    # Returns an array of Bill objects for PDF files matching +path_spec+.
+    # +path_spec+ may be either:
+    #  - an array of filenames e.g. ['data/file1.pdf','file2.pdf']
+    #  - or a single file or path spec e.g. './somepath/file1.pdf' or './somepath/*.pdf'
+    def load(path_spec)
+      path_spec = Dir[path_spec] unless path_spec.class <= Array
+      path_spec.each_with_object(new) do |filename,memo|
+        memo << SpsBill::Bill.new(filename)
+      end
+    end
+
+  end
+
+  def headers(dataset_selector)
+    case dataset_selector
+    when :total_amounts
+      ['invoice_month','amount']
+    when :electricity_usages
+      ['invoice_month','kwh','rate','amount']
+    when :gas_usages
+      ['invoice_month','kwh','rate','amount']
+    when :water_usages
+      ['invoice_month','cubic_m','rate','amount']
+    when :all_data
+      ['invoice_month','measure','kwh','cubic_m','rate','amount']
+    end
+  end
+
+  # Returns a hash of all data by month
+  # [[month,measure,kwh,cubic_m,rate,amount]]
+  # measure: total_charges,electricity,gas,water
+  def all_data
+    total_amounts(:all) + electricity_usages(:all) + gas_usages(:all) + water_usages(:all)
+  end
+
+  # Returns a hash of total bill amounts by month
+  # [[month,amount]]
+  def total_amounts(style=:solo)
+    each_with_object([]) do |bill,memo|
+      if style==:solo
+        memo << [bill.invoice_month.to_s,bill.total_amount]
+      else
+        memo << [bill.invoice_month.to_s,'total_charges',nil,nil,nil,bill.total_amount]
+      end
+    end
+  end
+
+  # Returns a hash of electricity_usages by month
+  # [[month,kwh,rate,amount]]
+  def electricity_usages(style=:solo)
+    each_with_object([]) do |bill,memo|
+      bill.electricity_usage.each do |usage|
+        if style==:solo
+          memo << [bill.invoice_month.to_s,usage[:kwh],usage[:rate],usage[:amount]]
+        else
+          memo << [bill.invoice_month.to_s,'electricity',usage[:kwh],nil,usage[:rate],usage[:amount]]
+        end
+      end
+    end
+  end
+
+  # Returns a hash of gas_usages by month
+  # [[month,kwh,rate,amount]]
+  def gas_usages(style=:solo)
+    each_with_object([]) do |bill,memo|
+      bill.gas_usage.each do |usage|
+        if style==:solo
+          memo << [bill.invoice_month.to_s,usage[:kwh],usage[:rate],usage[:amount]]
+        else
+          memo << [bill.invoice_month.to_s,'gas',usage[:kwh],nil,usage[:rate],usage[:amount]]
+        end
+      end
+    end
+  end
+
+  # Returns a hash of water_usages by month
+  # [[month,kwh,rate,amount]]
+  def water_usages(style=:solo)
+    each_with_object([]) do |bill,memo|
+      bill.water_usage.each do |usage|
+        if style==:solo
+          memo << [bill.invoice_month.to_s,usage[:cubic_m],usage[:rate],usage[:amount]]
+        else
+          memo << [bill.invoice_month.to_s,'water',nil,usage[:cubic_m],usage[:rate],usage[:amount]]
+        end
+      end
+    end
+  end
+
+end

data/lib/sps_bill/bill_parser.rb

@@ -0,0 +1,92 @@
+require 'date'
+
+# all the bill scanning and parsing intelligence
+module SpsBill::BillParser
+
+  ELECTRICITY_SERVICE_HEAD = "Electricity Services"
+  GAS_SERVICE_HEAD = "Gas Services by City Gas Pte Ltd"
+  WATER_SERVICE_HEAD = "Water Services by Public Utilities Board"
+
+  # Returns a collection of parser errors
+  def errors
+    @errors ||= []
+  end
+
+  # Command: scans and extracts billing details from the pdf doc
+  def do_complete_parse
+    return unless reader
+    methods.select{|m| m =~ /^parse_/ }.each do |m|
+      begin
+        send(m)
+      rescue => e
+        errors << "failure parsing #{source_file}:#{m} #{e.inspect}"
+      end
+    end
+  end
+
+  # Command: extracts the account number
+  def parse_account_number
+    @account_number = reader.text_in_rect(383.0,999.0,785.0,790.0,1).flatten.join('')
+  end
+
+  # Command: extracts the total amount due for the current month
+  def parse_total_amount
+    @total_amount = if ref = reader.text_position(/^Total Current Charges due on/)
+      total_parts = reader.text_in_rect(ref[:x] + 1,400.0,ref[:y] - 1,ref[:y] + 1,1)
+      total_parts.flatten.first.to_f
+    end
+  end
+
+  # Command: extracts the invoice date
+  def parse_invoice_date
+    @invoice_date = if ref = reader.text_position("Dated")
+      date_parts = reader.text_in_rect(ref[:x] + 1,999.0,ref[:y] - 1,ref[:y] + 1,1)
+      Date.parse(date_parts.first.join('-'))
+    end
+  end
+
+  # Command: extracts the invoice month (as Date, set to 1st of the month)
+  def parse_invoice_month
+    @invoice_month = if ref = reader.text_position("Dated")
+      date_parts = reader.text_in_rect(ref[:x] + 1,999.0,ref[:y] - 1,ref[:y] + 1,1)
+      m_parts = reader.text_in_rect(ref[:x]-200,ref[:x]-1,ref[:y] - 1,ref[:y] + 1,1)
+      Date.parse("#{date_parts.first.last}-#{m_parts.first.first}-01")
+    end
+  end
+
+  # Command: extracts an array of electricity usage charges. Each charge is a Hash:
+  # { kwh: float, rate: float, amount: float }
+  def parse_electricity_usage
+    @electricity_usage = if upper_ref = reader.text_position(ELECTRICITY_SERVICE_HEAD)
+      lower_ref = reader.text_position(GAS_SERVICE_HEAD)
+      lower_ref ||= reader.text_position(WATER_SERVICE_HEAD)
+      if lower_ref
+        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
+        raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
+      end
+    end
+  end
+
+  # Command: extracts an array of gas usage charges. Each charge is a Hash:
+  # { kwh: float, rate: float, amount: float }
+  def parse_gas_usage
+    @gas_usage = if upper_ref = reader.text_position(GAS_SERVICE_HEAD)
+      if lower_ref = reader.text_position(WATER_SERVICE_HEAD)
+        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
+        raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
+      end
+    end
+  end
+
+  # Command: extracts an array of water usage charges. Each charge is a Hash:
+  # { cubic_m: float, rate: float, amount: float }
+  def parse_water_usage
+    @water_usage = if upper_ref = reader.text_position(WATER_SERVICE_HEAD)
+      if lower_ref = reader.text_position("Waterborne Fee")
+        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
+        raw_data.map{|l| {:cubic_m => l[0].gsub(/cu m/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
+      end
+    end
+  end
+
+end

data/lib/sps_bill/shell.rb

@@ -0,0 +1,71 @@
+class SpsBill::Shell
+  attr_accessor :options, :fileset
+
+  # command line options definition
+  OPTIONS = %w(help verbose csv+ raw+ data=s)
+
+  # Usage message
+  def self.usage
+    puts <<-EOS
+
+SP Services Bill Scanner v#{SpsBill::Version::STRING}
+===================================
+
+Usage:
+  sps_bill [options]
+
+Command Options
+  -r  | --raw    raw data format (without headers)
+  -c  | --csv    output in CSV format (default)
+  -d= | --data=[charges,electricity,gas,water,all]
+
+    EOS
+  end
+
+  # +new+
+  def initialize(options)
+    @fileset = ARGV
+    @options = (options||{}).each{|k,v| {k => v} }
+  end
+
+  def run
+    if options[:help]
+      self.class.usage
+      return
+    end
+    case options[:data]
+    when /^c/
+      export(:total_amounts)
+    when /^e/
+      export(:electricity_usages)
+    when /^g/
+      export(:gas_usages)
+    when /^w/
+      export(:water_usages)
+    # when /^a/
+    else
+      export(:all_data)
+    end
+  end
+
+  def bills
+    @bills ||= SpsBill::BillCollection.load(fileset)
+  end
+
+  def export(dataset_selector)
+    format_header bills.headers(dataset_selector)
+    format_rows bills.send(dataset_selector)
+  end
+
+  def format_rows(data)
+    data.each do |row|
+      puts row.join(',')
+    end
+  end
+
+  def format_header(data)
+    return if options[:raw]
+    puts data.join(',')
+  end
+
+end

data/lib/sps_bill/version.rb

@@ -0,0 +1,9 @@
+module SpsBill
+  class Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+
+    STRING = [MAJOR, MINOR, PATCH].compact.join('.')
+  end
+end

data/lib/sps_bill.rb

@@ -0,0 +1,13 @@
+require 'pdf-reader'
+require 'pdf/object_hash'
+require 'pdf/positional_text_receiver'
+require 'pdf/textangle'
+require 'pdf/structured_reader'
+
+module SpsBill
+end
+require 'sps_bill/version'
+require 'sps_bill/bill_collection'
+require 'sps_bill/bill_parser'
+require 'sps_bill/bill'
+require 'sps_bill/shell'

data/scripts/scan_all_bills.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Demonstrates how to invoke the sps_bill command line to extract
+# all billing data for a set of files
+#
+# Example Usage:
+#
+#   scan_all_bills.sh my_bills/*.pdf > my_bill_data.csv
+#
+scriptPath=${0%/*}/
+
+v_files=${1:-help}
+
+if [ "${v_files}" == "help" ]
+then
+  echo "usage: scan_all_bills.sh ./path/to_pdf_files*.pdf"
+  exit
+fi
+
+${scriptPath}../bin/sps_bill --data=all $*

data/spec/fixtures/pdf_samples/junk_prefix.pdf

@@ -0,0 +1,71 @@
+<html>
+<head></head>
+%PDF-1.3
+%����
+1 0 obj
+<< /Creator <feff0050007200610077006e>
+/Producer <feff0050007200610077006e>
+>>
+endobj
+2 0 obj
+<< /Type /Catalog
+/Pages 3 0 R
+>>
+endobj
+3 0 obj
+<< /Type /Pages
+/Count 1
+/Kids [5 0 R]
+>>
+endobj
+4 0 obj
+<< /Length 157
+>>
+stream
+q
+
+BT
+36 747.384 Td
+/F1.0 12 Tf
+[<546869732050444620636f6e7461696e73206a756e6b20626566> 30 <6f72652074686520252d504446206d6172> -15 <6b> 20 <6572>] TJ
+ET
+
+Q
+
+endstream
+endobj
+5 0 obj
+<< /Type /Page
+/Parent 3 0 R
+/MediaBox [0 0 612.0 792.0]
+/Contents 4 0 R
+/Resources << /ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
+/Font << /F1.0 6 0 R
+>>
+>>
+>>
+endobj
+6 0 obj
+<< /Type /Font
+/Subtype /Type1
+/BaseFont /Helvetica
+/Encoding /WinAnsiEncoding
+>>
+endobj
+xref
+0 7
+0000000000 65535 f 
+0000000015 00000 n 
+0000000109 00000 n 
+0000000158 00000 n 
+0000000215 00000 n 
+0000000423 00000 n 
+0000000601 00000 n 
+trailer
+<< /Size 7
+/Root 2 0 R
+/Info 1 0 R
+>>
+startxref
+698
+%%EOF

data/spec/fixtures/personal_pdf_samples/expectations.yml.sample

@@ -0,0 +1,48 @@
+# rename this file to expectations.yml and use it to
+# test all your SPS bill files that you copy to spec/fixtures/personal_pdf_samples.
+#
+# For each bill file, create a section below (like the examples shown) and
+# update it with the actual values that should come from the bill.
+#
+# This is a YAML-format file, so beware that indentation is significant
+---
+my_sps_bill-2012-02.pdf:
+  :account_number: '8123123123'
+  :invoice_date: 2012-02-29
+  :invoice_month: 2012-02-01
+  :total_amount: 251.44
+  :electricity_usage:
+  - :kwh: 4.0
+    :rate: 0.241
+    :amount: 0.97
+  - :kwh: 616.0
+    :rate: 0.2558
+    :amount: 157.57
+  :gas_usage:
+  - :kwh: 18.0
+    :rate: 0.1799
+    :amount: 3.24
+  :water_usage:
+  - :cubic_m: 36.1
+    :rate: 1.17
+    :amount: 42.24
+  - :cubic_m: -3.0
+    :rate: 1.4
+    :amount: -4.2
+my_sps_bill-2012-03.pdf:
+  :account_number: '8123123123'
+  :invoice_date: 2012-03-31
+  :invoice_month: 2012-03-01
+  :total_amount: 235.7
+  :electricity_usage:
+  - :kwh: 519.0
+    :rate: 0.2558
+    :amount: 132.76
+  :gas_usage:
+  - :kwh: 15.0
+    :rate: 0.1799
+    :amount: 2.7
+  :water_usage:
+  - :cubic_m: 38.7
+    :rate: 1.17
+    :amount: 45.28

data/spec/integration/personal_samples_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+include PdfSamplesHelper
+
+describe "Personal PDF Samples" do
+
+  if personal_pdf_sample_names.empty?
+    pending %(
+
+    You can place the PDFs of your own bills in spec/fixtures/personal_pdf_samples.
+    They will be tested when you run the specs, but are hidden from being added to the
+    git repository.
+
+    )
+  end
+
+  # This will scan all *.pdf files in spec/fixtures/personal_pdf_samples
+  # and do basic verification of the file structure without any effort from you.
+  personal_pdf_sample_names.each do |sample|
+    describe sample do
+      let(:sample_name) { sample }
+      let(:bill) { SpsBill::Bill.new(sample_name) }
+
+      it_behaves_like "has a valid reader", :bill
+      it_behaves_like "has a valid account number", :bill
+      it_behaves_like "has a valid invoice date", :bill
+
+
+    end
+  end
+
+  # This will read spec/fixtures/personal_pdf_samples/expectations.yml
+  # and and test according to the definitions it contains.
+  #
+  # See spec/fixtures/personal_pdf_samples/expectations.yml.sample
+  # for details on how to setup the expectations.yml file
+  #
+  personal_pdf_sample_expectations.each do |sample_name,expectations|
+    describe sample_name do
+      let(:sample_file) { personal_pdf_sample_path.join(sample_name).to_s }
+      let(:bill) { SpsBill::Bill.new(sample_file) }
+      subject { bill }
+
+      if expectations[:account_number]
+        its(:account_number) { should eql(expectations[:account_number])}
+      end
+
+      if expectations[:total_amount]
+        its(:total_amount) { should eql(expectations[:total_amount])}
+      end
+
+      if expectations[:invoice_date]
+        its(:invoice_date) { should eql(expectations[:invoice_date])}
+      end
+
+      if expectations[:invoice_month]
+        its(:invoice_month) { should eql(expectations[:invoice_month])}
+      end
+
+      if expectations[:electricity_usage]
+        its(:electricity_usage) { should eql(expectations[:electricity_usage])}
+      end
+
+      if expectations[:gas_usage]
+        its(:gas_usage) { should eql(expectations[:gas_usage])}
+      end
+
+      if expectations[:water_usage]
+        its(:water_usage) { should eql(expectations[:water_usage])}
+      end
+
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'sps_bill'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  # == Mock Framework
+  #
+  # If you prefer to use mocha, flexmock or RR, uncomment the appropriate line:
+  #
+  # config.mock_with :mocha
+  # config.mock_with :flexmock
+  # config.mock_with :rr
+  config.mock_with :rspec
+
+  # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures
+  # config.fixture_path = "#{::Rails.root}/spec/fixtures"
+
+  # If you're not using ActiveRecord, or you'd prefer not to run each of your
+  # examples within a transaction, remove the following line or assign false
+  # instead of true.
+  # config.use_transactional_fixtures = true
+end

data/spec/support/bill_examples.rb

@@ -0,0 +1,31 @@
+shared_examples_for "has a valid reader" do |resource_key|
+  # args:
+  # +resource_key+ is the sym for the resource to test
+  describe "#reader" do
+    let(:resource) { eval "#{resource_key}" }
+    let(:reader) { resource.reader }
+    subject { reader }
+    it { should be_a(PDF::StructuredReader) }
+  end
+end
+
+shared_examples_for "has a valid account number" do |resource_key|
+  # args:
+  # +resource_key+ is the sym for the resource to test
+  describe "#account_number" do
+    let(:resource) { eval "#{resource_key}" }
+    subject { resource.account_number  }
+    it { should be_a(String) }
+    it { should match(/^\d+$/) }
+  end
+end
+
+shared_examples_for "has a valid invoice date" do |resource_key|
+  # args:
+  # +resource_key+ is the sym for the resource to test
+  describe "#invoice_date" do
+    let(:resource) { eval "#{resource_key}" }
+    subject { resource.invoice_date  }
+    it { should be_a(Date) }
+  end
+end

data/spec/support/pdf_samples_helper.rb

@@ -0,0 +1,37 @@
+require 'pathname'
+require 'yaml'
+
+module PdfSamplesHelper
+
+  def pdf_sample_path
+    Pathname.new(File.dirname(__FILE__)).join('..','fixtures','pdf_samples')
+  end
+
+  def personal_pdf_sample_path
+    Pathname.new(File.dirname(__FILE__)).join('..','fixtures','personal_pdf_samples')
+  end
+
+  def pdf_sample_name
+    pdf_sample_path.join('bill_a.pdf').to_s
+  end
+
+  def junk_prefix_pdf_sample_name
+    pdf_sample_path.join('junk_prefix.pdf').to_s
+  end
+
+  def personal_pdf_sample_names
+    Dir["#{File.dirname(__FILE__)}/../fixtures/personal_pdf_samples/*.pdf"]
+  end
+
+  def personal_pdf_sample_expectations_path
+    Pathname.new(File.dirname(__FILE__)).join('..','fixtures','personal_pdf_samples','expectations.yml')
+  end
+
+  def personal_pdf_sample_expectations
+    begin
+      YAML.load_file personal_pdf_sample_expectations_path
+    rescue
+      []
+    end
+  end
+end