sps_bill 0.1.0 → 0.1.1

data/.travis.yml

@@ -1,3 +1,8 @@
 # These are specific configuration settings required for travis-ci
 # see http://travis-ci.org/tardate/sps_bill_scanner
-rvm: 1.9.3
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - rbx-19mode
+  - jruby-19mode

data/CHANGELOG

@@ -0,0 +1,9 @@
+Version 0.1.1             Release: 1st August 2012
+==================================================
+* update to use pdf-reader-turtletext 0.2.2
+* convert bill parsing to use the more idiomatic bounding_box
+  syntax available in pdf-reader-turtletext
+
+Version 0.1.0              Release: 20th July 2012
+==================================================
+* Initial packaging and release

data/Gemfile

@@ -1,12 +1,11 @@
 source "http://rubygems.org"
 
-gem 'pdf-reader', '1.1.1'
+gem 'pdf-reader-turtletext', '~> 0.2.2'
 gem 'getoptions', '~> 0.3'
 
 group :development do
   gem 'bundler', '~> 1.1.4'
   gem 'jeweler', '~> 1.6.4'
-  gem 'rcov', '>= 0'
 end
 
 group :development, :test do

data/Gemfile.lock

@@ -3,7 +3,7 @@ GEM
   specs:
     Ascii85 (1.0.1)
     diff-lcs (1.1.3)
-    ffi (1.0.11)
+    ffi (1.1.0)
     getoptions (0.3)
     git (1.2.5)
     guard (1.2.3)
@@ -15,7 +15,7 @@ GEM
       bundler (~> 1.0)
       git (>= 1.2.5)
       rake
-    json (1.6.4)
+    json (1.7.3)
     listen (0.4.7)
       rb-fchange (~> 0.0.5)
       rb-fsevent (~> 0.9.1)
@@ -23,13 +23,14 @@ GEM
     pdf-reader (1.1.1)
       Ascii85 (~> 1.0.0)
       ruby-rc4
+    pdf-reader-turtletext (0.2.2)
+      pdf-reader (= 1.1.1)
     rake (0.9.2.2)
     rb-fchange (0.0.5)
       ffi
     rb-fsevent (0.9.1)
     rb-inotify (0.8.8)
       ffi (>= 0.5.0)
-    rcov (0.9.11)
     rdoc (3.12)
       json (~> 1.4)
     rspec (2.8.0)
@@ -51,8 +52,7 @@ DEPENDENCIES
   getoptions (~> 0.3)
   guard-rspec
   jeweler (~> 1.6.4)
-  pdf-reader (= 1.1.1)
+  pdf-reader-turtletext (~> 0.2.2)
   rake (~> 0.9.2.2)
-  rcov
   rdoc (~> 3.11)
   rspec (~> 2.8.0)

data/README.rdoc

@@ -1,12 +1,17 @@
 = SP Services Bill Scanner {<img src="https://secure.travis-ci.org/tardate/sps_bill_scanner.png" />}[http://travis-ci.org/tardate/sps_bill_scanner]
 
-Extracts bill details from SP Services PDF bills so that you can, um, do geeky data analysis n'stuff.
+Extracts bill details from SP Services PDF bills so that you can, um, do geeky data analysis n'stuff,
+and because I loathe data entry!
+One day we'll have {smart meters}[http://en.wikipedia.org/wiki/Smart_meter]
+and SP Services will let us download our raw meter data. But until then...
 
 If you are an SP Services subscriber, download your bills from https://services.spservices.sg
 
 If you are not an SP Services subscriber, this gem ain't going to be much use for you!
 
 Some example analysis using {R}[http://www.r-project.org/] is included in the <tt>scripts</tt> folder.
+The inspiration for hacking away with R comes from reading Sau Sheong's new book
+{Exploring Everyday Things with Ruby and R}[http://www.bookjetty.com/books/1449315151/exploring-data-learning-everyday]. Check it out!
 
 == Requirements and Known Limitations
 
@@ -53,11 +58,14 @@ Here's the basic outline:
   - copy this to <tt>spec/fixtures/personal_pdf_samples/expectations.yml</tt>
   - enter in the details that describe each bill you have added
   - now when you run <tt>rake</tt> it will also verify the data extracted from your
-    bills using expectations.yml
+    bills using <tt>expectations.yml</tt>
 
 Feel free to get in touch or discuss in the github issues area if you are trying to help but run
 into problems with this!
 
+If you are more interested in the data analytics, I'm keen to add more interesting R scripts to the collection.
+Your contributions are most welcome.
+
 == Installation
 
   gem install sps_bill
@@ -152,14 +160,16 @@ in the <tt>scripts</tt> folder.
 === sample data and analysis
 
 [data/all_services.csv.sample]  sample CSV data for a years worth of elec, gas, and water
-[data/all_services.sample.pdf]  PDF analysis produced by this script for all_services.csv.sample
+[data/all_services.sample.pdf]  PDF analysis produced by <tt>full_analysis.R</tt> using
+                                the <tt>all_services.csv.sample</tt> data set.
 [data/elec_and_water_only.csv.sample]  sample CSV data for a years worth of elec and water
-[data/elec_and_water_only.sample.pdf]  PDF analysis produced by this script for elec_and_water_only.csv.sample
+[data/elec_and_water_only.sample.pdf]  PDF analysis produced by <tt>full_analysis.R</tt> using
+                                       the <tt>elec_and_water_only.csv.sample</tt> data set.
 
 === example run
 
   ./scan_all_bills.sh ../path_to_my_bills/*.pdf > my_bill_data.csv
-  ./full_analysis.R data_file.csv my_bill_data.csv
+  ./full_analysis.R my_bill_data.csv
 
 This will have produced an analysis of all your bills in <tt>full_analysis.pdf</tt>.
 

data/lib/sps_bill.rb

@@ -1,8 +1,4 @@
-require 'pdf-reader'
-require 'pdf/object_hash'
-require 'pdf/positional_text_receiver'
-require 'pdf/textangle'
-require 'pdf/structured_reader'
+require 'pdf-reader-turtletext'
 
 module SpsBill
 end

data/lib/sps_bill/bill.rb

@@ -10,15 +10,18 @@ class SpsBill::Bill
 
   # accessors for the various bill components
   #
-  # electricity_usage charges is an array of hashed values:
-  # [{ kwh: float, rate: float, amount: float }]
-  # gas_usage charges is an array of hashed values:
-  # [{ kwh: float, rate: float, amount: float }]
-  # water_usage charges is an array of hashed values:
-  # [{ cubic_m: float, rate: float, amount: float }]
-  #
+
   attr_reader :account_number,:total_amount,:invoice_date,:invoice_month
-  attr_reader :electricity_usage,:gas_usage,:water_usage
+
+  # electricity_usage is an array of hashed values:
+  #   [{ kwh: float, rate: float, amount: float }]
+  attr_reader :electricity_usage
+  # gas_usage is an array of hashed values:
+  #   [{ kwh: float, rate: float, amount: float }]
+  attr_reader :gas_usage
+  # water_usage is an array of hashed values:
+  #   [{ cubic_m: float, rate: float, amount: float }]
+  attr_reader :water_usage
 
   # +source+ is a file name or stream-like object
   def initialize(source)
@@ -28,7 +31,7 @@ class SpsBill::Bill
 
   # Returns the PDF reader isntance
   def reader
-    @reader ||= PDF::StructuredReader.new(source_file) if source_file
+    @reader ||= PDF::Reader::Turtletext.new(source_file) if source_file
   end
 
   # Return a pretty(-ish) text format of the core bill details

data/lib/sps_bill/bill_collection.rb

@@ -12,8 +12,8 @@ class SpsBill::BillCollection < Array
 
     # Returns an array of Bill objects for PDF files matching +path_spec+.
     # +path_spec+ may be either:
-    #  - an array of filenames e.g. ['data/file1.pdf','file2.pdf']
-    #  - or a single file or path spec e.g. './somepath/file1.pdf' or './somepath/*.pdf'
+    # - an array of filenames e.g. ['data/file1.pdf','file2.pdf']
+    # - or a single file or path spec e.g. './somepath/file1.pdf' or './somepath/*.pdf'
     def load(path_spec)
       path_spec = Dir[path_spec] unless path_spec.class <= Array
       path_spec.each_with_object(new) do |filename,memo|
@@ -23,30 +23,33 @@ class SpsBill::BillCollection < Array
 
   end
 
+  # Returns the suitable array of headers for +dataset_selector+
   def headers(dataset_selector)
     case dataset_selector
     when :total_amounts
-      ['invoice_month','amount']
+      %w(invoice_month amount)
     when :electricity_usages
-      ['invoice_month','kwh','rate','amount']
+      %w(invoice_month kwh rate amount)
     when :gas_usages
-      ['invoice_month','kwh','rate','amount']
+      %w(invoice_month kwh rate amount)
     when :water_usages
-      ['invoice_month','cubic_m','rate','amount']
+      %w(invoice_month cubic_m rate amount)
     when :all_data
-      ['invoice_month','measure','kwh','cubic_m','rate','amount']
+      %w(invoice_month measure kwh cubic_m rate amount)
     end
   end
 
-  # Returns a hash of all data by month
-  # [[month,measure,kwh,cubic_m,rate,amount]]
+  # Returns an array of all data by month
+  #   [[month,measure,kwh,cubic_m,rate,amount]]
   # measure: total_charges,electricity,gas,water
   def all_data
     total_amounts(:all) + electricity_usages(:all) + gas_usages(:all) + water_usages(:all)
   end
 
-  # Returns a hash of total bill amounts by month
-  # [[month,amount]]
+  # Returns an array of total bill amounts by month
+  #   [[month,amount]]
+  # when +style+ is :solo, returns minimal array to describe this data set in isolation,
+  # else returns a normalised sparse array that is common to all data sets
   def total_amounts(style=:solo)
     each_with_object([]) do |bill,memo|
       if style==:solo
@@ -57,8 +60,10 @@ class SpsBill::BillCollection < Array
     end
   end
 
-  # Returns a hash of electricity_usages by month
-  # [[month,kwh,rate,amount]]
+  # Returns an array of electricity_usages by month
+  #   [[month,kwh,rate,amount]]
+  # when +style+ is :solo, returns minimal array to describe this data set in isolation,
+  # else returns a normalised sparse array that is common to all data sets
   def electricity_usages(style=:solo)
     each_with_object([]) do |bill,memo|
       bill.electricity_usage.each do |usage|
@@ -71,8 +76,10 @@ class SpsBill::BillCollection < Array
     end
   end
 
-  # Returns a hash of gas_usages by month
-  # [[month,kwh,rate,amount]]
+  # Returns an array of gas_usages by month
+  #   [[month,kwh,rate,amount]]
+  # when +style+ is :solo, returns minimal array to describe this data set in isolation,
+  # else returns a normalised sparse array that is common to all data sets
   def gas_usages(style=:solo)
     each_with_object([]) do |bill,memo|
       bill.gas_usage.each do |usage|
@@ -85,8 +92,10 @@ class SpsBill::BillCollection < Array
     end
   end
 
-  # Returns a hash of water_usages by month
-  # [[month,kwh,rate,amount]]
+  # Returns an array of water_usages by month
+  #   [[month,kwh,rate,amount]]
+  # when +style+ is :solo, returns minimal array to describe this data set in isolation,
+  # else returns a normalised sparse array that is common to all data sets
   def water_usages(style=:solo)
     each_with_object([]) do |bill,memo|
       bill.water_usage.each do |usage|

data/lib/sps_bill/bill_parser.rb

@@ -3,9 +3,12 @@ require 'date'
 # all the bill scanning and parsing intelligence
 module SpsBill::BillParser
 
-  ELECTRICITY_SERVICE_HEAD = "Electricity Services"
-  GAS_SERVICE_HEAD = "Gas Services by City Gas Pte Ltd"
-  WATER_SERVICE_HEAD = "Water Services by Public Utilities Board"
+  ELECTRICITY_SERVICE_HEADER = /Electricity Services/i
+  ELECTRICITY_SERVICE_FOOTER = /Gas Services|Water Services/i
+  GAS_SERVICE_HEADER = /Gas Services/i
+  GAS_SERVICE_FOOTER = /Water Services/i
+  WATER_SERVICE_HEADER = /Water Services/i
+  WATER_SERVICE_FOOTER = /Waterborne Fee/i
 
   # Returns a collection of parser errors
   def errors
@@ -26,66 +29,109 @@ module SpsBill::BillParser
 
   # Command: extracts the account number
   def parse_account_number
-    @account_number = reader.text_in_rect(383.0,999.0,785.0,790.0,1).flatten.join('')
+    region = reader.bounding_box do
+      exclusive!
+      below 'Dated'
+      above 'Type'
+      right_of 'Account No'
+    end
+    # text will be returned like this:
+    #   [[":", "8123123123"]]
+    @account_number = region.text.flatten.last
   end
 
   # Command: extracts the total amount due for the current month
   def parse_total_amount
-    @total_amount = if ref = reader.text_position(/^Total Current Charges due on/)
-      total_parts = reader.text_in_rect(ref[:x] + 1,400.0,ref[:y] - 1,ref[:y] + 1,1)
-      total_parts.flatten.first.to_f
+    region = reader.bounding_box do
+      inclusive!
+      below /^Total Current Charges due on/
+      above /^Total Current Charges due on/
+      right_of /^Total Current Charges due on/
+      left_of 400.0
     end
+    # text will be returned like this:
+    #   [["Total Current Charges due on 14 Jun 2011 (Tue)", "251.44"]]
+    @total_amount = region.text.flatten.last.to_f
   end
 
   # Command: extracts the invoice date
   def parse_invoice_date
-    @invoice_date = if ref = reader.text_position("Dated")
-      date_parts = reader.text_in_rect(ref[:x] + 1,999.0,ref[:y] - 1,ref[:y] + 1,1)
-      Date.parse(date_parts.first.join('-'))
+    region = reader.bounding_box do
+      inclusive!
+      below 'Dated'
+      above 'Dated'
+      right_of 'Dated'
     end
+    # text will be returned like this:
+    #   [["Dated", "31", "May", "2011"]]
+    date_string = region.text.flatten.slice(1..3).join('-')
+    @invoice_date = Date.parse(date_string)
   end
 
   # Command: extracts the invoice month (as Date, set to 1st of the month)
   def parse_invoice_month
-    @invoice_month = if ref = reader.text_position("Dated")
-      date_parts = reader.text_in_rect(ref[:x] + 1,999.0,ref[:y] - 1,ref[:y] + 1,1)
-      m_parts = reader.text_in_rect(ref[:x]-200,ref[:x]-1,ref[:y] - 1,ref[:y] + 1,1)
-      Date.parse("#{date_parts.first.last}-#{m_parts.first.first}-01")
+    region = reader.bounding_box do
+      inclusive!
+      below 'Dated'
+      above 'Dated'
+    end
+    # text will be returned like this:
+    #   [["May", "11", "Bill", "Dated", "31", "May", "2011"]]
+    date_array = ['01'] + region.text.flatten.slice(0..1)
+    if (yy = date_array[2]).length == 2
+      date_array[2] = "20#{yy}" # WARNING: converting 2-digit date. Assumed to be 21st C
     end
+    @invoice_month = Date.parse(date_array.join('-'))
   end
 
-  # Command: extracts an array of electricity usage charges. Each charge is a Hash:
-  # { kwh: float, rate: float, amount: float }
+  # Command: extracts an array of electricity usage charges. Each element is a Hash:
+  #   { kwh: float, rate: float, amount: float }
   def parse_electricity_usage
-    @electricity_usage = if upper_ref = reader.text_position(ELECTRICITY_SERVICE_HEAD)
-      lower_ref = reader.text_position(GAS_SERVICE_HEAD)
-      lower_ref ||= reader.text_position(WATER_SERVICE_HEAD)
-      if lower_ref
-        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
-        raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
-      end
+    region = reader.bounding_box do
+      exclusive!
+      below ELECTRICITY_SERVICE_HEADER
+      above ELECTRICITY_SERVICE_FOOTER
+      right_of 240.0
+      left_of 450.0
+    end
+    # text will be returned like this:
+    #   [["4 kWh", "0.2410", "0.97"], ["616 kWh", "0.2558", "157.57"]]
+    @electricity_usage = unless (raw_data = region.text).empty?
+      raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
     end
   end
 
-  # Command: extracts an array of gas usage charges. Each charge is a Hash:
-  # { kwh: float, rate: float, amount: float }
+  # Command: extracts an array of gas usage charges. Each element is a Hash:
+  #   { kwh: float, rate: float, amount: float }
   def parse_gas_usage
-    @gas_usage = if upper_ref = reader.text_position(GAS_SERVICE_HEAD)
-      if lower_ref = reader.text_position(WATER_SERVICE_HEAD)
-        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
-        raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
-      end
+    region = reader.bounding_box do
+      exclusive!
+      below GAS_SERVICE_HEADER
+      above GAS_SERVICE_FOOTER
+      right_of 240.0
+      left_of 450.0
+    end
+    # text will be returned like this:
+    #   [["4 kWh", "0.2410", "0.97"], ["616 kWh", "0.2558", "157.57"]]
+    @gas_usage = unless (raw_data = region.text).empty?
+      raw_data.map{|l| {:kwh => l[0].gsub(/kwh/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
     end
   end
 
-  # Command: extracts an array of water usage charges. Each charge is a Hash:
-  # { cubic_m: float, rate: float, amount: float }
+  # Command: extracts an array of water usage charges. Each element is a Hash:
+  #   { cubic_m: float, rate: float, amount: float }
   def parse_water_usage
-    @water_usage = if upper_ref = reader.text_position(WATER_SERVICE_HEAD)
-      if lower_ref = reader.text_position("Waterborne Fee")
-        raw_data = reader.text_in_rect(240.0,450.0,lower_ref[:y]+1,upper_ref[:y],1)
-        raw_data.map{|l| {:cubic_m => l[0].gsub(/cu m/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
-      end
+    region = reader.bounding_box do
+      exclusive!
+      below WATER_SERVICE_HEADER
+      above WATER_SERVICE_FOOTER
+      right_of 240.0
+      left_of 450.0
+    end
+    # text will be returned like this:
+    #   [["36.1 Cu M", "1.1700", "42.24"], ["-3.0 Cu M", "1.4000", "-4.20"]]
+    @water_usage = unless (raw_data = region.text).empty?
+      raw_data.map{|l| {:cubic_m => l[0].gsub(/cu m/i,'').to_f, :rate => l[1].to_f, :amount => l[2].to_f} }
     end
   end
 

data/lib/sps_bill/shell.rb

@@ -12,13 +12,15 @@ SP Services Bill Scanner v#{SpsBill::Version::STRING}
 ===================================
 
 Usage:
-  sps_bill [options]
+  sps_bill [options] file-spec
 
 Command Options
   -r  | --raw    raw data format (without headers)
   -c  | --csv    output in CSV format (default)
   -d= | --data=[charges,electricity,gas,water,all]
 
+file-spec is a path to the PDF bill(s) to read.
+
     EOS
   end
 
@@ -29,7 +31,7 @@ Command Options
   end
 
   def run
-    if options[:help]
+    if options[:help] or fileset.empty?
       self.class.usage
       return
     end